diff options
Diffstat (limited to 'mysys')
87 files changed, 7240 insertions, 4881 deletions
diff --git a/mysys/CMakeLists.txt b/mysys/CMakeLists.txt index 3bed6f76fd7..704ddf8bb49 100644 --- a/mysys/CMakeLists.txt +++ b/mysys/CMakeLists.txt @@ -16,13 +16,14 @@ INCLUDE_DIRECTORIES(${ZLIB_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/include ${CMAKE_SOURCE_DIR}/mysys) -SET(MYSYS_SOURCES array.c charset-def.c charset.c checksum.c default.c - errors.c hash.c list.c md5.c mf_cache.c mf_dirname.c mf_fn_ext.c +SET(MYSYS_SOURCES array.c charset-def.c charset.c checksum.c my_default.c + errors.c hash.c list.c + mf_cache.c mf_dirname.c mf_fn_ext.c mf_format.c mf_getdate.c mf_iocache.c mf_iocache2.c mf_keycache.c mf_keycaches.c mf_loadpath.c mf_pack.c mf_path.c mf_qsort.c mf_qsort2.c mf_radix.c mf_same.c mf_sort.c mf_soundex.c mf_arr_appstr.c mf_tempdir.c mf_tempfile.c mf_unixpath.c mf_wcomp.c mulalloc.c my_access.c - my_aes.c my_alloc.c my_bit.c my_bitmap.c my_chsize.c + my_alloc.c my_bit.c my_bitmap.c my_chsize.c my_compress.c my_copy.c my_create.c my_delete.c my_div.c my_error.c my_file.c my_fopen.c my_fstream.c my_gethwaddr.c my_getopt.c my_getsystime.c my_getwd.c my_compare.c my_init.c @@ -32,13 +33,16 @@ SET(MYSYS_SOURCES array.c charset-def.c charset.c checksum.c default.c my_static.c my_symlink.c my_symlink2.c my_sync.c my_thr_init.c my_basename.c my_write.c ptr_cmp.c queues.c stacktrace.c - rijndael.c sha1.c string.c thr_alarm.c thr_lock.c thr_mutex.c - thr_rwlock.c tree.c typelib.c base64.c my_memmem.c my_getpagesize.c + string.c thr_alarm.c thr_lock.c thr_mutex.c + thr_rwlock.c thr_timer.c + tree.c typelib.c base64.c my_memmem.c + my_getpagesize.c lf_alloc-pin.c lf_dynarray.c lf_hash.c safemalloc.c my_new.cc my_atomic.c my_getncpus.c my_safehash.c my_chmod.c my_rnd.c my_uuid.c wqueue.c waiting_threads.c ma_dyncol.c - my_rdtsc.c my_context.c file_logger.c) + my_rdtsc.c my_context.c psi_noop.c + file_logger.c my_dlerror.c) IF (WIN32) SET (MYSYS_SOURCES ${MYSYS_SOURCES} my_winthread.c my_wincond.c my_winerr.c my_winfile.c my_windac.c my_conio.c) @@ -67,8 +71,8 @@ IF(HAVE_MLOCK) ENDIF() ADD_CONVENIENCE_LIBRARY(mysys ${MYSYS_SOURCES}) -TARGET_LINK_LIBRARIES(mysys dbug strings ${ZLIB_LIBRARY} - ${LIBNSL} ${LIBM} ${LIBRT} ${LIBSOCKET} ${LIBEXECINFO} ${LIBDL}) +TARGET_LINK_LIBRARIES(mysys dbug strings mysys_ssl ${ZLIB_LIBRARY} + ${LIBNSL} ${LIBM} ${LIBRT} ${LIBDL} ${LIBSOCKET} ${LIBEXECINFO}) DTRACE_INSTRUMENT(mysys) IF(HAVE_BFD_H) @@ -88,6 +92,10 @@ ADD_EXECUTABLE(thr_lock thr_lock.c) TARGET_LINK_LIBRARIES(thr_lock mysys) SET_TARGET_PROPERTIES(thr_lock PROPERTIES COMPILE_FLAGS "-DMAIN") +ADD_EXECUTABLE(thr_timer thr_timer.c) +TARGET_LINK_LIBRARIES(thr_timer mysys) +SET_TARGET_PROPERTIES(thr_timer PROPERTIES COMPILE_FLAGS "-DMAIN") + IF(MSVC) INSTALL_DEBUG_TARGET(mysys DESTINATION ${INSTALL_LIBDIR}/debug) ENDIF() diff --git a/mysys/array.c b/mysys/array.c index c969da83586..5548ca1b694 100644 --- a/mysys/array.c +++ b/mysys/array.c @@ -28,13 +28,13 @@ init_buffer Initial buffer pointer init_alloc Number of initial elements alloc_increment Increment for adding new elements + my_flags Flags to my_malloc DESCRIPTION init_dynamic_array() initiates array and allocate space for init_alloc eilements. Array is usable even if space allocation failed, hence, the function never returns TRUE. - Static buffers must begin immediately after the array structure. RETURN VALUE FALSE Ok @@ -42,12 +42,12 @@ my_bool init_dynamic_array2(DYNAMIC_ARRAY *array, uint element_size, void *init_buffer, uint init_alloc, - uint alloc_increment) + uint alloc_increment, myf my_flags) { DBUG_ENTER("init_dynamic_array2"); if (!alloc_increment) { - alloc_increment=max((8192-MALLOC_OVERHEAD)/element_size,16); + alloc_increment=MY_MAX((8192-MALLOC_OVERHEAD)/element_size,16); if (init_alloc > 8 && alloc_increment > init_alloc * 2) alloc_increment=init_alloc*2; } @@ -55,25 +55,24 @@ my_bool init_dynamic_array2(DYNAMIC_ARRAY *array, uint element_size, array->max_element=init_alloc; array->alloc_increment=alloc_increment; array->size_of_element=element_size; + array->malloc_flags= my_flags; + DBUG_ASSERT((my_flags & MY_INIT_BUFFER_USED) == 0); if ((array->buffer= init_buffer)) + { + array->malloc_flags|= MY_INIT_BUFFER_USED; DBUG_RETURN(FALSE); + } /* Since the dynamic array is usable even if allocation fails here malloc should not throw an error */ if (init_alloc && - !(array->buffer= (uchar*) my_malloc(element_size*init_alloc, MYF(0)))) + !(array->buffer= (uchar*) my_malloc(element_size*init_alloc, + MYF(my_flags)))) array->max_element=0; DBUG_RETURN(FALSE); } -my_bool init_dynamic_array(DYNAMIC_ARRAY *array, uint element_size, - uint init_alloc, uint alloc_increment) -{ - /* placeholder to preserve ABI */ - return my_init_dynamic_array_ci(array, element_size, init_alloc, - alloc_increment); -} /* Insert element at the end of array. Allocate memory if needed. @@ -87,11 +86,11 @@ my_bool init_dynamic_array(DYNAMIC_ARRAY *array, uint element_size, FALSE Ok */ -my_bool insert_dynamic(DYNAMIC_ARRAY *array, const uchar* element) +my_bool insert_dynamic(DYNAMIC_ARRAY *array, const void * element) { - uchar* buffer; + void *buffer; if (array->elements == array->max_element) - { /* Call only when nessesary */ + { /* Call only when necessary */ if (!(buffer=alloc_dynamic(array))) return TRUE; } @@ -122,31 +121,33 @@ my_bool insert_dynamic(DYNAMIC_ARRAY *array, const uchar* element) 0 Error */ -uchar *alloc_dynamic(DYNAMIC_ARRAY *array) +void *alloc_dynamic(DYNAMIC_ARRAY *array) { DBUG_ENTER("alloc_dynamic"); if (array->elements == array->max_element) { char *new_ptr; - if (array->buffer == (uchar *)(array + 1)) + if (array->malloc_flags & MY_INIT_BUFFER_USED) { /* - In this senerio, the buffer is statically preallocated, + In this scenario, the buffer is statically preallocated, so we have to create an all-new malloc since we overflowed */ if (!(new_ptr= (char *) my_malloc((array->max_element+ array->alloc_increment) * array->size_of_element, - MYF(MY_WME)))) + MYF(array->malloc_flags | MY_WME)))) DBUG_RETURN(0); memcpy(new_ptr, array->buffer, array->elements * array->size_of_element); + array->malloc_flags&= ~MY_INIT_BUFFER_USED; } else if (!(new_ptr=(char*) my_realloc(array->buffer,(array->max_element+ array->alloc_increment)* array->size_of_element, - MYF(MY_WME | MY_ALLOW_ZERO_PTR)))) + MYF(MY_WME | MY_ALLOW_ZERO_PTR | + array->malloc_flags)))) DBUG_RETURN(0); array->buffer= (uchar*) new_ptr; array->max_element+=array->alloc_increment; @@ -167,7 +168,7 @@ uchar *alloc_dynamic(DYNAMIC_ARRAY *array) 0 Array is empty */ -uchar *pop_dynamic(DYNAMIC_ARRAY *array) +void *pop_dynamic(DYNAMIC_ARRAY *array) { if (array->elements) return array->buffer+(--array->elements * array->size_of_element); @@ -192,7 +193,7 @@ uchar *pop_dynamic(DYNAMIC_ARRAY *array) FALSE Ok */ -my_bool set_dynamic(DYNAMIC_ARRAY *array, uchar* element, uint idx) +my_bool set_dynamic(DYNAMIC_ARRAY *array, const void *element, uint idx) { if (idx >= array->elements) { @@ -234,7 +235,7 @@ my_bool allocate_dynamic(DYNAMIC_ARRAY *array, uint max_elements) uchar *new_ptr; size= (max_elements + array->alloc_increment)/array->alloc_increment; size*= array->alloc_increment; - if (array->buffer == (uchar *)(array + 1)) + if (array->malloc_flags & MY_INIT_BUFFER_USED) { /* In this senerio, the buffer is statically preallocated, @@ -242,14 +243,16 @@ my_bool allocate_dynamic(DYNAMIC_ARRAY *array, uint max_elements) */ if (!(new_ptr= (uchar *) my_malloc(size * array->size_of_element, - MYF(MY_WME)))) + MYF(array->malloc_flags | MY_WME)))) DBUG_RETURN(0); memcpy(new_ptr, array->buffer, array->elements * array->size_of_element); - } + array->malloc_flags&= ~MY_INIT_BUFFER_USED; + } else if (!(new_ptr= (uchar*) my_realloc(array->buffer,size* array->size_of_element, - MYF(MY_WME | MY_ALLOW_ZERO_PTR)))) + MYF(MY_WME | MY_ALLOW_ZERO_PTR | + array->malloc_flags)))) DBUG_RETURN(TRUE); array->buffer= new_ptr; array->max_element= size; @@ -268,7 +271,7 @@ my_bool allocate_dynamic(DYNAMIC_ARRAY *array, uint max_elements) idx Index of element wanted. */ -void get_dynamic(DYNAMIC_ARRAY *array, uchar* element, uint idx) +void get_dynamic(DYNAMIC_ARRAY *array, void *element, uint idx) { if (idx >= array->elements) { @@ -295,15 +298,11 @@ void delete_dynamic(DYNAMIC_ARRAY *array) /* Just mark as empty if we are using a static buffer */ - if (array->buffer == (uchar *)(array + 1)) - array->elements= 0; - else - if (array->buffer) - { + if (!(array->malloc_flags & MY_INIT_BUFFER_USED) && array->buffer) my_free(array->buffer); - array->buffer=0; - array->elements=array->max_element=0; - } + + array->buffer= 0; + array->elements= array->max_element= 0; } /* @@ -323,7 +322,24 @@ void delete_dynamic_element(DYNAMIC_ARRAY *array, uint idx) (array->elements-idx)*array->size_of_element); } +/* + Wrapper around delete_dynamic, calling a FREE function on every + element, before releasing the memory + SYNOPSIS + delete_dynamic_with_callback() + array + f The function to be called on every element before + deleting the array; +*/ +void delete_dynamic_with_callback(DYNAMIC_ARRAY *array, FREE_FUNC f) { + uint i; + char *ptr= (char*) array->buffer; + for (i= 0; i < array->elements; i++, ptr+= array->size_of_element) { + f(ptr); + } + delete_dynamic(array); +} /* Free unused memory @@ -335,24 +351,25 @@ void delete_dynamic_element(DYNAMIC_ARRAY *array, uint idx) void freeze_size(DYNAMIC_ARRAY *array) { - uint elements=max(array->elements,1); + uint elements; /* Do nothing if we are using a static buffer */ - if (array->buffer == (uchar *)(array + 1)) + if (array->malloc_flags & MY_INIT_BUFFER_USED) return; - if (array->buffer && array->max_element != elements) + elements= MY_MAX(array->elements, 1); + if (array->buffer && array->max_element > elements) { array->buffer=(uchar*) my_realloc(array->buffer, - elements*array->size_of_element, - MYF(MY_WME)); - array->max_element=elements; + elements*array->size_of_element, + MYF(MY_WME | array->malloc_flags)); + array->max_element= elements; } } - +#ifdef NOT_USED /* Get the index of a dynamic element @@ -363,16 +380,17 @@ void freeze_size(DYNAMIC_ARRAY *array) */ -int get_index_dynamic(DYNAMIC_ARRAY *array, uchar* element) +int get_index_dynamic(DYNAMIC_ARRAY *array, void* element) { size_t ret; - if (array->buffer > element) + if (array->buffer > (uchar*) element) return -1; - ret= (element - array->buffer) / array->size_of_element; + ret= ((uchar*) element - array->buffer) / array->size_of_element; if (ret > array->elements) return -1; return ret; } +#endif diff --git a/mysys/base64.c b/mysys/base64.c index b48bcb85e03..04411418857 100644 --- a/mysys/base64.c +++ b/mysys/base64.c @@ -1,5 +1,5 @@ -/* Copyright (c) 2003-2008 MySQL AB, 2009 Sun Microsystems, Inc. - Use is subject to license terms. +/* Copyright (c) 2003, 2010, Oracle and/or its affiliates. + Copyright (c) 2013, MariaDB Foundation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -17,7 +17,6 @@ #include <my_global.h> #include <m_string.h> /* strchr() */ #include <m_ctype.h> /* my_isspace() */ -#include <base64.h> #ifndef MAIN @@ -25,6 +24,20 @@ static char base64_table[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" "abcdefghijklmnopqrstuvwxyz" "0123456789+/"; +/** + * Maximum length base64_needed_encoded_length() + * can handle without signed integer overflow. + */ +int +base64_encode_max_arg_length() +{ + /* + base64_needed_encoded_length(1589695686) -> 2147483646 (7FFFFFFE) + base64_needed_encoded_length(1589695687) -> -2147483645 + */ + return 0x5EC0D4C6; /* 1589695686 */ +} + int base64_needed_encoded_length(int length_of_data) @@ -39,10 +52,20 @@ base64_needed_encoded_length(int length_of_data) } +/** + * Maximum length supported by base64_decode(). + */ +int +base64_decode_max_arg_length() +{ + return 0x7FFFFFFF; +} + + int base64_needed_decoded_length(int length_of_encoded_data) { - return (int) ceil(length_of_encoded_data * 3 / 4); + return (int) ((longlong) length_of_encoded_data + 3) / 4 * 3; } @@ -51,6 +74,11 @@ base64_needed_decoded_length(int length_of_encoded_data) Note: We require that dst is pre-allocated to correct size. See base64_needed_encoded_length(). + + Note: We add line separators every 76 characters. + + Note: The output string is properly padded with the '=' character, + so the length of the output string is always divisable by 4. */ int @@ -101,130 +129,233 @@ base64_encode(const void *src, size_t src_len, char *dst) } -static inline uint -pos(unsigned char c) +/* + Base64 decoder stream +*/ +typedef struct my_base64_decoder_t { - return (uint) (strchr(base64_table, c) - base64_table); -} - - -#define SKIP_SPACE(src, i, size) \ -{ \ - while (i < size && my_isspace(&my_charset_latin1, * src)) \ - { \ - i++; \ - src++; \ - } \ - if (i == size) \ - { \ - break; \ - } \ -} + const char *src; /* Pointer to the current input position */ + const char *end; /* Pointer to the end of input buffer */ + uint c; /* Collect bits into this number */ + int error; /* Error code */ + uchar state; /* Character number in the current group of 4 */ + uchar mark; /* Number of padding marks in the current group */ +} MY_BASE64_DECODER; /* - Decode a base64 string - - SYNOPSIS - base64_decode() - src Pointer to base64-encoded string - len Length of string at 'src' - dst Pointer to location where decoded data will be stored - end_ptr Pointer to variable that will refer to the character - after the end of the encoded data that were decoded. Can - be NULL. - - DESCRIPTION - - The base64-encoded data in the range ['src','*end_ptr') will be - decoded and stored starting at 'dst'. The decoding will stop - after 'len' characters have been read from 'src', or when padding - occurs in the base64-encoded data. In either case: if 'end_ptr' is - non-null, '*end_ptr' will be set to point to the character after - the last read character, even in the presence of error. - - NOTE - We require that 'dst' is pre-allocated to correct size. - - SEE ALSO - base64_needed_decoded_length(). - - RETURN VALUE - Number of bytes written at 'dst' or -1 in case of failure + Helper table for decoder. + -2 means "space character" + -1 means "bad character" + Non-negative values mean valid base64 encoding character. */ -int -base64_decode(const char *src_base, size_t len, - void *dst, const char **end_ptr) +static int8 +from_base64_table[]= { - char b[3]; - size_t i= 0; - char *dst_base= (char *)dst; - char const *src= src_base; - char *d= dst_base; - size_t j; - - while (i < len) +/*00*/ -1,-1,-1,-1,-1,-1,-1,-1,-1,-2,-2,-2,-2,-2,-1,-1, +/*10*/ -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, +/*20*/ -2,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,62,-1,-1,-1,63, /* !"#$%&'()*+,-./ */ +/*30*/ 52,53,54,55,56,57,58,59,60,61,-1,-1,-1,-1,-1,-1, /* 0123456789:;<=>? */ +/*40*/ -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14, /* @ABCDEFGHIJKLMNO */ +/*50*/ 15,16,17,18,19,20,21,22,23,24,25,-1,-1,-1,-1,-1, /* PQRSTUVWXYZ[\]^_ */ +/*60*/ -1,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40, /* `abcdefghijklmno */ +/*70*/ 41,42,43,44,45,46,47,48,49,50,51,-1,-1,-1,-1,-1, /* pqrstuvwxyz{|}~ */ +/*80*/ -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, +/*90*/ -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, +/*A0*/ -2,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, +/*B0*/ -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, +/*C0*/ -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, +/*D0*/ -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, +/*E0*/ -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, +/*F0*/ -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1 +}; + + +/** + * Skip leading spaces in a base64 encoded stream + * and stop on the first non-space character. + * decoder->src will point to the first non-space character, + * or to the end of the input string. + * In case when end-of-input met on unexpected position, + * decoder->error is also set to 1. + * + * See http://en.wikipedia.org/wiki/Base64 for the base64 encoding details + * + * @param decoder Pointer to MY_BASE64_DECODER + * + * @return + * FALSE on success (there are some more non-space input characters) + * TRUE on error (end-of-input found) + */ + +static inline my_bool +my_base64_decoder_skip_spaces(MY_BASE64_DECODER *decoder) +{ + for ( ; decoder->src < decoder->end; decoder->src++) { - unsigned c= 0; - size_t mark= 0; + if (from_base64_table[(uchar) *decoder->src] != -2) + return FALSE; + } + if (decoder->state > 0) + decoder->error= 1; /* Unexpected end-of-input found */ + return TRUE; +} - SKIP_SPACE(src, i, len); - c += pos(*src++); - c <<= 6; - i++; +/** + * Convert the next character in a base64 encoded stream + * to a number in the range [0..63] + * and mix it with the previously collected value in decoder->c. + * + * @param decode base64 decoding stream + * + * @return + * FALSE on success + * TRUE on error (invalid base64 character found) + */ +static inline my_bool +my_base64_add(MY_BASE64_DECODER *decoder) +{ + int res; + decoder->c <<= 6; + if ((res= from_base64_table[(uchar) *decoder->src++]) < 0) + return (decoder->error= TRUE); + decoder->c+= (uint) res; + return FALSE; +} - SKIP_SPACE(src, i, len); - c += pos(*src++); - c <<= 6; - i++; - - SKIP_SPACE(src, i, len); +/** + * Get the next character from a base64 encoded stream. + * Skip spaces, then scan the next base64 character or a pad character + * and collect bits into decoder->c. + * + * @param decoder Pointer to MY_BASE64_DECODER + * @return + * FALSE on success (a valid base64 encoding character found) + * TRUE on error (unexpected character or unexpected end-of-input found) + */ +static my_bool +my_base64_decoder_getch(MY_BASE64_DECODER *decoder) +{ + if (my_base64_decoder_skip_spaces(decoder)) + return TRUE; /* End-of-input */ - if (*src != '=') - c += pos(*src++); - else + if (!my_base64_add(decoder)) /* Valid base64 character found */ + { + if (decoder->mark) { - src += 2; /* There should be two bytes padding */ - i= len; - mark= 2; - c <<= 6; - goto end; + /* If we have scanned '=' already, then only '=' is valid */ + DBUG_ASSERT(decoder->state == 3); + decoder->error= 1; + decoder->src--; + return TRUE; /* expected '=', but encoding character found */ } - c <<= 6; - i++; - - SKIP_SPACE(src, i, len); + decoder->state++; + return FALSE; + } - if (*src != '=') - c += pos(*src++); + /* Process error */ + switch (decoder->state) + { + case 0: + case 1: + decoder->src--; + return TRUE; /* base64 character expected */ + break; + + case 2: + case 3: + if (decoder->src[-1] == '=') + { + decoder->error= 0; /* Not an error - it's a pad character */ + decoder->mark++; + } else { - src += 1; /* There should be one byte padding */ - i= len; - mark= 1; - goto end; + decoder->src--; + return TRUE; /* base64 character or '=' expected */ } - i++; + break; + + default: + DBUG_ASSERT(0); + return TRUE; /* Wrong state, should not happen */ + } + + decoder->state++; + return FALSE; +} + + +/** + * Decode a base64 string + * The base64-encoded data in the range ['src','*end_ptr') will be + * decoded and stored starting at 'dst'. The decoding will stop + * after 'len' characters have been read from 'src', or when padding + * occurs in the base64-encoded data. In either case: if 'end_ptr' is + * non-null, '*end_ptr' will be set to point to the character after + * the last read character, even in the presence of error. + * + * Note: 'dst' must have sufficient space to store the decoded data. + * Use base64_needed_decoded_length() to calculate the correct space size. + * + * Note: we allow spaces and line separators at any position. + * + * @param src Pointer to base64-encoded string + * @param len Length of string at 'src' + * @param dst Pointer to location where decoded data will be stored + * @param end_ptr Pointer to variable that will refer to the character + * after the end of the encoded data that were decoded. + * Can be NULL. + * @flags flags e.g. allow multiple chunks + * @return Number of bytes written at 'dst', or -1 in case of failure + */ +int +base64_decode(const char *src_base, size_t len, + void *dst, const char **end_ptr, int flags) +{ + char *d= (char*) dst; + MY_BASE64_DECODER decoder; + + decoder.src= src_base; + decoder.end= src_base + len; + decoder.error= 0; + decoder.mark= 0; - end: - b[0]= (c >> 16) & 0xff; - b[1]= (c >> 8) & 0xff; - b[2]= (c >> 0) & 0xff; + for ( ; ; ) + { + decoder.c= 0; + decoder.state= 0; + + if (my_base64_decoder_getch(&decoder) || + my_base64_decoder_getch(&decoder) || + my_base64_decoder_getch(&decoder) || + my_base64_decoder_getch(&decoder)) + break; - for (j=0; j<3-mark; j++) - *d++= b[j]; + *d++= (decoder.c >> 16) & 0xff; + *d++= (decoder.c >> 8) & 0xff; + *d++= (decoder.c >> 0) & 0xff; + + if (decoder.mark) + { + d-= decoder.mark; + if (!(flags & MY_BASE64_DECODE_ALLOW_MULTIPLE_CHUNKS)) + break; + decoder.mark= 0; + } } + /* Return error if there are more non-space characters */ + decoder.state= 0; + if (!my_base64_decoder_skip_spaces(&decoder)) + decoder.error= 1; + if (end_ptr != NULL) - *end_ptr= src; + *end_ptr= decoder.src; - /* - The variable 'i' is set to 'len' when padding has been read, so it - does not actually reflect the number of bytes read from 'src'. - */ - return i != len ? -1 : (int) (d - dst_base); + return decoder.error ? -1 : (int) (d - (char*) dst); } diff --git a/mysys/charset-def.c b/mysys/charset-def.c index e9f2ecdea49..defb5c0062d 100644 --- a/mysys/charset-def.c +++ b/mysys/charset-def.c @@ -24,6 +24,7 @@ #ifdef HAVE_UCA_COLLATIONS #ifdef HAVE_CHARSET_ucs2 +extern struct charset_info_st my_charset_ucs2_german2_uca_ci; extern struct charset_info_st my_charset_ucs2_icelandic_uca_ci; extern struct charset_info_st my_charset_ucs2_latvian_uca_ci; extern struct charset_info_st my_charset_ucs2_romanian_uca_ci; @@ -42,12 +43,18 @@ extern struct charset_info_st my_charset_ucs2_roman_uca_ci; extern struct charset_info_st my_charset_ucs2_persian_uca_ci; extern struct charset_info_st my_charset_ucs2_esperanto_uca_ci; extern struct charset_info_st my_charset_ucs2_hungarian_uca_ci; +extern struct charset_info_st my_charset_ucs2_croatian_mysql561_uca_ci; extern struct charset_info_st my_charset_ucs2_sinhala_uca_ci; +extern struct charset_info_st my_charset_ucs2_unicode_520_ci; +extern struct charset_info_st my_charset_ucs2_vietnamese_ci; extern struct charset_info_st my_charset_ucs2_croatian_uca_ci; +extern struct charset_info_st my_charset_ucs2_myanmar_uca_ci; +extern struct charset_info_st my_charset_ucs2_thai_520_w2; #endif #ifdef HAVE_CHARSET_utf32 +extern struct charset_info_st my_charset_utf32_german2_uca_ci; extern struct charset_info_st my_charset_utf32_icelandic_uca_ci; extern struct charset_info_st my_charset_utf32_latvian_uca_ci; extern struct charset_info_st my_charset_utf32_romanian_uca_ci; @@ -66,12 +73,18 @@ extern struct charset_info_st my_charset_utf32_roman_uca_ci; extern struct charset_info_st my_charset_utf32_persian_uca_ci; extern struct charset_info_st my_charset_utf32_esperanto_uca_ci; extern struct charset_info_st my_charset_utf32_hungarian_uca_ci; +extern struct charset_info_st my_charset_utf32_croatian_mysql561_uca_ci; extern struct charset_info_st my_charset_utf32_sinhala_uca_ci; +extern struct charset_info_st my_charset_utf32_unicode_520_ci; +extern struct charset_info_st my_charset_utf32_vietnamese_ci; extern struct charset_info_st my_charset_utf32_croatian_uca_ci; +extern struct charset_info_st my_charset_utf32_myanmar_uca_ci; +extern struct charset_info_st my_charset_utf32_thai_520_w2; #endif /* HAVE_CHARSET_utf32 */ #ifdef HAVE_CHARSET_utf16 +extern struct charset_info_st my_charset_utf16_german2_uca_ci; extern struct charset_info_st my_charset_utf16_icelandic_uca_ci; extern struct charset_info_st my_charset_utf16_latvian_uca_ci; extern struct charset_info_st my_charset_utf16_romanian_uca_ci; @@ -90,12 +103,18 @@ extern struct charset_info_st my_charset_utf16_roman_uca_ci; extern struct charset_info_st my_charset_utf16_persian_uca_ci; extern struct charset_info_st my_charset_utf16_esperanto_uca_ci; extern struct charset_info_st my_charset_utf16_hungarian_uca_ci; +extern struct charset_info_st my_charset_utf16_croatian_mysql561_uca_ci; extern struct charset_info_st my_charset_utf16_sinhala_uca_ci; +extern struct charset_info_st my_charset_utf16_unicode_520_ci; +extern struct charset_info_st my_charset_utf16_vietnamese_ci; extern struct charset_info_st my_charset_utf16_croatian_uca_ci; +extern struct charset_info_st my_charset_utf16_myanmar_uca_ci; +extern struct charset_info_st my_charset_utf16_thai_520_w2; #endif /* HAVE_CHARSET_utf16 */ #ifdef HAVE_CHARSET_utf8 +extern struct charset_info_st my_charset_utf8_german2_uca_ci; extern struct charset_info_st my_charset_utf8_icelandic_uca_ci; extern struct charset_info_st my_charset_utf8_latvian_uca_ci; extern struct charset_info_st my_charset_utf8_romanian_uca_ci; @@ -114,14 +133,20 @@ extern struct charset_info_st my_charset_utf8_roman_uca_ci; extern struct charset_info_st my_charset_utf8_persian_uca_ci; extern struct charset_info_st my_charset_utf8_esperanto_uca_ci; extern struct charset_info_st my_charset_utf8_hungarian_uca_ci; +extern struct charset_info_st my_charset_utf8_croatian_mysql561_uca_ci; extern struct charset_info_st my_charset_utf8_sinhala_uca_ci; +extern struct charset_info_st my_charset_utf8_unicode_520_ci; +extern struct charset_info_st my_charset_utf8_vietnamese_ci; extern struct charset_info_st my_charset_utf8_croatian_uca_ci; +extern struct charset_info_st my_charset_utf8_myanmar_uca_ci; +extern struct charset_info_st my_charset_utf8_thai_520_w2; #ifdef HAVE_UTF8_GENERAL_CS extern struct charset_info_st my_charset_utf8_general_cs; #endif #endif #ifdef HAVE_CHARSET_utf8mb4 +extern struct charset_info_st my_charset_utf8mb4_german2_uca_ci; extern struct charset_info_st my_charset_utf8mb4_icelandic_uca_ci; extern struct charset_info_st my_charset_utf8mb4_latvian_uca_ci; extern struct charset_info_st my_charset_utf8mb4_romanian_uca_ci; @@ -140,8 +165,13 @@ extern struct charset_info_st my_charset_utf8mb4_roman_uca_ci; extern struct charset_info_st my_charset_utf8mb4_persian_uca_ci; extern struct charset_info_st my_charset_utf8mb4_esperanto_uca_ci; extern struct charset_info_st my_charset_utf8mb4_hungarian_uca_ci; +extern struct charset_info_st my_charset_utf8mb4_croatian_mysql561_uca_ci; extern struct charset_info_st my_charset_utf8mb4_sinhala_uca_ci; +extern struct charset_info_st my_charset_utf8mb4_unicode_520_ci; +extern struct charset_info_st my_charset_utf8mb4_vietnamese_ci; extern struct charset_info_st my_charset_utf8mb4_croatian_uca_ci; +extern struct charset_info_st my_charset_utf8mb4_myanmar_uca_ci; +extern struct charset_info_st my_charset_utf8mb4_thai_520_w2; #endif /* HAVE_CHARSET_utf8mb4 */ #endif /* HAVE_UCA_COLLATIONS */ @@ -211,6 +241,7 @@ my_bool init_compiled_charsets(myf flags __attribute__((unused))) add_compiled_collation(&my_charset_ucs2_general_mysql500_ci); #ifdef HAVE_UCA_COLLATIONS add_compiled_collation(&my_charset_ucs2_unicode_ci); + add_compiled_collation(&my_charset_ucs2_german2_uca_ci); add_compiled_collation(&my_charset_ucs2_icelandic_uca_ci); add_compiled_collation(&my_charset_ucs2_latvian_uca_ci); add_compiled_collation(&my_charset_ucs2_romanian_uca_ci); @@ -229,8 +260,13 @@ my_bool init_compiled_charsets(myf flags __attribute__((unused))) add_compiled_collation(&my_charset_ucs2_persian_uca_ci); add_compiled_collation(&my_charset_ucs2_esperanto_uca_ci); add_compiled_collation(&my_charset_ucs2_hungarian_uca_ci); + add_compiled_collation(&my_charset_ucs2_croatian_mysql561_uca_ci); add_compiled_collation(&my_charset_ucs2_sinhala_uca_ci); + add_compiled_collation(&my_charset_ucs2_unicode_520_ci); + add_compiled_collation(&my_charset_ucs2_vietnamese_ci); add_compiled_collation(&my_charset_ucs2_croatian_uca_ci); + add_compiled_collation(&my_charset_ucs2_myanmar_uca_ci); + add_compiled_collation(&my_charset_ucs2_thai_520_w2); #endif #endif @@ -248,6 +284,7 @@ my_bool init_compiled_charsets(myf flags __attribute__((unused))) #endif #ifdef HAVE_UCA_COLLATIONS add_compiled_collation(&my_charset_utf8_unicode_ci); + add_compiled_collation(&my_charset_utf8_german2_uca_ci); add_compiled_collation(&my_charset_utf8_icelandic_uca_ci); add_compiled_collation(&my_charset_utf8_latvian_uca_ci); add_compiled_collation(&my_charset_utf8_romanian_uca_ci); @@ -266,8 +303,13 @@ my_bool init_compiled_charsets(myf flags __attribute__((unused))) add_compiled_collation(&my_charset_utf8_persian_uca_ci); add_compiled_collation(&my_charset_utf8_esperanto_uca_ci); add_compiled_collation(&my_charset_utf8_hungarian_uca_ci); + add_compiled_collation(&my_charset_utf8_croatian_mysql561_uca_ci); add_compiled_collation(&my_charset_utf8_sinhala_uca_ci); + add_compiled_collation(&my_charset_utf8_unicode_520_ci); + add_compiled_collation(&my_charset_utf8_vietnamese_ci); add_compiled_collation(&my_charset_utf8_croatian_uca_ci); + add_compiled_collation(&my_charset_utf8_myanmar_uca_ci); + add_compiled_collation(&my_charset_utf8_thai_520_w2); #endif #endif /* HAVE_CHARSET_utf8 */ @@ -277,6 +319,7 @@ my_bool init_compiled_charsets(myf flags __attribute__((unused))) add_compiled_collation(&my_charset_utf8mb4_bin); #ifdef HAVE_UCA_COLLATIONS add_compiled_collation(&my_charset_utf8mb4_unicode_ci); + add_compiled_collation(&my_charset_utf8mb4_german2_uca_ci); add_compiled_collation(&my_charset_utf8mb4_icelandic_uca_ci); add_compiled_collation(&my_charset_utf8mb4_latvian_uca_ci); add_compiled_collation(&my_charset_utf8mb4_romanian_uca_ci); @@ -295,8 +338,13 @@ my_bool init_compiled_charsets(myf flags __attribute__((unused))) add_compiled_collation(&my_charset_utf8mb4_persian_uca_ci); add_compiled_collation(&my_charset_utf8mb4_esperanto_uca_ci); add_compiled_collation(&my_charset_utf8mb4_hungarian_uca_ci); + add_compiled_collation(&my_charset_utf8mb4_croatian_mysql561_uca_ci); add_compiled_collation(&my_charset_utf8mb4_sinhala_uca_ci); + add_compiled_collation(&my_charset_utf8mb4_unicode_520_ci); + add_compiled_collation(&my_charset_utf8mb4_vietnamese_ci); add_compiled_collation(&my_charset_utf8mb4_croatian_uca_ci); + add_compiled_collation(&my_charset_utf8mb4_myanmar_uca_ci); + add_compiled_collation(&my_charset_utf8mb4_thai_520_w2); #endif /* HAVE_UCA_COLLATIONS */ #endif /* HAVE_CHARSET_utf8mb4 */ @@ -304,8 +352,11 @@ my_bool init_compiled_charsets(myf flags __attribute__((unused))) #ifdef HAVE_CHARSET_utf16 add_compiled_collation(&my_charset_utf16_general_ci); add_compiled_collation(&my_charset_utf16_bin); + add_compiled_collation(&my_charset_utf16le_general_ci); + add_compiled_collation(&my_charset_utf16le_bin); #ifdef HAVE_UCA_COLLATIONS add_compiled_collation(&my_charset_utf16_unicode_ci); + add_compiled_collation(&my_charset_utf16_german2_uca_ci); add_compiled_collation(&my_charset_utf16_icelandic_uca_ci); add_compiled_collation(&my_charset_utf16_latvian_uca_ci); add_compiled_collation(&my_charset_utf16_romanian_uca_ci); @@ -324,8 +375,13 @@ my_bool init_compiled_charsets(myf flags __attribute__((unused))) add_compiled_collation(&my_charset_utf16_persian_uca_ci); add_compiled_collation(&my_charset_utf16_esperanto_uca_ci); add_compiled_collation(&my_charset_utf16_hungarian_uca_ci); + add_compiled_collation(&my_charset_utf16_croatian_mysql561_uca_ci); add_compiled_collation(&my_charset_utf16_sinhala_uca_ci); + add_compiled_collation(&my_charset_utf16_unicode_520_ci); + add_compiled_collation(&my_charset_utf16_vietnamese_ci); add_compiled_collation(&my_charset_utf16_croatian_uca_ci); + add_compiled_collation(&my_charset_utf16_myanmar_uca_ci); + add_compiled_collation(&my_charset_utf16_thai_520_w2); #endif /* HAVE_UCA_COLLATIONS */ #endif /* HAVE_CHARSET_utf16 */ @@ -335,6 +391,7 @@ my_bool init_compiled_charsets(myf flags __attribute__((unused))) add_compiled_collation(&my_charset_utf32_bin); #ifdef HAVE_UCA_COLLATIONS add_compiled_collation(&my_charset_utf32_unicode_ci); + add_compiled_collation(&my_charset_utf32_german2_uca_ci); add_compiled_collation(&my_charset_utf32_icelandic_uca_ci); add_compiled_collation(&my_charset_utf32_latvian_uca_ci); add_compiled_collation(&my_charset_utf32_romanian_uca_ci); @@ -353,8 +410,13 @@ my_bool init_compiled_charsets(myf flags __attribute__((unused))) add_compiled_collation(&my_charset_utf32_persian_uca_ci); add_compiled_collation(&my_charset_utf32_esperanto_uca_ci); add_compiled_collation(&my_charset_utf32_hungarian_uca_ci); + add_compiled_collation(&my_charset_utf32_croatian_mysql561_uca_ci); add_compiled_collation(&my_charset_utf32_sinhala_uca_ci); + add_compiled_collation(&my_charset_utf32_unicode_520_ci); + add_compiled_collation(&my_charset_utf32_vietnamese_ci); add_compiled_collation(&my_charset_utf32_croatian_uca_ci); + add_compiled_collation(&my_charset_utf32_myanmar_uca_ci); + add_compiled_collation(&my_charset_utf32_thai_520_w2); #endif /* HAVE_UCA_COLLATIONS */ #endif /* HAVE_CHARSET_utf32 */ diff --git a/mysys/charset.c b/mysys/charset.c index cc1d0a0111e..ad3eb78ae0e 100644 --- a/mysys/charset.c +++ b/mysys/charset.c @@ -214,6 +214,8 @@ copy_uca_collation(struct charset_info_st *to, CHARSET_INFO *from) to->max_sort_char= from->max_sort_char; to->mbminlen= from->mbminlen; to->mbmaxlen= from->mbmaxlen; + to->caseup_multiply= from->caseup_multiply; + to->casedn_multiply= from->casedn_multiply; to->state|= MY_CS_AVAILABLE | MY_CS_LOADED | MY_CS_STRNXFRM | MY_CS_UNICODE; } @@ -249,7 +251,8 @@ static int add_collation(struct charset_info_st *cs) return MY_XML_ERROR; newcs->caseup_multiply= newcs->casedn_multiply= 1; - + newcs->levels_for_order= 1; + if (!strcmp(cs->csname,"ucs2") ) { #if defined(HAVE_CHARSET_ucs2) && defined(HAVE_UCA_COLLATIONS) @@ -294,6 +297,7 @@ static int add_collation(struct charset_info_st *cs) simple_cs_init_functions(newcs); newcs->mbminlen= 1; newcs->mbmaxlen= 1; + newcs->strxfrm_multiply= 1; if (simple_cs_is_full(newcs)) { newcs->state |= MY_CS_LOADED; @@ -350,13 +354,65 @@ static int add_collation(struct charset_info_st *cs) } +/** + Report character set initialization errors and warnings. + Be silent by default: no warnings on the client side. +*/ +static void +default_reporter(enum loglevel level __attribute__ ((unused)), + const char *format __attribute__ ((unused)), + ...) +{ +} +my_error_reporter my_charset_error_reporter= default_reporter; + + +/** + Wrappers for memory functions my_malloc (and friends) + with C-compatbile API without extra "myf" argument. +*/ +static void * +my_once_alloc_c(size_t size) +{ return my_once_alloc(size, MYF(MY_WME)); } + + +static void * +my_malloc_c(size_t size) +{ return my_malloc(size, MYF(MY_WME)); } + + +static void * +my_realloc_c(void *old, size_t size) +{ return my_realloc(old, size, MYF(MY_WME|MY_ALLOW_ZERO_PTR)); } + + +/** + Initialize character set loader to use mysys memory management functions. + @param loader Loader to initialize +*/ +void +my_charset_loader_init_mysys(MY_CHARSET_LOADER *loader) +{ + loader->error[0]= '\0'; + loader->once_alloc= my_once_alloc_c; + loader->malloc= my_malloc_c; + loader->realloc= my_realloc_c; + loader->free= my_free; + loader->reporter= my_charset_error_reporter; + loader->add_collation= add_collation; +} + + #define MY_MAX_ALLOWED_BUF 1024*1024 #define MY_CHARSET_INDEX "Index.xml" const char *charsets_dir= NULL; -static my_bool my_read_charset_file(const char *filename, myf myflags) +static my_bool +my_read_charset_file(MY_CHARSET_LOADER *loader, + const char *filename, + myf myflags) { uchar *buf; int fd; @@ -375,14 +431,11 @@ static my_bool my_read_charset_file(const char *filename, myf myflags) if (tmp_len != len) goto error; - if (my_parse_charset_xml((char*) buf,len,add_collation)) + if (my_parse_charset_xml(loader, (char *) buf, len)) { -#ifdef NOT_YET - printf("ERROR at line %d pos %d '%s'\n", - my_xml_error_lineno(&p)+1, - my_xml_error_pos(&p), - my_xml_error_string(&p)); -#endif + my_printf_error(EE_UNKNOWN_CHARSET, "Error while parsing '%s': %s\n", + MYF(0), filename, loader->error); + goto error; } my_free(buf); @@ -426,21 +479,62 @@ void add_compiled_collation(struct charset_info_st *cs) cs->state|= MY_CS_AVAILABLE; } -static void *cs_alloc(size_t size) + +static my_pthread_once_t charsets_initialized= MY_PTHREAD_ONCE_INIT; +static my_pthread_once_t charsets_template= MY_PTHREAD_ONCE_INIT; + +typedef struct { - return my_once_alloc(size, MYF(MY_WME)); + ulonglong use_count; +} MY_COLLATION_STATISTICS; + + +static MY_COLLATION_STATISTICS my_collation_statistics[MY_ALL_CHARSETS_SIZE]; + + +my_bool my_collation_is_known_id(uint id) +{ + return id > 0 && id < array_elements(all_charsets) && all_charsets[id] ? + TRUE : FALSE; } -static my_pthread_once_t charsets_initialized= MY_PTHREAD_ONCE_INIT; -static my_pthread_once_t charsets_template= MY_PTHREAD_ONCE_INIT; +/* + Collation use statistics functions do not lock + counters to avoid mutex contention. This can lose + some counter increments with high thread concurrency. + But this should be Ok, as we don't need exact numbers. +*/ +static inline void my_collation_statistics_inc_use_count(uint id) +{ + DBUG_ASSERT(my_collation_is_known_id(id)); + my_collation_statistics[id].use_count++; +} + + +ulonglong my_collation_statistics_get_use_count(uint id) +{ + DBUG_ASSERT(my_collation_is_known_id(id)); + return my_collation_statistics[id].use_count; +} + + +const char *my_collation_get_tailoring(uint id) +{ + /* all_charsets[id]->tailoring is never changed after server startup. */ + DBUG_ASSERT(my_collation_is_known_id(id)); + return all_charsets[id]->tailoring; +} + static void init_available_charsets(void) { char fname[FN_REFLEN + sizeof(MY_CHARSET_INDEX)]; struct charset_info_st **cs; + MY_CHARSET_LOADER loader; bzero((char*) &all_charsets,sizeof(all_charsets)); + bzero((char*) &my_collation_statistics, sizeof(my_collation_statistics)); init_compiled_charsets(MYF(0)); /* Copy compiled charsets */ @@ -457,8 +551,9 @@ static void init_available_charsets(void) } } + my_charset_loader_init_mysys(&loader); strmov(get_charsets_dir(fname), MY_CHARSET_INDEX); - my_read_charset_file(fname, MYF(0)); + my_read_charset_file(&loader, fname, MYF(0)); } @@ -547,7 +642,8 @@ const char *get_charset_name(uint charset_number) } -static CHARSET_INFO *get_internal_charset(uint cs_number, myf flags) +static CHARSET_INFO * +get_internal_charset(MY_CHARSET_LOADER *loader, uint cs_number, myf flags) { char buf[FN_REFLEN]; struct charset_info_st *cs; @@ -557,7 +653,10 @@ static CHARSET_INFO *get_internal_charset(uint cs_number, myf flags) if ((cs= (struct charset_info_st*) all_charsets[cs_number])) { if (cs->state & MY_CS_READY) /* if CS is already initialized */ - return cs; + { + my_collation_statistics_inc_use_count(cs_number); + return cs; + } /* To make things thread safe we are not allowing other threads to interfere @@ -567,20 +666,25 @@ static CHARSET_INFO *get_internal_charset(uint cs_number, myf flags) if (!(cs->state & (MY_CS_COMPILED|MY_CS_LOADED))) /* if CS is not in memory */ { + MY_CHARSET_LOADER loader; strxmov(get_charsets_dir(buf), cs->csname, ".xml", NullS); - my_read_charset_file(buf,flags); + my_charset_loader_init_mysys(&loader); + my_read_charset_file(&loader, buf, flags); } if (cs->state & MY_CS_AVAILABLE) { if (!(cs->state & MY_CS_READY)) { - if ((cs->cset->init && cs->cset->init(cs, cs_alloc)) || - (cs->coll->init && cs->coll->init(cs, cs_alloc))) + if ((cs->cset->init && cs->cset->init(cs, loader)) || + (cs->coll->init && cs->coll->init(cs, loader))) + { cs= NULL; + } else cs->state|= MY_CS_READY; } + my_collation_statistics_inc_use_count(cs_number); } else cs= NULL; @@ -593,16 +697,19 @@ static CHARSET_INFO *get_internal_charset(uint cs_number, myf flags) CHARSET_INFO *get_charset(uint cs_number, myf flags) { - CHARSET_INFO *cs; + CHARSET_INFO *cs= NULL; + if (cs_number == default_charset_info->number) return default_charset_info; my_pthread_once(&charsets_initialized, init_available_charsets); - - if (cs_number >= array_elements(all_charsets)) - return NULL; - - cs=get_internal_charset(cs_number, flags); + + if (cs_number < array_elements(all_charsets)) + { + MY_CHARSET_LOADER loader; + my_charset_loader_init_mysys(&loader); + cs= get_internal_charset(&loader, cs_number, flags); + } if (!cs && (flags & MY_WME)) { @@ -615,29 +722,58 @@ CHARSET_INFO *get_charset(uint cs_number, myf flags) return cs; } -CHARSET_INFO *get_charset_by_name(const char *cs_name, myf flags) + +/** + Find collation by name: extended version of get_charset_by_name() + to return error messages to the caller. + @param loader Character set loader + @param name Collation name + @param flags Flags + @return NULL on error, pointer to collation on success +*/ + +CHARSET_INFO * +my_collation_get_by_name(MY_CHARSET_LOADER *loader, + const char *name, myf flags) { uint cs_number; CHARSET_INFO *cs; my_pthread_once(&charsets_initialized, init_available_charsets); - cs_number=get_collation_number(cs_name); - cs= cs_number ? get_internal_charset(cs_number,flags) : NULL; + cs_number= get_collation_number(name); + my_charset_loader_init_mysys(loader); + cs= cs_number ? get_internal_charset(loader, cs_number, flags) : NULL; if (!cs && (flags & MY_WME)) { char index_file[FN_REFLEN + sizeof(MY_CHARSET_INDEX)]; strmov(get_charsets_dir(index_file),MY_CHARSET_INDEX); - my_error(EE_UNKNOWN_COLLATION, MYF(ME_BELL), cs_name, index_file); + my_error(EE_UNKNOWN_COLLATION, MYF(ME_BELL), name, index_file); } - return cs; } -CHARSET_INFO *get_charset_by_csname(const char *cs_name, - uint cs_flags, - myf flags) +CHARSET_INFO *get_charset_by_name(const char *cs_name, myf flags) +{ + MY_CHARSET_LOADER loader; + my_charset_loader_init_mysys(&loader); + return my_collation_get_by_name(&loader, cs_name, flags); +} + + +/** + Find character set by name: extended version of get_charset_by_csname() + to return error messages to the caller. + @param loader Character set loader + @param name Collation name + @param cs_flags Character set flags (e.g. default or binary collation) + @param flags Flags + @return NULL on error, pointer to collation on success +*/ +CHARSET_INFO * +my_charset_get_by_name(MY_CHARSET_LOADER *loader, + const char *cs_name, uint cs_flags, myf flags) { uint cs_number; CHARSET_INFO *cs; @@ -647,7 +783,7 @@ CHARSET_INFO *get_charset_by_csname(const char *cs_name, my_pthread_once(&charsets_initialized, init_available_charsets); cs_number= get_charset_number(cs_name, cs_flags); - cs= cs_number ? get_internal_charset(cs_number, flags) : NULL; + cs= cs_number ? get_internal_charset(loader, cs_number, flags) : NULL; if (!cs && (flags & MY_WME)) { @@ -660,6 +796,15 @@ CHARSET_INFO *get_charset_by_csname(const char *cs_name, } +CHARSET_INFO * +get_charset_by_csname(const char *cs_name, uint cs_flags, myf flags) +{ + MY_CHARSET_LOADER loader; + my_charset_loader_init_mysys(&loader); + return my_charset_get_by_name(&loader, cs_name, cs_flags, flags); +} + + /** Resolve character set by the character set name (utf8, latin1, ...). @@ -857,8 +1002,11 @@ CHARSET_INFO *fs_character_set() As we're now interested in cp932 only, let's just detect it using strcmp(). */ - fs_cset_cache= !strcmp(buf, "cp932") ? - &my_charset_cp932_japanese_ci : &my_charset_bin; + fs_cset_cache= + #ifdef HAVE_CHARSET_cp932 + !strcmp(buf, "cp932") ? &my_charset_cp932_japanese_ci : + #endif + &my_charset_bin; } return fs_cset_cache; } diff --git a/mysys/checksum.c b/mysys/checksum.c index 7bc52c6a178..a948785d935 100644 --- a/mysys/checksum.c +++ b/mysys/checksum.c @@ -18,8 +18,6 @@ #include <my_sys.h> #include <zlib.h> -ulong my_crc_dbug_check= ~0; /* Cannot happen */ - /* Calculate a long checksum for a memoryblock. @@ -34,7 +32,5 @@ ha_checksum my_checksum(ha_checksum crc, const uchar *pos, size_t length) { crc= (ha_checksum) crc32((uint)crc, pos, (uint) length); DBUG_PRINT("info", ("crc: %lu", (ulong) crc)); - if ((ulong)crc == my_crc_dbug_check) - my_debug_put_break_here(); return crc; } diff --git a/mysys/errors.c b/mysys/errors.c index 0f2edbc4ae5..2d29740a689 100644 --- a/mysys/errors.c +++ b/mysys/errors.c @@ -21,41 +21,41 @@ const char *globerrs[GLOBERRS]= { - "Can't create/write to file '%s' (Errcode: %d)", - "Error reading file '%s' (Errcode: %d)", - "Error writing file '%s' (Errcode: %d)", - "Error on close of '%s' (Errcode: %d)", + "Can't create/write to file '%s' (Errcode: %M)", + "Error reading file '%s' (Errcode: %M)", + "Error writing file '%s' (Errcode: %M)", + "Error on close of '%s' (Errcode: %M)", "Out of memory (Needed %u bytes)", - "Error on delete of '%s' (Errcode: %d)", - "Error on rename of '%s' to '%s' (Errcode: %d)", + "Error on delete of '%s' (Errcode: %M)", + "Error on rename of '%s' to '%s' (Errcode: %M)", "", - "Unexpected eof found when reading file '%s' (Errcode: %d)", - "Can't lock file (Errcode: %d)", - "Can't unlock file (Errcode: %d)", - "Can't read dir of '%s' (Errcode: %d)", - "Can't get stat of '%s' (Errcode: %d)", - "Can't change size of file (Errcode: %d)", - "Can't open stream from handle (Errcode: %d)", - "Can't get working dirctory (Errcode: %d)", - "Can't change dir to '%s' (Errcode: %d)", + "Unexpected end-of-file found when reading file '%s' (Errcode: %M)", + "Can't lock file (Errcode: %M)", + "Can't unlock file (Errcode: %M)", + "Can't read dir of '%s' (Errcode: %M)", + "Can't get stat of '%s' (Errcode: %M)", + "Can't change size of file (Errcode: %M)", + "Can't open stream from handle (Errcode: %M)", + "Can't get working directory (Errcode: %M)", + "Can't change dir to '%s' (Errcode: %M)", "Warning: '%s' had %d links", "Warning: %d files and %d streams is left open\n", - "Disk is full writing '%s' (Errcode: %d). Waiting for someone to free space... (Expect up to %d secs delay for server to continue after freeing disk space)", - "Can't create directory '%s' (Errcode: %d)", + "Disk is full writing '%s' (Errcode: %M). Waiting for someone to free space... (Expect up to %d secs delay for server to continue after freeing disk space)", + "Can't create directory '%s' (Errcode: %M)", "Character set '%s' is not a compiled character set and is not specified in the '%s' file", - "Out of resources when opening file '%s' (Errcode: %d)", - "Can't read value for symlink '%s' (Error %d)", - "Can't create symlink '%s' pointing at '%s' (Error %d)", - "Error on realpath() on '%s' (Error %d)", - "Can't sync file '%s' to disk (Errcode: %d)", + "Out of resources when opening file '%s' (Errcode: %M)", + "Can't read value for symlink '%s' (Errcode: %M)", + "Can't create symlink '%s' pointing at '%s' (Errcode: %M)", + "Error on realpath() on '%s' (Errcode: %M)", + "Can't sync file '%s' to disk (Errcode: %M)", "Collation '%s' is not a compiled collation and is not specified in the '%s' file", - "File '%s' not found (Errcode: %d)", + "File '%s' not found (Errcode: %M)", "File '%s' (fileno: %d) was not closed", - "Can't change ownership of the file '%s' (Errcode: %d)", - "Can't change permissions of the file '%s' (Errcode: %d)", - "Can't seek in file '%s' (Errcode: %d)", - "Can't change mode for file '%s' to 0x%lx (Error: %d)", - "Warning: Can't copy ownership for file '%s' (Error: %d)" + "Can't change ownership of the file '%s' (Errcode: %M)", + "Can't change permissions of the file '%s' (Errcode: %M)", + "Can't seek in file '%s' (Errcode: %M)", + "Can't change mode for file '%s' to 0x%lx (Errcode: %M)", + "Warning: Can't copy ownership for file '%s' (Errcode: %M)" }; void init_glob_errs(void) @@ -67,40 +67,40 @@ void init_glob_errs(void) void init_glob_errs() { - EE(EE_CANTCREATEFILE) = "Can't create/write to file '%s' (Errcode: %d)"; - EE(EE_READ) = "Error reading file '%s' (Errcode: %d)"; - EE(EE_WRITE) = "Error writing file '%s' (Errcode: %d)"; - EE(EE_BADCLOSE) = "Error on close of '%'s (Errcode: %d)"; + EE(EE_CANTCREATEFILE) = "Can't create/write to file '%s' (Errcode: %M)"; + EE(EE_READ) = "Error reading file '%s' (Errcode: %M)"; + EE(EE_WRITE) = "Error writing file '%s' (Errcode: %M)"; + EE(EE_BADCLOSE) = "Error on close of '%'s (Errcode: %M)"; EE(EE_OUTOFMEMORY) = "Out of memory (Needed %u bytes)"; - EE(EE_DELETE) = "Error on delete of '%s' (Errcode: %d)"; - EE(EE_LINK) = "Error on rename of '%s' to '%s' (Errcode: %d)"; - EE(EE_EOFERR) = "Unexpected eof found when reading file '%s' (Errcode: %d)"; - EE(EE_CANTLOCK) = "Can't lock file (Errcode: %d)"; - EE(EE_CANTUNLOCK) = "Can't unlock file (Errcode: %d)"; - EE(EE_DIR) = "Can't read dir of '%s' (Errcode: %d)"; - EE(EE_STAT) = "Can't get stat of '%s' (Errcode: %d)"; - EE(EE_CANT_CHSIZE) = "Can't change size of file (Errcode: %d)"; - EE(EE_CANT_OPEN_STREAM)= "Can't open stream from handle (Errcode: %d)"; - EE(EE_GETWD) = "Can't get working directory (Errcode: %d)"; - EE(EE_SETWD) = "Can't change dir to '%s' (Errcode: %d)"; + EE(EE_DELETE) = "Error on delete of '%s' (Errcode: %M)"; + EE(EE_LINK) = "Error on rename of '%s' to '%s' (Errcode: %M)"; + EE(EE_EOFERR) = "Unexpected eof found when reading file '%s' (Errcode: %M)"; + EE(EE_CANTLOCK) = "Can't lock file (Errcode: %M)"; + EE(EE_CANTUNLOCK) = "Can't unlock file (Errcode: %M)"; + EE(EE_DIR) = "Can't read dir of '%s' (Errcode: %M)"; + EE(EE_STAT) = "Can't get stat of '%s' (Errcode: %M)"; + EE(EE_CANT_CHSIZE) = "Can't change size of file (Errcode: %M)"; + EE(EE_CANT_OPEN_STREAM)= "Can't open stream from handle (Errcode: %M)"; + EE(EE_GETWD) = "Can't get working directory (Errcode: %M)"; + EE(EE_SETWD) = "Can't change dir to '%s' (Errcode: %M)"; EE(EE_LINK_WARNING) = "Warning: '%s' had %d links"; EE(EE_OPEN_WARNING) = "Warning: %d files and %d streams is left open\n"; - EE(EE_DISK_FULL) = "Disk is full writing '%s'. Waiting for someone to free space..."; - EE(EE_CANT_MKDIR) ="Can't create directory '%s' (Errcode: %d)"; + EE(EE_DISK_FULL) = "Disk is full writing '%s' (Errcode: %M). Waiting for someone to free space... (Expect up to %d secs delay for server to continue after freeing disk space)", + EE(EE_CANT_MKDIR) ="Can't create directory '%s' (Errcode: %M)"; EE(EE_UNKNOWN_CHARSET)= "Character set '%s' is not a compiled character set and is not specified in the %s file"; - EE(EE_OUT_OF_FILERESOURCES)="Out of resources when opening file '%s' (Errcode: %d)"; - EE(EE_CANT_READLINK)= "Can't read value for symlink '%s' (Error %d)"; - EE(EE_CANT_SYMLINK)= "Can't create symlink '%s' pointing at '%s' (Error %d)"; - EE(EE_REALPATH)= "Error on realpath() on '%s' (Error %d)"; - EE(EE_SYNC)= "Can't sync file '%s' to disk (Errcode: %d)"; + EE(EE_OUT_OF_FILERESOURCES)="Out of resources when opening file '%s' (Errcode: %M)"; + EE(EE_CANT_READLINK)= "Can't read value for symlink '%s' (Errcode: %M)"; + EE(EE_CANT_SYMLINK)= "Can't create symlink '%s' pointing at '%s' (Errcode: %M)"; + EE(EE_REALPATH)= "Error on realpath() on '%s' (Errcode: %M)"; + EE(EE_SYNC)= "Can't sync file '%s' to disk (Errcode: %M)"; EE(EE_UNKNOWN_COLLATION)= "Collation '%s' is not a compiled collation and is not specified in the %s file"; - EE(EE_FILENOTFOUND) = "File '%s' not found (Errcode: %d)"; + EE(EE_FILENOTFOUND) = "File '%s' not found (Errcode: %M)"; EE(EE_FILE_NOT_CLOSED) = "File '%s' (fileno: %d) was not closed"; - EE(EE_CHANGE_OWNERSHIP) = "Can't change ownership of the file '%s' (Errcode: %d)"; - EE(EE_CHANGE_PERMISSIONS) = "Can't change permissions of the file '%s' (Errcode: %d)"; - EE(EE_CANT_SEEK) = "Can't seek in file '%s' (Errcode: %d)"; - EE(EE_CANT_CHMOD) = "Can't change mode for file '%s' to 0x%lx (Error: %d)"; - EE(EE_CANT_COPY_OWNERSHIP)= "Warning: Can't copy ownership for file '%s' (Error: %d)"; + EE(EE_CHANGE_OWNERSHIP) = "Can't change ownership of the file '%s' (Errcode: %M)"; + EE(EE_CHANGE_PERMISSIONS) = "Can't change permissions of the file '%s' (Errcode: %M)"; + EE(EE_CANT_SEEK) = "Can't seek in file '%s' (Errcode: %M)"; + EE(EE_CANT_CHMOD) = "Can't change mode for file '%s' to 0x%lx (Errcode: %M)"; + EE(EE_CANT_COPY_OWNERSHIP)= "Warning: Can't copy ownership for file '%s' (Errcode: %M)"; } #endif diff --git a/mysys/hash.c b/mysys/hash.c index b93f6b666d6..3c17dcb991c 100644 --- a/mysys/hash.c +++ b/mysys/hash.c @@ -1,4 +1,5 @@ -/* Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved. +/* Copyright (c) 2000, 2010, Oracle and/or its affiliates. + Copyright (c) 2011, 2013, Monty Program Ab. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -39,12 +40,12 @@ static void movelink(HASH_LINK *array,uint pos,uint next_link,uint newlink); static int hashcmp(const HASH *hash, HASH_LINK *pos, const uchar *key, size_t length); -static my_hash_value_type calc_hash(const HASH *hash, - const uchar *key, size_t length) +my_hash_value_type my_hash_sort(CHARSET_INFO *cs, const uchar *key, + size_t length) { - ulong nr1=1, nr2=4; - hash->charset->coll->hash_sort(hash->charset,(uchar*) key,length,&nr1,&nr2); - return (my_hash_value_type)nr1; + ulong nr1= 1, nr2= 4; + cs->coll->hash_sort(cs, (uchar*) key, length, &nr1, &nr2); + return (my_hash_value_type) nr1; } /** @@ -59,7 +60,8 @@ static my_hash_value_type calc_hash(const HASH *hash, as required during insertion. @param[in,out] hash The hash that is initialized - @param[in] charset The charater set information + @param[in[ growth_size size incrememnt for the underlying dynarray + @param[in] charset The character set information @param[in] size The hash size @param[in] key_offest The key offset for the hash @param[in] key_length The length of the key used in @@ -67,14 +69,16 @@ static my_hash_value_type calc_hash(const HASH *hash, @param[in] get_key get the key for the hash @param[in] free_element pointer to the function that does cleanup - @return inidicates success or failure of initialization + @param[in] flags flags set in the hash + @return indicates success or failure of initialization @retval 0 success @retval 1 failure */ my_bool -_my_hash_init(HASH *hash, uint growth_size, CHARSET_INFO *charset, +my_hash_init2(HASH *hash, uint growth_size, CHARSET_INFO *charset, ulong size, size_t key_offset, size_t key_length, my_hash_get_key get_key, + my_hash_function hash_function, void (*free_element)(void*), uint flags) { my_bool res; @@ -86,11 +90,13 @@ _my_hash_init(HASH *hash, uint growth_size, CHARSET_INFO *charset, hash->key_length=key_length; hash->blength=1; hash->get_key=get_key; + hash->hash_function= hash_function ? hash_function : my_hash_sort; hash->free=free_element; hash->flags=flags; hash->charset=charset; - res= my_init_dynamic_array_ci(&hash->array, - sizeof(HASH_LINK), size, growth_size); + res= init_dynamic_array2(&hash->array, sizeof(HASH_LINK), NULL, size, + growth_size, MYF((flags & HASH_THREAD_SPECIFIC ? + MY_THREAD_SPECIFIC : 0))); DBUG_RETURN(res); } @@ -108,14 +114,19 @@ _my_hash_init(HASH *hash, uint growth_size, CHARSET_INFO *charset, static inline void my_hash_free_elements(HASH *hash) { + uint records= hash->records; + /* + Set records to 0 early to guard against anyone looking at the structure + during the free process + */ + hash->records= 0; if (hash->free) { HASH_LINK *data=dynamic_element(&hash->array,0,HASH_LINK*); - HASH_LINK *end= data + hash->records; + HASH_LINK *end= data + records; while (data < end) (*hash->free)((data++)->data); } - hash->records=0; } @@ -195,7 +206,8 @@ static uint my_hash_rec_mask(const HASH *hash, HASH_LINK *pos, { size_t length; uchar *key= (uchar*) my_hash_key(hash, pos->data, &length, 0); - return my_hash_mask(calc_hash(hash, key, length), buffmax, maxlength); + return my_hash_mask(hash->hash_function(hash->charset, key, length), buffmax, + maxlength); } @@ -209,7 +221,7 @@ my_hash_value_type rec_hashnr(HASH *hash,const uchar *record) { size_t length; uchar *key= (uchar*) my_hash_key(hash, record, &length, 0); - return calc_hash(hash,key,length); + return hash->hash_function(hash->charset, key, length); } @@ -229,12 +241,6 @@ uchar* my_hash_search_using_hash_value(const HASH *hash, key, length, &state); } -my_hash_value_type my_calc_hash(const HASH *hash, - const uchar *key, size_t length) -{ - return calc_hash(hash, key, length ? length : hash->key_length); -} - /* Search after a record based on a key @@ -249,7 +255,8 @@ uchar* my_hash_first(const HASH *hash, const uchar *key, size_t length, uchar *res; if (my_hash_inited(hash)) res= my_hash_first_from_hash_value(hash, - calc_hash(hash, key, length ? length : hash->key_length), + hash->hash_function(hash->charset, key, + length ? length : hash->key_length), key, length, current_record); else res= 0; @@ -516,6 +523,9 @@ my_bool my_hash_insert(HASH *info, const uchar *record) The record with the same record ptr is removed. If there is a free-function it's called if record was found. + hash->free() is guarantee to be called only after the row has been + deleted from the hash and the hash can be reused by other threads. + @return @retval 0 ok @retval 1 Record not found @@ -608,7 +618,7 @@ exit: /** Update keys when record has changed. - This is much more efficent than using a delete & insert. + This is much more efficient than using a delete & insert. */ my_bool my_hash_update(HASH *hash, uchar *record, uchar *old_key, @@ -639,9 +649,9 @@ my_bool my_hash_update(HASH *hash, uchar *record, uchar *old_key, /* Search after record with key */ - idx= my_hash_mask(calc_hash(hash, old_key, (old_key_length ? - old_key_length : - hash->key_length)), + idx= my_hash_mask(hash->hash_function(hash->charset, old_key, + (old_key_length ? old_key_length : + hash->key_length)), blength, records); new_index= my_hash_mask(rec_hashnr(hash, record), blength, records); if (idx == new_index) diff --git a/mysys/lf_alloc-pin.c b/mysys/lf_alloc-pin.c index 59f60f06696..b9260b115bd 100644 --- a/mysys/lf_alloc-pin.c +++ b/mysys/lf_alloc-pin.c @@ -111,7 +111,7 @@ #define LF_PINBOX_MAX_PINS 65536 -static void _lf_pinbox_real_free(LF_PINS *pins); +static void lf_pinbox_real_free(LF_PINS *pins); /* Initialize a pinbox. Normally called from lf_alloc_init. @@ -139,7 +139,7 @@ void lf_pinbox_destroy(LF_PINBOX *pinbox) Get pins from a pinbox. Usually called via lf_alloc_get_pins() or lf_hash_get_pins(). - SYNOPSYS + SYNOPSIS pinbox - DESCRIPTION @@ -150,7 +150,7 @@ void lf_pinbox_destroy(LF_PINBOX *pinbox) It is assumed that pins belong to a thread and are not transferable between threads. */ -LF_PINS *_lf_pinbox_get_pins(LF_PINBOX *pinbox) +LF_PINS *lf_pinbox_get_pins(LF_PINBOX *pinbox) { struct st_my_thread_var *var; uint32 pins, next, top_ver; @@ -177,12 +177,12 @@ LF_PINS *_lf_pinbox_get_pins(LF_PINBOX *pinbox) note that the first allocated element has index 1 (pins==1). index 0 is reserved to mean "NULL pointer" */ - el= (LF_PINS *)_lf_dynarray_lvalue(&pinbox->pinarray, pins); + el= (LF_PINS *)lf_dynarray_lvalue(&pinbox->pinarray, pins); if (unlikely(!el)) return 0; break; } - el= (LF_PINS *)_lf_dynarray_value(&pinbox->pinarray, pins); + el= (LF_PINS *)lf_dynarray_value(&pinbox->pinarray, pins); next= el->link; } while (!my_atomic_cas32((int32 volatile*) &pinbox->pinstack_top_ver, (int32*) &top_ver, @@ -212,7 +212,7 @@ LF_PINS *_lf_pinbox_get_pins(LF_PINBOX *pinbox) empty the purgatory (XXX deadlock warning below!), push LF_PINS structure to a stack */ -void _lf_pinbox_put_pins(LF_PINS *pins) +void lf_pinbox_put_pins(LF_PINS *pins) { LF_PINBOX *pinbox= pins->pinbox; uint32 top_ver, nr; @@ -229,19 +229,15 @@ void _lf_pinbox_put_pins(LF_PINS *pins) /* XXX this will deadlock if other threads will wait for - the caller to do something after _lf_pinbox_put_pins(), + the caller to do something after lf_pinbox_put_pins(), and they would have pinned addresses that the caller wants to free. Thus: only free pins when all work is done and nobody can wait for you!!! */ while (pins->purgatory_count) { - _lf_pinbox_real_free(pins); + lf_pinbox_real_free(pins); if (pins->purgatory_count) - { - my_atomic_rwlock_wrunlock(&pins->pinbox->pinarray.lock); pthread_yield(); - my_atomic_rwlock_wrlock(&pins->pinbox->pinarray.lock); - } } top_ver= pinbox->pinstack_top_ver; do @@ -271,14 +267,14 @@ static int ptr_cmp(void **a, void **b) Free an object allocated via pinbox allocator DESCRIPTION - add an object to purgatory. if necessary, call _lf_pinbox_real_free() + add an object to purgatory. if necessary, calllf_pinbox_real_free() to actually free something. */ -void _lf_pinbox_free(LF_PINS *pins, void *addr) +void lf_pinbox_free(LF_PINS *pins, void *addr) { add_to_purgatory(pins, addr); if (pins->purgatory_count % LF_PURGATORY_SIZE == 0) - _lf_pinbox_real_free(pins); + lf_pinbox_real_free(pins); } struct st_harvester { @@ -287,13 +283,13 @@ struct st_harvester { }; /* - callback for _lf_dynarray_iterate: + callback forlf_dynarray_iterate: scan all pins of all threads and accumulate all pins */ static int harvest_pins(LF_PINS *el, struct st_harvester *hv) { int i; - LF_PINS *el_end= el+min(hv->npins, LF_DYNARRAY_LEVEL_LENGTH); + LF_PINS *el_end= el+MY_MIN(hv->npins, LF_DYNARRAY_LEVEL_LENGTH); for (; el < el_end; el++) { for (i= 0; i < LF_PINBOX_PINS; i++) @@ -314,7 +310,7 @@ static int harvest_pins(LF_PINS *el, struct st_harvester *hv) } /* - callback for _lf_dynarray_iterate: + callback forlf_dynarray_iterate: scan all pins of all threads and see if addr is present there */ static int match_pins(LF_PINS *el, void *addr) @@ -328,19 +324,13 @@ static int match_pins(LF_PINS *el, void *addr) return 0; } -#if STACK_DIRECTION < 0 -#define available_stack_size(CUR,END) (long) ((char*)(CUR) - (char*)(END)) -#else -#define available_stack_size(CUR,END) (long) ((char*)(END) - (char*)(CUR)) -#endif - #define next_node(P, X) (*((uchar * volatile *)(((uchar *)(X)) + (P)->free_ptr_offset))) #define anext_node(X) next_node(&allocator->pinbox, (X)) /* Scan the purgatory and free everything that can be freed */ -static void _lf_pinbox_real_free(LF_PINS *pins) +static void lf_pinbox_real_free(LF_PINS *pins) { int npins; void *list; @@ -363,7 +353,7 @@ static void _lf_pinbox_real_free(LF_PINS *pins) hv.granary= addr; hv.npins= npins; /* scan the dynarray and accumulate all pinned addresses */ - _lf_dynarray_iterate(&pinbox->pinarray, + lf_dynarray_iterate(&pinbox->pinarray, (lf_dynarray_func)harvest_pins, &hv); npins= hv.granary-addr; @@ -398,7 +388,7 @@ static void _lf_pinbox_real_free(LF_PINS *pins) } else /* no alloca - no cookie. linear search here */ { - if (_lf_dynarray_iterate(&pinbox->pinarray, + if (lf_dynarray_iterate(&pinbox->pinarray, (lf_dynarray_func)match_pins, cur)) goto found; } @@ -419,10 +409,8 @@ found: /* lock-free memory allocator for fixed-size objects */ -LF_REQUIRE_PINS(1) - /* - callback for _lf_pinbox_real_free to free a list of unpinned objects - + callback forlf_pinbox_real_free to free a list of unpinned objects - add it back to the allocator stack DESCRIPTION @@ -449,7 +437,7 @@ static void alloc_free(uchar *first, /* initialize lock-free allocator - SYNOPSYS + SYNOPSIS allocator - size a size of an object to allocate free_ptr_offset an offset inside the object to a sizeof(void *) @@ -504,7 +492,7 @@ void lf_alloc_destroy(LF_ALLOCATOR *allocator) Pop an unused object from the stack or malloc it is the stack is empty. pin[0] is used, it's removed on return. */ -void *_lf_alloc_new(LF_PINS *pins) +void *lf_alloc_new(LF_PINS *pins) { LF_ALLOCATOR *allocator= (LF_ALLOCATOR *)(pins->pinbox->free_func_arg); uchar *node; @@ -513,7 +501,7 @@ void *_lf_alloc_new(LF_PINS *pins) do { node= allocator->top; - _lf_pin(pins, 0, node); + lf_pin(pins, 0, node); } while (node != allocator->top && LF_BACKOFF); if (!node) { @@ -530,7 +518,7 @@ void *_lf_alloc_new(LF_PINS *pins) (void *)&node, anext_node(node))) break; } - _lf_unpin(pins, 0); + lf_unpin(pins, 0); return node; } diff --git a/mysys/lf_dynarray.c b/mysys/lf_dynarray.c index 3d072fd063e..bb6cbcefc49 100644 --- a/mysys/lf_dynarray.c +++ b/mysys/lf_dynarray.c @@ -44,7 +44,6 @@ void lf_dynarray_init(LF_DYNARRAY *array, uint element_size) { bzero(array, sizeof(*array)); array->size_of_element= element_size; - my_atomic_rwlock_init(&array->lock); } static void recursive_free(void **alloc, int level) @@ -68,7 +67,6 @@ void lf_dynarray_destroy(LF_DYNARRAY *array) int i; for (i= 0; i < LF_DYNARRAY_LEVELS; i++) recursive_free(array->level[i], i); - my_atomic_rwlock_destroy(&array->lock); } static const ulong dynarray_idxes_in_prev_levels[LF_DYNARRAY_LEVELS]= @@ -95,7 +93,7 @@ static const ulong dynarray_idxes_in_prev_level[LF_DYNARRAY_LEVELS]= Returns a valid lvalue pointer to the element number 'idx'. Allocates memory if necessary. */ -void *_lf_dynarray_lvalue(LF_DYNARRAY *array, uint idx) +void *lf_dynarray_lvalue(LF_DYNARRAY *array, uint idx) { void * ptr, * volatile * ptr_ptr= 0; int i; @@ -124,7 +122,7 @@ void *_lf_dynarray_lvalue(LF_DYNARRAY *array, uint idx) { uchar *alloc, *data; alloc= my_malloc(LF_DYNARRAY_LEVEL_LENGTH * array->size_of_element + - max(array->size_of_element, sizeof(void *)), + MY_MAX(array->size_of_element, sizeof(void *)), MYF(MY_WME|MY_ZEROFILL)); if (unlikely(!alloc)) return(NULL); @@ -148,7 +146,7 @@ void *_lf_dynarray_lvalue(LF_DYNARRAY *array, uint idx) Returns a pointer to the element number 'idx' or NULL if an element does not exists */ -void *_lf_dynarray_value(LF_DYNARRAY *array, uint idx) +void *lf_dynarray_value(LF_DYNARRAY *array, uint idx) { void * ptr, * volatile * ptr_ptr= 0; int i; @@ -189,14 +187,14 @@ static int recursive_iterate(LF_DYNARRAY *array, void *ptr, int level, DESCRIPTION lf_dynarray consists of a set of arrays, LF_DYNARRAY_LEVEL_LENGTH elements - each. _lf_dynarray_iterate() calls user-supplied function on every array + each. lf_dynarray_iterate() calls user-supplied function on every array from the set. It is the fastest way to scan the array, faster than - for (i=0; i < N; i++) { func(_lf_dynarray_value(dynarray, i)); } + for (i=0; i < N; i++) { func(lf_dynarray_value(dynarray, i)); } NOTE if func() returns non-zero, the scan is aborted */ -int _lf_dynarray_iterate(LF_DYNARRAY *array, lf_dynarray_func func, void *arg) +int lf_dynarray_iterate(LF_DYNARRAY *array, lf_dynarray_func func, void *arg) { int i, res; for (i= 0; i < LF_DYNARRAY_LEVELS; i++) diff --git a/mysys/lf_hash.c b/mysys/lf_hash.c index 8d74aaf8cc2..5e407e9caec 100644 --- a/mysys/lf_hash.c +++ b/mysys/lf_hash.c @@ -25,14 +25,13 @@ #include <my_global.h> #include <m_string.h> #include <my_sys.h> +#include <mysys_err.h> #include <my_bit.h> #include <lf.h> -LF_REQUIRE_PINS(3) - /* An element of the list */ typedef struct { - intptr volatile link; /* a pointer to the next element in a listand a flag */ + intptr volatile link; /* a pointer to the next element in a list and a flag */ uint32 hashnr; /* reversed hash number, for sorting */ const uchar *key; size_t keylen; @@ -60,64 +59,84 @@ typedef struct { #define PTR(V) (LF_SLIST *)((V) & (~(intptr)1)) #define DELETED(V) ((V) & 1) -/* - DESCRIPTION +/** walk the list, searching for an element or invoking a callback + Search for hashnr/key/keylen in the list starting from 'head' and position the cursor. The list is ORDER BY hashnr, key - RETURN - 0 - not found - 1 - found + @param head start walking the list from this node + @param cs charset for comparing keys, NULL if callback is used + @param hashnr hash number to search for + @param key key to search for OR data for the callback + @param keylen length of the key to compare, 0 if callback is used + @param cursor for returning the found element + @param pins see lf_alloc-pin.c + @param callback callback action, invoked for every element - NOTE + @note cursor is positioned in either case pins[0..2] are used, they are NOT removed on return + callback might see some elements twice (because of retries) + + @return + if find: 0 - not found + 1 - found + if callback: + 0 - ok + 1 - error (callbck returned 1) */ static int l_find(LF_SLIST * volatile *head, CHARSET_INFO *cs, uint32 hashnr, - const uchar *key, uint keylen, CURSOR *cursor, LF_PINS *pins) + const uchar *key, uint keylen, CURSOR *cursor, LF_PINS *pins, + my_hash_walk_action callback) { uint32 cur_hashnr; const uchar *cur_key; uint cur_keylen; intptr link; + DBUG_ASSERT(!cs || !callback); /* should not be set both */ + DBUG_ASSERT(!keylen || !callback); /* should not be set both */ + retry: cursor->prev= (intptr *)head; do { /* PTR() isn't necessary below, head is a dummy node */ cursor->curr= (LF_SLIST *)(*cursor->prev); - _lf_pin(pins, 1, cursor->curr); + lf_pin(pins, 1, cursor->curr); } while (my_atomic_loadptr((void**)cursor->prev) != cursor->curr && LF_BACKOFF); for (;;) { if (unlikely(!cursor->curr)) return 0; /* end of the list */ + + cur_hashnr= cursor->curr->hashnr; + cur_keylen= cursor->curr->keylen; + cur_key= cursor->curr->key; + do { - /* QQ: XXX or goto retry ? */ link= cursor->curr->link; cursor->next= PTR(link); - _lf_pin(pins, 0, cursor->next); + lf_pin(pins, 0, cursor->next); } while (link != cursor->curr->link && LF_BACKOFF); - cur_hashnr= cursor->curr->hashnr; - cur_key= cursor->curr->key; - cur_keylen= cursor->curr->keylen; - if (my_atomic_loadptr((void**)cursor->prev) != cursor->curr) - { - (void)LF_BACKOFF; - goto retry; - } + if (!DELETED(link)) { - if (cur_hashnr >= hashnr) + if (unlikely(callback)) + { + if (cur_hashnr & 1 && callback(cursor->curr + 1, (void*)key)) + return 1; + } + else if (cur_hashnr >= hashnr) { int r= 1; if (cur_hashnr > hashnr || - (r= my_strnncoll(cs, (uchar*) cur_key, cur_keylen, (uchar*) key, - keylen)) >= 0) + (r= my_strnncoll(cs, cur_key, cur_keylen, key, keylen)) >= 0) return !r; } cursor->prev= &(cursor->curr->link); - _lf_pin(pins, 2, cursor->curr); + if (!(cur_hashnr & 1)) /* dummy node */ + head= (LF_SLIST **)cursor->prev; + lf_pin(pins, 2, cursor->curr); } else { @@ -126,16 +145,13 @@ retry: and remove this deleted node */ if (my_atomic_casptr((void **) cursor->prev, - (void **)(char*) &cursor->curr, cursor->next)) - _lf_alloc_free(pins, cursor->curr); + (void **) &cursor->curr, cursor->next) && LF_BACKOFF) + lf_alloc_free(pins, cursor->curr); else - { - (void)LF_BACKOFF; goto retry; - } } cursor->curr= cursor->next; - _lf_pin(pins, 1, cursor->curr); + lf_pin(pins, 1, cursor->curr); } } @@ -161,7 +177,7 @@ static LF_SLIST *l_insert(LF_SLIST * volatile *head, CHARSET_INFO *cs, for (;;) { if (l_find(head, cs, node->hashnr, node->key, node->keylen, - &cursor, pins) && + &cursor, pins, 0) && (flags & LF_HASH_UNIQUE)) { res= 0; /* duplicate found */ @@ -180,12 +196,12 @@ static LF_SLIST *l_insert(LF_SLIST * volatile *head, CHARSET_INFO *cs, } } } - _lf_unpin(pins, 0); - _lf_unpin(pins, 1); - _lf_unpin(pins, 2); + lf_unpin(pins, 0); + lf_unpin(pins, 1); + lf_unpin(pins, 2); /* Note that cursor.curr is not pinned here and the pointer is unreliable, - the object may dissapear anytime. But if it points to a dummy node, the + the object may disappear anytime. But if it points to a dummy node, the pointer is safe, because dummy nodes are never freed - initialize_bucket() uses this fact. */ @@ -212,7 +228,7 @@ static int l_delete(LF_SLIST * volatile *head, CHARSET_INFO *cs, uint32 hashnr, for (;;) { - if (!l_find(head, cs, hashnr, key, keylen, &cursor, pins)) + if (!l_find(head, cs, hashnr, key, keylen, &cursor, pins, 0)) { res= 1; /* not found */ break; @@ -227,7 +243,7 @@ static int l_delete(LF_SLIST * volatile *head, CHARSET_INFO *cs, uint32 hashnr, /* and remove it from the list */ if (my_atomic_casptr((void **)cursor.prev, (void **)(char*)&cursor.curr, cursor.next)) - _lf_alloc_free(pins, cursor.curr); + lf_alloc_free(pins, cursor.curr); else { /* @@ -236,16 +252,16 @@ static int l_delete(LF_SLIST * volatile *head, CHARSET_INFO *cs, uint32 hashnr, (to ensure the number of "set DELETED flag" actions is equal to the number of "remove from the list" actions) */ - l_find(head, cs, hashnr, key, keylen, &cursor, pins); + l_find(head, cs, hashnr, key, keylen, &cursor, pins, 0); } res= 0; break; } } } - _lf_unpin(pins, 0); - _lf_unpin(pins, 1); - _lf_unpin(pins, 2); + lf_unpin(pins, 0); + lf_unpin(pins, 1); + lf_unpin(pins, 2); return res; } @@ -267,13 +283,13 @@ static LF_SLIST *l_search(LF_SLIST * volatile *head, CHARSET_INFO *cs, LF_PINS *pins) { CURSOR cursor; - int res= l_find(head, cs, hashnr, key, keylen, &cursor, pins); + int res= l_find(head, cs, hashnr, key, keylen, &cursor, pins, 0); if (res) - _lf_pin(pins, 2, cursor.curr); + lf_pin(pins, 2, cursor.curr); else - _lf_unpin(pins, 2); - _lf_unpin(pins, 1); - _lf_unpin(pins, 0); + lf_unpin(pins, 2); + lf_unpin(pins, 1); + lf_unpin(pins, 0); return res ? cursor.curr : 0; } @@ -292,18 +308,24 @@ static inline const uchar* hash_key(const LF_HASH *hash, @note, that the hash value is limited to 2^31, because we need one bit to distinguish between normal and dummy nodes. */ -static inline uint calc_hash(LF_HASH *hash, const uchar *key, uint keylen) +static inline my_hash_value_type calc_hash(const CHARSET_INFO *cs, + const uchar *key, + size_t keylen) { ulong nr1= 1, nr2= 4; - hash->charset->coll->hash_sort(hash->charset, (uchar*) key, keylen, - &nr1, &nr2); - return nr1 & INT_MAX32; + cs->coll->hash_sort(cs, (uchar*) key, keylen, &nr1, &nr2); + return nr1; } #define MAX_LOAD 1.0 /* average number of elements in a bucket */ static int initialize_bucket(LF_HASH *, LF_SLIST * volatile*, uint, LF_PINS *); +static void default_initializer(LF_HASH *hash, void *dst, const void *src) +{ + memcpy(dst, src, hash->element_size); +} + /* Initializes lf_hash, the arguments are compatible with hash_init @@ -311,11 +333,13 @@ static int initialize_bucket(LF_HASH *, LF_SLIST * volatile*, uint, LF_PINS *); lf_alloc and a size of memcpy'ed block size in lf_hash_insert. Typically they are the same, indeed. But LF_HASH::element_size can be decreased after lf_hash_init, and then lf_alloc will allocate larger block that - lf_hash_insert will copy over. It is desireable if part of the element + lf_hash_insert will copy over. It is desirable if part of the element is expensive to initialize - for example if there is a mutex or DYNAMIC_ARRAY. In this case they should be initialize in the LF_ALLOCATOR::constructor, and lf_hash_insert should not overwrite them. - See wt_init() for example. + + The above works well with PODS. For more complex cases (e.g. C++ classes + with private members) use initializer function. */ void lf_hash_init(LF_HASH *hash, uint element_size, uint flags, uint key_offset, uint key_length, my_hash_get_key get_key, @@ -332,12 +356,14 @@ void lf_hash_init(LF_HASH *hash, uint element_size, uint flags, hash->key_offset= key_offset; hash->key_length= key_length; hash->get_key= get_key; + hash->initializer= default_initializer; + hash->hash_function= calc_hash; DBUG_ASSERT(get_key ? !key_offset && !key_length : key_length); } void lf_hash_destroy(LF_HASH *hash) { - LF_SLIST *el, **head= (LF_SLIST **)_lf_dynarray_value(&hash->array, 0); + LF_SLIST *el, **head= (LF_SLIST **)lf_dynarray_value(&hash->array, 0); if (head) { @@ -374,15 +400,14 @@ int lf_hash_insert(LF_HASH *hash, LF_PINS *pins, const void *data) int csize, bucket, hashnr; LF_SLIST *node, * volatile *el; - lf_rwlock_by_pins(pins); - node= (LF_SLIST *)_lf_alloc_new(pins); + node= (LF_SLIST *)lf_alloc_new(pins); if (unlikely(!node)) return -1; - memcpy(node+1, data, hash->element_size); + hash->initializer(hash, node + 1, data); node->key= hash_key(hash, (uchar *)(node+1), &node->keylen); - hashnr= calc_hash(hash, node->key, node->keylen); + hashnr= hash->hash_function(hash->charset, node->key, node->keylen) & INT_MAX32; bucket= hashnr % hash->size; - el= _lf_dynarray_lvalue(&hash->array, bucket); + el= lf_dynarray_lvalue(&hash->array, bucket); if (unlikely(!el)) return -1; if (*el == NULL && unlikely(initialize_bucket(hash, el, bucket, pins))) @@ -390,14 +415,12 @@ int lf_hash_insert(LF_HASH *hash, LF_PINS *pins, const void *data) node->hashnr= my_reverse_bits(hashnr) | 1; /* normal node */ if (l_insert(el, hash->charset, node, pins, hash->flags)) { - _lf_alloc_free(pins, node); - lf_rwunlock_by_pins(pins); + lf_alloc_free(pins, node); return 1; } csize= hash->size; if ((my_atomic_add32(&hash->count, 1)+1.0) / csize > MAX_LOAD) my_atomic_cas32(&hash->size, &csize, csize*2); - lf_rwunlock_by_pins(pins); return 0; } @@ -409,36 +432,31 @@ int lf_hash_insert(LF_HASH *hash, LF_PINS *pins, const void *data) RETURN 0 - deleted 1 - didn't (not found) - -1 - out of memory NOTE see l_delete() for pin usage notes */ int lf_hash_delete(LF_HASH *hash, LF_PINS *pins, const void *key, uint keylen) { LF_SLIST * volatile *el; - uint bucket, hashnr= calc_hash(hash, (uchar *)key, keylen); + uint bucket, hashnr; - bucket= hashnr % hash->size; - lf_rwlock_by_pins(pins); - el= _lf_dynarray_lvalue(&hash->array, bucket); - if (unlikely(!el)) - return -1; - /* - note that we still need to initialize_bucket here, - we cannot return "node not found", because an old bucket of that - node may've been split and the node was assigned to a new bucket - that was never accessed before and thus is not initialized. - */ - if (*el == NULL && unlikely(initialize_bucket(hash, el, bucket, pins))) - return -1; + hashnr= hash->hash_function(hash->charset, (uchar *)key, keylen) & INT_MAX32; + + /* hide OOM errors - if we cannot initialize a bucket, try the previous one */ + for (bucket= hashnr % hash->size; ;bucket= my_clear_highest_bit(bucket)) + { + el= lf_dynarray_lvalue(&hash->array, bucket); + if (el && (*el || initialize_bucket(hash, el, bucket, pins) == 0)) + break; + if (unlikely(bucket == 0)) + return 1; /* if there's no bucket==0, the hash is empty */ + } if (l_delete(el, hash->charset, my_reverse_bits(hashnr) | 1, (uchar *)key, keylen, pins)) { - lf_rwunlock_by_pins(pins); return 1; } my_atomic_add32(&hash->count, -1); - lf_rwunlock_by_pins(pins); return 0; } @@ -447,29 +465,73 @@ int lf_hash_delete(LF_HASH *hash, LF_PINS *pins, const void *key, uint keylen) a pointer to an element with the given key (if a hash is not unique and there're many elements with this key - the "first" matching element) NULL if nothing is found - MY_ERRPTR if OOM NOTE see l_search() for pin usage notes */ -void *lf_hash_search(LF_HASH *hash, LF_PINS *pins, const void *key, uint keylen) +void *lf_hash_search_using_hash_value(LF_HASH *hash, LF_PINS *pins, + my_hash_value_type hashnr, + const void *key, uint keylen) { LF_SLIST * volatile *el, *found; - uint bucket, hashnr= calc_hash(hash, (uchar *)key, keylen); + uint bucket; - bucket= hashnr % hash->size; - lf_rwlock_by_pins(pins); - el= _lf_dynarray_lvalue(&hash->array, bucket); - if (unlikely(!el)) - return MY_ERRPTR; - if (*el == NULL && unlikely(initialize_bucket(hash, el, bucket, pins))) - return MY_ERRPTR; + /* hide OOM errors - if we cannot initialize a bucket, try the previous one */ + for (bucket= hashnr % hash->size; ;bucket= my_clear_highest_bit(bucket)) + { + el= lf_dynarray_lvalue(&hash->array, bucket); + if (el && (*el || initialize_bucket(hash, el, bucket, pins) == 0)) + break; + if (unlikely(bucket == 0)) + return 0; /* if there's no bucket==0, the hash is empty */ + } found= l_search(el, hash->charset, my_reverse_bits(hashnr) | 1, (uchar *)key, keylen, pins); - lf_rwunlock_by_pins(pins); return found ? found+1 : 0; } + +/** + Iterate over all elements in hash and call function with the element + + @note + If one of 'action' invocations returns 1 the iteration aborts. + 'action' might see some elements twice! + + @retval 0 ok + @retval 1 error (action returned 1) +*/ +int lf_hash_iterate(LF_HASH *hash, LF_PINS *pins, + my_hash_walk_action action, void *argument) +{ + CURSOR cursor; + uint bucket= 0; + int res; + LF_SLIST * volatile *el; + + el= lf_dynarray_lvalue(&hash->array, bucket); + if (unlikely(!el)) + return 0; /* if there's no bucket==0, the hash is empty */ + if (*el == NULL && unlikely(initialize_bucket(hash, el, bucket, pins))) + return 0; /* if there's no bucket==0, the hash is empty */ + + res= l_find(el, 0, 0, (uchar*)argument, 0, &cursor, pins, action); + + lf_unpin(pins, 2); + lf_unpin(pins, 1); + lf_unpin(pins, 0); + return res; +} + +void *lf_hash_search(LF_HASH *hash, LF_PINS *pins, const void *key, uint keylen) +{ + return lf_hash_search_using_hash_value(hash, pins, + hash->hash_function(hash->charset, + (uchar*) key, + keylen) & INT_MAX32, + key, keylen); +} + static const uchar *dummy_key= (uchar*)""; /* @@ -483,12 +545,15 @@ static int initialize_bucket(LF_HASH *hash, LF_SLIST * volatile *node, uint parent= my_clear_highest_bit(bucket); LF_SLIST *dummy= (LF_SLIST *)my_malloc(sizeof(LF_SLIST), MYF(MY_WME)); LF_SLIST **tmp= 0, *cur; - LF_SLIST * volatile *el= _lf_dynarray_lvalue(&hash->array, parent); + LF_SLIST * volatile *el= lf_dynarray_lvalue(&hash->array, parent); if (unlikely(!el || !dummy)) return -1; if (*el == NULL && bucket && unlikely(initialize_bucket(hash, el, parent, pins))) + { + my_free(dummy); return -1; + } dummy->hashnr= my_reverse_bits(bucket) | 0; /* dummy node */ dummy->key= dummy_key; dummy->keylen= 0; diff --git a/mysys/ma_dyncol.c b/mysys/ma_dyncol.c index 62227ab6834..53edfc5d87b 100644 --- a/mysys/ma_dyncol.c +++ b/mysys/ma_dyncol.c @@ -1,5 +1,5 @@ -/* Copyright (c) 2011, Monty Program Ab - Copyright (c) 2011, Oleksandr Byelkin +/* Copyright (c) 2011, 2013, Monty Program Ab + Copyright (c) 2011, 2012, Oleksandr Byelkin Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -26,33 +26,103 @@ SUCH DAMAGE. */ +/* + Numeric format: + =============== + * Fixed header part + 1 byte flags: + 0,1 bits - <offset size> - 1 + 2-7 bits - 0 + 2 bytes column counter + * Columns directory sorted by column number, each entry contains of: + 2 bytes column number + <offset size> bytes (1-4) combined offset from beginning of + the data segment + 3 bit type + * Data of above columns size of data and length depend on type + + Columns with names: + =================== + * Fixed header part + 1 byte flags: + 0,1 bits - <offset size> - 2 + 2 bit - 1 (means format with names) + 3,4 bits - 00 (means <names offset size> - 2, + now 2 is the only supported size) + 5-7 bits - 0 + 2 bytes column counter + * Variable header part (now it is actually fixed part) + <names offset size> (2) bytes size of stored names pool + * Column directory sorted by names, each consists of + <names offset size> (2) bytes offset of name + <offset size> bytes (2-5)bytes combined offset from beginning of + the data segment + 4 bit type + * Names stored one after another + * Data of above columns size of data and length depend on type +*/ + #include "mysys_priv.h" #include <m_string.h> #include <ma_dyncol.h> +#include <my_time.h> +uint32 copy_and_convert(char *to, uint32 to_length, CHARSET_INFO *to_cs, + const char *from, uint32 from_length, + CHARSET_INFO *from_cs, uint *errors); /* Flag byte bits 2 bits which determinate size of offset in the header -1 */ /* mask to get above bits */ -#define DYNCOL_FLG_OFFSET 3 -/* All known flags mask */ -#define DYNCOL_FLG_KNOWN 3 +#define DYNCOL_FLG_OFFSET (1|2) +#define DYNCOL_FLG_NAMES 4 +#define DYNCOL_FLG_NMOFFSET (8|16) +/** + All known flags mask that could be set. + + @note DYNCOL_FLG_NMOFFSET should be 0 for now. +*/ +#define DYNCOL_FLG_KNOWN (1|2|4) + +/* formats */ +enum enum_dyncol_format +{ + dyncol_fmt_num= 0, + dyncol_fmt_str= 1 +}; /* dynamic column size reserve */ #define DYNCOL_SYZERESERVE 80 +#define DYNCOL_OFFSET_ERROR 0xffffffff + /* length of fixed string header 1 byte - flags, 2 bytes - columns counter */ #define FIXED_HEADER_SIZE 3 +/* + length of fixed string header with names + 1 byte - flags, 2 bytes - columns counter, 2 bytes - name pool size +*/ +#define FIXED_HEADER_SIZE_NM 5 #define COLUMN_NUMBER_SIZE 2 +/* 2 bytes offset from the name pool */ +#define COLUMN_NAMEPTR_SIZE 2 + +#define MAX_OFFSET_LENGTH 4 +#define MAX_OFFSET_LENGTH_NM 5 + +#define DYNCOL_NUM_CHAR 6 -#define MAX_OFFSET_LENGTH 5 +my_bool mariadb_dyncol_has_names(DYNAMIC_COLUMN *str) +{ + if (str->length < 1) + return FALSE; + return MY_TEST(str->str[0] & DYNCOL_FLG_NAMES); +} static enum enum_dyncol_func_result dynamic_column_time_store(DYNAMIC_COLUMN *str, - MYSQL_TIME *value); + MYSQL_TIME *value, enum enum_dyncol_format format); static enum enum_dyncol_func_result dynamic_column_date_store(DYNAMIC_COLUMN *str, MYSQL_TIME *value); @@ -62,6 +132,546 @@ dynamic_column_time_read_internal(DYNAMIC_COLUMN_VALUE *store_it_here, static enum enum_dyncol_func_result dynamic_column_date_read_internal(DYNAMIC_COLUMN_VALUE *store_it_here, uchar *data, size_t length); +static enum enum_dyncol_func_result +dynamic_column_get_internal(DYNAMIC_COLUMN *str, + DYNAMIC_COLUMN_VALUE *store_it_here, + uint num_key, LEX_STRING *str_key); +static enum enum_dyncol_func_result +dynamic_column_exists_internal(DYNAMIC_COLUMN *str, uint num_key, + LEX_STRING *str_key); +static enum enum_dyncol_func_result +dynamic_column_update_many_fmt(DYNAMIC_COLUMN *str, + uint add_column_count, + void *column_keys, + DYNAMIC_COLUMN_VALUE *values, + my_bool string_keys); +static int plan_sort_num(const void *a, const void *b); +static int plan_sort_named(const void *a, const void *b); + +/* + Structure to hold information about dynamic columns record and + iterate through it. +*/ + +struct st_dyn_header +{ + uchar *header, *nmpool, *dtpool, *data_end; + size_t offset_size; + size_t entry_size; + size_t header_size; + size_t nmpool_size; + size_t data_size; + /* dyncol_fmt_num - numeric columns, dyncol_fmt_str - column names */ + enum enum_dyncol_format format; + uint column_count; + + uchar *entry, *data, *name; + size_t offset; + size_t length; + enum enum_dynamic_column_type type; +}; + +typedef struct st_dyn_header DYN_HEADER; + +static inline my_bool read_fixed_header(DYN_HEADER *hdr, + DYNAMIC_COLUMN *str); +static void set_fixed_header(DYNAMIC_COLUMN *str, + uint offset_size, + uint column_count); + +/* + Calculate entry size (E) and header size (H) by offset size (O) and column + count (C) and fixed part of entry size (F). +*/ + +#define calc_param(E,H,F,O,C) do { \ + (*(E))= (O) + F; \ + (*(H))= (*(E)) * (C); \ +}while(0); + + +/** + Name pool size functions, for numeric format it is 0 +*/ + +static size_t name_size_num(void *keys __attribute__((unused)), + uint i __attribute__((unused))) +{ + return 0; +} + + +/** + Name pool size functions. +*/ +static size_t name_size_named(void *keys, uint i) +{ + return ((LEX_STRING *) keys)[i].length; +} + + +/** + Comparator function for references on column numbers for qsort + (numeric format) +*/ + +static int column_sort_num(const void *a, const void *b) +{ + return **((uint **)a) - **((uint **)b); +} + +/** + Comparator function for references on column numbers for qsort + (names format) +*/ + +int mariadb_dyncol_column_cmp_named(const LEX_STRING *s1, const LEX_STRING *s2) +{ + /* + We compare instead of subtraction to avoid data loss in case of huge + length difference (more then fit in int). + */ + int rc= (s1->length > s2->length ? 1 : + (s1->length < s2->length ? -1 : 0)); + if (rc == 0) + rc= memcmp((void *)s1->str, (void *)s2->str, + (size_t) s1->length); + return rc; +} + + +/** + Comparator function for references on column numbers for qsort + (names format) +*/ + +static int column_sort_named(const void *a, const void *b) +{ + return mariadb_dyncol_column_cmp_named(*((LEX_STRING **)a), + *((LEX_STRING **)b)); +} + + +/** + Check limit function (numeric format) +*/ + +static my_bool check_limit_num(const void *val) +{ + return **((uint **)val) > UINT_MAX16; +} + + +/** + Check limit function (names format) +*/ + +static my_bool check_limit_named(const void *val) +{ + return (*((LEX_STRING **)val))->length > MAX_NAME_LENGTH; +} + + +/** + Write numeric format static header part. +*/ + +static void set_fixed_header_num(DYNAMIC_COLUMN *str, DYN_HEADER *hdr) +{ + set_fixed_header(str, (uint)hdr->offset_size, hdr->column_count); + hdr->header= (uchar *)str->str + FIXED_HEADER_SIZE; + hdr->nmpool= hdr->dtpool= hdr->header + hdr->header_size; +} + + +/** + Write names format static header part. +*/ + +static void set_fixed_header_named(DYNAMIC_COLUMN *str, DYN_HEADER *hdr) +{ + DBUG_ASSERT(hdr->column_count <= 0xffff); + DBUG_ASSERT(hdr->offset_size <= MAX_OFFSET_LENGTH_NM); + /* size of data offset, named format flag, size of names offset (0 means 2) */ + str->str[0]= + (char) ((str->str[0] & ~(DYNCOL_FLG_OFFSET | DYNCOL_FLG_NMOFFSET)) | + (hdr->offset_size - 2) | DYNCOL_FLG_NAMES); + int2store(str->str + 1, hdr->column_count); /* columns number */ + int2store(str->str + 3, hdr->nmpool_size); + hdr->header= (uchar *)str->str + FIXED_HEADER_SIZE_NM; + hdr->nmpool= hdr->header + hdr->header_size; + hdr->dtpool= hdr->nmpool + hdr->nmpool_size; +} + + +/** + Store offset and type information in the given place + + @param place Beginning of the index entry + @param offset_size Size of offset field in bytes + @param type Type to be written + @param offset Offset to be written +*/ + +static my_bool type_and_offset_store_num(uchar *place, size_t offset_size, + DYNAMIC_COLUMN_TYPE type, + size_t offset) +{ + ulong val = (((ulong) offset) << 3) | (type - 1); + DBUG_ASSERT(type != DYN_COL_NULL); + DBUG_ASSERT(((type - 1) & (~7)) == 0); /* fit in 3 bits */ + DBUG_ASSERT(offset_size >= 1 && offset_size <= 4); + + /* Index entry starts with column number; jump over it */ + place+= COLUMN_NUMBER_SIZE; + + switch (offset_size) { + case 1: + if (offset >= 0x1f) /* all 1 value is reserved */ + return TRUE; + place[0]= (uchar)val; + break; + case 2: + if (offset >= 0x1fff) /* all 1 value is reserved */ + return TRUE; + int2store(place, val); + break; + case 3: + if (offset >= 0x1fffff) /* all 1 value is reserved */ + return TRUE; + int3store(place, val); + break; + case 4: + if (offset >= 0x1fffffff) /* all 1 value is reserved */ + return TRUE; + int4store(place, val); + break; + default: + return TRUE; + } + return FALSE; +} + + +static my_bool type_and_offset_store_named(uchar *place, size_t offset_size, + DYNAMIC_COLUMN_TYPE type, + size_t offset) +{ + ulonglong val = (((ulong) offset) << 4) | (type - 1); + DBUG_ASSERT(type != DYN_COL_NULL); + DBUG_ASSERT(((type - 1) & (~0xf)) == 0); /* fit in 4 bits */ + DBUG_ASSERT(offset_size >= 2 && offset_size <= 5); + + /* Index entry starts with name offset; jump over it */ + place+= COLUMN_NAMEPTR_SIZE; + switch (offset_size) { + case 2: + if (offset >= 0xfff) /* all 1 value is reserved */ + return TRUE; + int2store(place, val); + break; + case 3: + if (offset >= 0xfffff) /* all 1 value is reserved */ + return TRUE; + int3store(place, val); + break; + case 4: + if (offset >= 0xfffffff) /* all 1 value is reserved */ + return TRUE; + int4store(place, val); + break; + case 5: +#if SIZEOF_SIZE_T > 4 + if (offset >= 0xfffffffffull) /* all 1 value is reserved */ + return TRUE; +#endif + int5store(place, val); + break; + case 1: + default: + return TRUE; + } + return FALSE; +} + +/** + Write numeric format header entry + 2 bytes - column number + 1-4 bytes - data offset combined with type + + @param hdr descriptor of dynamic column record + @param column_key pointer to uint (column number) + @param value value which will be written (only type used) + @param offset offset of the data +*/ + +static my_bool put_header_entry_num(DYN_HEADER *hdr, + void *column_key, + DYNAMIC_COLUMN_VALUE *value, + size_t offset) +{ + uint *column_number= (uint *)column_key; + int2store(hdr->entry, *column_number); + DBUG_ASSERT(hdr->nmpool_size == 0); + if (type_and_offset_store_num(hdr->entry, hdr->offset_size, + value->type, + offset)) + return TRUE; + hdr->entry= hdr->entry + hdr->entry_size; + return FALSE; +} + + +/** + Write names format header entry + 1 byte - name length + 2 bytes - name offset in the name pool + 1-4 bytes - data offset combined with type + + @param hdr descriptor of dynamic column record + @param column_key pointer to LEX_STRING (column name) + @param value value which will be written (only type used) + @param offset offset of the data +*/ + +static my_bool put_header_entry_named(DYN_HEADER *hdr, + void *column_key, + DYNAMIC_COLUMN_VALUE *value, + size_t offset) +{ + LEX_STRING *column_name= (LEX_STRING *)column_key; + DBUG_ASSERT(column_name->length <= MAX_NAME_LENGTH); + DBUG_ASSERT(hdr->name - hdr->nmpool < (long) 0x10000L); + int2store(hdr->entry, hdr->name - hdr->nmpool); + memcpy(hdr->name, column_name->str, column_name->length); + DBUG_ASSERT(hdr->nmpool_size != 0 || column_name->length == 0); + if (type_and_offset_store_named(hdr->entry, hdr->offset_size, + value->type, + offset)) + return TRUE; + hdr->entry+= hdr->entry_size; + hdr->name+= column_name->length; + return FALSE; +} + + +/** + Calculate length of offset field for given data length + + @param data_length Length of the data segment + + @return number of bytes +*/ + +static size_t dynamic_column_offset_bytes_num(size_t data_length) +{ + if (data_length < 0x1f) /* all 1 value is reserved */ + return 1; + if (data_length < 0x1fff) /* all 1 value is reserved */ + return 2; + if (data_length < 0x1fffff) /* all 1 value is reserved */ + return 3; + if (data_length < 0x1fffffff) /* all 1 value is reserved */ + return 4; + return MAX_OFFSET_LENGTH + 1; /* For an error generation*/ +} + +static size_t dynamic_column_offset_bytes_named(size_t data_length) +{ + if (data_length < 0xfff) /* all 1 value is reserved */ + return 2; + if (data_length < 0xfffff) /* all 1 value is reserved */ + return 3; + if (data_length < 0xfffffff) /* all 1 value is reserved */ + return 4; +#if SIZEOF_SIZE_T > 4 + if (data_length < 0xfffffffffull) /* all 1 value is reserved */ +#endif + return 5; + return MAX_OFFSET_LENGTH_NM + 1; /* For an error generation */ +} + +/** + Read offset and type information from index entry + + @param type Where to put type info + @param offset Where to put offset info + @param place beginning of the type and offset + @param offset_size Size of offset field in bytes +*/ + +static my_bool type_and_offset_read_num(DYNAMIC_COLUMN_TYPE *type, + size_t *offset, + uchar *place, size_t offset_size) +{ + ulong UNINIT_VAR(val); + ulong UNINIT_VAR(lim); + + DBUG_ASSERT(offset_size >= 1 && offset_size <= 4); + + switch (offset_size) { + case 1: + val= (ulong)place[0]; + lim= 0x1f; + break; + case 2: + val= uint2korr(place); + lim= 0x1fff; + break; + case 3: + val= uint3korr(place); + lim= 0x1fffff; + break; + case 4: + val= uint4korr(place); + lim= 0x1fffffff; + break; + default: + DBUG_ASSERT(0); /* impossible */ + return 1; + } + *type= (val & 0x7) + 1; + *offset= val >> 3; + return (*offset >= lim); +} + +static my_bool type_and_offset_read_named(DYNAMIC_COLUMN_TYPE *type, + size_t *offset, + uchar *place, size_t offset_size) +{ + ulonglong UNINIT_VAR(val); + ulonglong UNINIT_VAR(lim); + DBUG_ASSERT(offset_size >= 2 && offset_size <= 5); + + switch (offset_size) { + case 2: + val= uint2korr(place); + lim= 0xfff; + break; + case 3: + val= uint3korr(place); + lim= 0xfffff; + break; + case 4: + val= uint4korr(place); + lim= 0xfffffff; + break; + case 5: + val= uint5korr(place); + lim= 0xfffffffffull; + break; + case 1: + default: + DBUG_ASSERT(0); /* impossible */ + return 1; + } + *type= (val & 0xf) + 1; + *offset= val >> 4; + return (*offset >= lim); +} + +/** + Format descriptor, contain constants and function references for + format processing +*/ + +struct st_service_funcs +{ + /* size of fixed header */ + uint fixed_hdr; + /* size of fixed part of header entry */ + uint fixed_hdr_entry; + + /*size of array element which stores keys */ + uint key_size_in_array; + + /* Maximum data offset size in bytes */ + size_t max_offset_size; + + size_t (*name_size) + (void *, uint); + int (*column_sort) + (const void *a, const void *b); + my_bool (*check_limit) + (const void *val); + void (*set_fixed_hdr) + (DYNAMIC_COLUMN *str, DYN_HEADER *hdr); + my_bool (*put_header_entry)(DYN_HEADER *hdr, + void *column_key, + DYNAMIC_COLUMN_VALUE *value, + size_t offset); + int (*plan_sort)(const void *a, const void *b); + size_t (*dynamic_column_offset_bytes)(size_t data_length); + my_bool (*type_and_offset_read)(DYNAMIC_COLUMN_TYPE *type, + size_t *offset, + uchar *place, size_t offset_size); + +}; + + +/** + Actual our 2 format descriptors +*/ + +static struct st_service_funcs fmt_data[2]= +{ + { + FIXED_HEADER_SIZE, + COLUMN_NUMBER_SIZE, + sizeof(uint), + MAX_OFFSET_LENGTH, + &name_size_num, + &column_sort_num, + &check_limit_num, + &set_fixed_header_num, + &put_header_entry_num, + &plan_sort_num, + &dynamic_column_offset_bytes_num, + &type_and_offset_read_num + }, + { + FIXED_HEADER_SIZE_NM, + COLUMN_NAMEPTR_SIZE, + sizeof(LEX_STRING), + MAX_OFFSET_LENGTH_NM, + &name_size_named, + &column_sort_named, + &check_limit_named, + &set_fixed_header_named, + &put_header_entry_named, + &plan_sort_named, + &dynamic_column_offset_bytes_named, + &type_and_offset_read_named + } +}; + + +/** + Read dynamic column record header and fill the descriptor + + @param hdr dynamic columns record descriptor to fill + @param str dynamic columns record + + @return ER_DYNCOL_* return code +*/ + +static enum enum_dyncol_func_result +init_read_hdr(DYN_HEADER *hdr, DYNAMIC_COLUMN *str) +{ + if (read_fixed_header(hdr, str)) + return ER_DYNCOL_FORMAT; + hdr->header= (uchar*)str->str + fmt_data[hdr->format].fixed_hdr; + calc_param(&hdr->entry_size, &hdr->header_size, + fmt_data[hdr->format].fixed_hdr_entry, hdr->offset_size, + hdr->column_count); + hdr->nmpool= hdr->header + hdr->header_size; + hdr->dtpool= hdr->nmpool + hdr->nmpool_size; + hdr->data_size= str->length - fmt_data[hdr->format].fixed_hdr - + hdr->header_size - hdr->nmpool_size; + hdr->data_end= (uchar*)str->str + str->length; + return ER_DYNCOL_OK; +} + /** Initialize dynamic column string with (make it empty but correct format) @@ -73,7 +683,7 @@ dynamic_column_date_read_internal(DYNAMIC_COLUMN_VALUE *store_it_here, @retval TRUE error */ -static my_bool dynamic_column_init_str(DYNAMIC_COLUMN *str, size_t size) +static my_bool dynamic_column_init_named(DYNAMIC_COLUMN *str, size_t size) { DBUG_ASSERT(size != 0); @@ -82,11 +692,8 @@ static my_bool dynamic_column_init_str(DYNAMIC_COLUMN *str, size_t size) - First \0 is flags - other 2 \0 is number of fields */ - if (init_dynamic_string(str, NULL, - size + FIXED_HEADER_SIZE, DYNCOL_SYZERESERVE)) + if (init_dynamic_string(str, NULL, size, DYNCOL_SYZERESERVE)) return TRUE; - bzero(str->str, FIXED_HEADER_SIZE); - str->length= FIXED_HEADER_SIZE; return FALSE; } @@ -259,7 +866,7 @@ dynamic_column_uint_read(DYNAMIC_COLUMN_VALUE *store_it_here, static size_t dynamic_column_sint_bytes(longlong val) { return dynamic_column_uint_bytes((val << 1) ^ - (val < 0 ? ULL(0xffffffffffffffff) : 0)); + (val < 0 ? 0xffffffffffffffffull : 0)); } @@ -277,7 +884,7 @@ dynamic_column_sint_store(DYNAMIC_COLUMN *str, longlong val) { return dynamic_column_uint_store(str, (val << 1) ^ - (val < 0 ? ULL(0xffffffffffffffff) : 0)); + (val < 0 ? 0xffffffffffffffffULL : 0)); } @@ -299,7 +906,7 @@ dynamic_column_sint_read(DYNAMIC_COLUMN_VALUE *store_it_here, dynamic_column_uint_read(store_it_here, data, length); val= store_it_here->x.ulong_value; if (val & 1) - val= (val >> 1) ^ ULL(0xffffffffffffffff); + val= (val >> 1) ^ 0xffffffffffffffffULL; else val>>= 1; store_it_here->x.long_value= (longlong) val; @@ -318,7 +925,8 @@ dynamic_column_sint_read(DYNAMIC_COLUMN_VALUE *store_it_here, */ static size_t -dynamic_column_value_len(DYNAMIC_COLUMN_VALUE *value) +dynamic_column_value_len(DYNAMIC_COLUMN_VALUE *value, + enum enum_dyncol_format format) { switch (value->type) { case DYN_COL_NULL: @@ -360,14 +968,22 @@ dynamic_column_value_len(DYNAMIC_COLUMN_VALUE *value) decimal_bin_size(precision, scale)); } case DYN_COL_DATETIME: - /* date+time in bits: 14 + 4 + 5 + 10 + 6 + 6 + 20 + 1 66bits ~= 9 bytes */ - return 9; + if (format == dyncol_fmt_num || value->x.time_value.second_part) + /* date+time in bits: 14 + 4 + 5 + 10 + 6 + 6 + 20 + 1 66bits ~= 9 bytes*/ + return 9; + else + return 6; case DYN_COL_DATE: /* date in dits: 14 + 4 + 5 = 23bits ~= 3bytes*/ return 3; case DYN_COL_TIME: - /* time in bits: 10 + 6 + 6 + 20 + 1 = 43bits ~= 6bytes*/ - return 6; + if (format == dyncol_fmt_num || value->x.time_value.second_part) + /* time in bits: 10 + 6 + 6 + 20 + 1 = 43bits ~= 6bytes*/ + return 6; + else + return 3; + case DYN_COL_DYNCOL: + return value->x.string.value.length; } DBUG_ASSERT(0); return 0; @@ -436,6 +1052,22 @@ dynamic_column_string_store(DYNAMIC_COLUMN *str, LEX_STRING *string, return ER_DYNCOL_OK; } +/** + Append the string with given string value. + + @param str the string where to put the value + @param val the value to put in the string + + @return ER_DYNCOL_* return code +*/ + +static enum enum_dyncol_func_result +dynamic_column_dyncol_store(DYNAMIC_COLUMN *str, LEX_STRING *string) +{ + if (dynstr_append_mem(str, string->str, string->length)) + return ER_DYNCOL_RESOURCE; + return ER_DYNCOL_OK; +} /** Read string value of given length from the packed string @@ -464,6 +1096,26 @@ dynamic_column_string_read(DYNAMIC_COLUMN_VALUE *store_it_here, return ER_DYNCOL_OK; } +/** + Read Dynamic columns packet string value of given length + from the packed string + + @param store_it_here The structure to store the value + @param data The packed string which should be read + @param length The length (in bytes) of the value in nthe string + + @return ER_DYNCOL_* return code +*/ + +static enum enum_dyncol_func_result +dynamic_column_dyncol_read(DYNAMIC_COLUMN_VALUE *store_it_here, + uchar *data, size_t length) +{ + store_it_here->x.string.charset= &my_charset_bin; + store_it_here->x.string.value.length= length; + store_it_here->x.string.value.str= (char*) data; + return ER_DYNCOL_OK; +} /** Append the string with given decimal value. @@ -506,7 +1158,7 @@ dynamic_column_decimal_store(DYNAMIC_COLUMN *str, @param value The value structure which sould be setup. */ -void dynamic_column_prepare_decimal(DYNAMIC_COLUMN_VALUE *value) +void mariadb_dyncol_prepare_decimal(DYNAMIC_COLUMN_VALUE *value) { value->x.decimal.value.buf= value->x.decimal.buffer; value->x.decimal.value.len= DECIMAL_BUFF_LENGTH; @@ -515,6 +1167,12 @@ void dynamic_column_prepare_decimal(DYNAMIC_COLUMN_VALUE *value) decimal_make_zero(&value->x.decimal.value); } +void dynamic_column_prepare_decimal(DYNAMIC_COLUMN_VALUE *value) +{ + mariadb_dyncol_prepare_decimal(value); +} + + /** Read decimal value of given length from the string @@ -570,7 +1228,8 @@ dynamic_column_decimal_read(DYNAMIC_COLUMN_VALUE *store_it_here, */ static enum enum_dyncol_func_result -dynamic_column_date_time_store(DYNAMIC_COLUMN *str, MYSQL_TIME *value) +dynamic_column_date_time_store(DYNAMIC_COLUMN *str, MYSQL_TIME *value, + enum enum_dyncol_format format) { enum enum_dyncol_func_result rc; /* @@ -579,7 +1238,7 @@ dynamic_column_date_time_store(DYNAMIC_COLUMN *str, MYSQL_TIME *value) <123456><123456><123456><123456><123456><123456><123456><123456><123456> */ if ((rc= dynamic_column_date_store(str, value)) || - (rc= dynamic_column_time_store(str, value))) + (rc= dynamic_column_time_store(str, value, format))) return rc; return ER_DYNCOL_OK; } @@ -605,11 +1264,12 @@ dynamic_column_date_time_read(DYNAMIC_COLUMN_VALUE *store_it_here, 12345678901234123412345 1123456789012345612345612345678901234567890 <123456><123456><123456><123456><123456><123456><123456><123456><123456> */ - if (length != 9) + if (length != 9 && length != 6) goto err; store_it_here->x.time_value.time_type= MYSQL_TIMESTAMP_DATETIME; if ((rc= dynamic_column_date_read_internal(store_it_here, data, 3)) || - (rc= dynamic_column_time_read_internal(store_it_here, data + 3, 6))) + (rc= dynamic_column_time_read_internal(store_it_here, data + 3, + length - 3))) goto err; return ER_DYNCOL_OK; @@ -629,7 +1289,8 @@ err: */ static enum enum_dyncol_func_result -dynamic_column_time_store(DYNAMIC_COLUMN *str, MYSQL_TIME *value) +dynamic_column_time_store(DYNAMIC_COLUMN *str, MYSQL_TIME *value, + enum enum_dyncol_format format) { uchar *buf; if (dynstr_realloc(str, 6)) @@ -651,19 +1312,35 @@ dynamic_column_time_store(DYNAMIC_COLUMN *str, MYSQL_TIME *value) DBUG_ASSERT(value->minute <= 59); DBUG_ASSERT(value->second <= 59); DBUG_ASSERT(value->second_part <= 999999); + if (format == dyncol_fmt_num || value->second_part) + { /* 00000!<-hours--><min-><sec-><---microseconds---> 1123456789012345612345612345678901234567890 <123456><123456><123456><123456><123456><123456> */ - buf[0]= (value->second_part & 0xff); - buf[1]= ((value->second_part & 0xff00) >> 8); - buf[2]= (uchar)(((value->second & 0xf) << 4) | - ((value->second_part & 0xf0000) >> 16)); - buf[3]= ((value->minute << 2) | ((value->second & 0x30) >> 4)); - buf[4]= (value->hour & 0xff); - buf[5]= ((value->neg ? 0x4 : 0) | (value->hour >> 8)); - str->length+= 6; + buf[0]= (value->second_part & 0xff); + buf[1]= ((value->second_part & 0xff00) >> 8); + buf[2]= (uchar)(((value->second & 0xf) << 4) | + ((value->second_part & 0xf0000) >> 16)); + buf[3]= ((value->minute << 2) | ((value->second & 0x30) >> 4)); + buf[4]= (value->hour & 0xff); + buf[5]= ((value->neg ? 0x4 : 0) | (value->hour >> 8)); + str->length+= 6; + } + else + { + /* + !<-hours--><min-><sec-> + 11234567890123456123456 + <123456><123456><123456> + */ + buf[0]= (value->second) | ((value->minute & 0x3) << 6); + buf[1]= (value->minute >> 2) | ((value->hour & 0xf) << 4); + buf[2]= (value->hour >> 4) | (value->neg ? 0x80 : 0); + str->length+= 3; + } + return ER_DYNCOL_OK; } @@ -702,21 +1379,37 @@ static enum enum_dyncol_func_result dynamic_column_time_read_internal(DYNAMIC_COLUMN_VALUE *store_it_here, uchar *data, size_t length) { - if (length != 6) + if (length != 6 && length != 3) goto err; - /* - 00000!<-hours--><min-><sec-><---microseconds---> - 1123456789012345612345612345678901234567890 - <123456><123456><123456><123456><123456><123456> - */ - store_it_here->x.time_value.second_part= (data[0] | - (data[1] << 8) | - ((data[2] & 0xf) << 16)); - store_it_here->x.time_value.second= ((data[2] >> 4) | - ((data[3] & 0x3) << 4)); - store_it_here->x.time_value.minute= (data[3] >> 2); - store_it_here->x.time_value.hour= (((((uint)data[5]) & 0x3 ) << 8) | data[4]); - store_it_here->x.time_value.neg= ((data[5] & 0x4) ? 1 : 0); + if (length == 6) + { + /* + 00000!<-hours--><min-><sec-><---microseconds---> + 1123456789012345612345612345678901234567890 + <123456><123456><123456><123456><123456><123456> + */ + store_it_here->x.time_value.second_part= (data[0] | + (data[1] << 8) | + ((data[2] & 0xf) << 16)); + store_it_here->x.time_value.second= ((data[2] >> 4) | + ((data[3] & 0x3) << 4)); + store_it_here->x.time_value.minute= (data[3] >> 2); + store_it_here->x.time_value.hour= (((((uint)data[5]) & 0x3 ) << 8) | data[4]); + store_it_here->x.time_value.neg= ((data[5] & 0x4) ? 1 : 0); + } + else + { + /* + !<-hours--><min-><sec-> + 11234567890123456123456 + <123456><123456><123456> + */ + store_it_here->x.time_value.second_part= 0; + store_it_here->x.time_value.second= (data[0] & 0x3f); + store_it_here->x.time_value.minute= (data[0] >> 6) | ((data[1] & 0xf) << 2); + store_it_here->x.time_value.hour= (data[1] >> 4) | ((data[2] & 0x3f) << 4); + store_it_here->x.time_value.neg= ((data[2] & 0x80) ? 1 : 0); + } if (store_it_here->x.time_value.second > 59 || store_it_here->x.time_value.minute > 59 || store_it_here->x.time_value.hour > 838 || @@ -841,7 +1534,8 @@ err: */ static enum enum_dyncol_func_result -data_store(DYNAMIC_COLUMN *str, DYNAMIC_COLUMN_VALUE *value) +data_store(DYNAMIC_COLUMN *str, DYNAMIC_COLUMN_VALUE *value, + enum enum_dyncol_format format) { switch (value->type) { case DYN_COL_INT: @@ -857,13 +1551,15 @@ data_store(DYNAMIC_COLUMN *str, DYNAMIC_COLUMN_VALUE *value) return dynamic_column_decimal_store(str, &value->x.decimal.value); case DYN_COL_DATETIME: /* date+time in bits: 14 + 4 + 5 + 5 + 6 + 6 40bits = 5 bytes */ - return dynamic_column_date_time_store(str, &value->x.time_value); + return dynamic_column_date_time_store(str, &value->x.time_value, format); case DYN_COL_DATE: /* date in dits: 14 + 4 + 5 = 23bits ~= 3bytes*/ return dynamic_column_date_store(str, &value->x.time_value); case DYN_COL_TIME: /* time in bits: 5 + 6 + 6 = 17bits ~= 3bytes*/ - return dynamic_column_time_store(str, &value->x.time_value); + return dynamic_column_time_store(str, &value->x.time_value, format); + case DYN_COL_DYNCOL: + return dynamic_column_dyncol_store(str, &value->x.string.value); case DYN_COL_NULL: break; /* Impossible */ } @@ -873,117 +1569,6 @@ data_store(DYNAMIC_COLUMN *str, DYNAMIC_COLUMN_VALUE *value) /** - Calculate length of offset field for given data length - - @param data_length Length of the data segment - - @return number of bytes -*/ - -static size_t dynamic_column_offset_bytes(size_t data_length) -{ - if (data_length < 0x1f) /* all 1 value is reserved */ - return 1; - if (data_length < 0x1fff) /* all 1 value is reserved */ - return 2; - if (data_length < 0x1fffff) /* all 1 value is reserved */ - return 3; - if (data_length < 0x1fffffff) /* all 1 value is reserved */ - return 4; - return MAX_OFFSET_LENGTH; /* For future */ -} - -/** - Store offset and type information in the given place - - @param place Beginning of the index entry - @param offset_size Size of offset field in bytes - @param type Type to be written - @param offset Offset to be written -*/ - -static void type_and_offset_store(uchar *place, size_t offset_size, - DYNAMIC_COLUMN_TYPE type, - size_t offset) -{ - ulong val = (((ulong) offset) << 3) | (type - 1); - DBUG_ASSERT(type != DYN_COL_NULL); - DBUG_ASSERT(((type - 1) & (~7)) == 0); /* fit in 3 bits */ - - /* Index entry starts with column number; Jump over it */ - place+= COLUMN_NUMBER_SIZE; - - switch (offset_size) { - case 1: - DBUG_ASSERT(offset < 0x1f); /* all 1 value is reserved */ - place[0]= (uchar)val; - break; - case 2: - DBUG_ASSERT(offset < 0x1fff); /* all 1 value is reserved */ - int2store(place, val); - break; - case 3: - DBUG_ASSERT(offset < 0x1fffff); /* all 1 value is reserved */ - int3store(place, val); - break; - case 4: - DBUG_ASSERT(offset < 0x1fffffff); /* all 1 value is reserved */ - int4store(place, val); - break; - default: - DBUG_ASSERT(0); /* impossible */ - } -} - - -/** - Read offset and type information from index entry - - @param type Where to put type info - @param offset Where to put offset info - @param place Beginning of the index entry - @param offset_size Size of offset field in bytes -*/ - -static void type_and_offset_read(DYNAMIC_COLUMN_TYPE *type, - size_t *offset, - uchar *place, size_t offset_size) -{ - ulong UNINIT_VAR(val); - - place+= COLUMN_NUMBER_SIZE; /* skip column number */ - switch (offset_size) { - case 1: - val= (ulong)place[0]; - break; - case 2: - val= uint2korr(place); - break; - case 3: - val= uint3korr(place); - break; - case 4: - val= uint4korr(place); - break; - default: - DBUG_ASSERT(0); /* impossible */ - } - *type= (val & 0x7) + 1; - *offset= val >> 3; -} - - -/** - Comparator function for references on column numbers for qsort -*/ - -static int column_sort(const void *a, const void *b) -{ - return **((uint **)a) - **((uint **)b); -} - - -/** Write information to the fixed header @param str String where to write the header @@ -996,34 +1581,20 @@ static void set_fixed_header(DYNAMIC_COLUMN *str, uint column_count) { DBUG_ASSERT(column_count <= 0xffff); - DBUG_ASSERT(offset_size <= 4); + DBUG_ASSERT(offset_size <= MAX_OFFSET_LENGTH); str->str[0]= ((str->str[0] & ~DYNCOL_FLG_OFFSET) | (offset_size - 1)); /* size of offset */ int2store(str->str + 1, column_count); /* columns number */ DBUG_ASSERT((str->str[0] & (~DYNCOL_FLG_KNOWN)) == 0); } -/* - Calculate entry size (E) and header size (H) by offset size (O) and column - count (C). -*/ - -#define calc_param(E,H,O,C) do { \ - (*(E))= (O) + COLUMN_NUMBER_SIZE; \ - (*(H))= (*(E)) * (C); \ -}while(0); - - /** Adds columns into the empty string - @param str String where to write the data - @param header_size Size of the header without fixed part - @param offset_size Size of offset field in bytes + @param str String where to write the data (the record) + @param hdr Dynamic columns record descriptor @param column_count Number of columns in the arrays - @parem not_null_count Number of non-null columns in the arrays - @param data_size Size of the data segment - @param column_numbers Array of columns numbers + @param column_keys Array of columns keys (uint or LEX_STRING) @param values Array of columns values @param new_str True if we need to allocate new string @@ -1032,42 +1603,61 @@ static void set_fixed_header(DYNAMIC_COLUMN *str, static enum enum_dyncol_func_result dynamic_new_column_store(DYNAMIC_COLUMN *str, - size_t header_size, - size_t offset_size, + DYN_HEADER *hdr, uint column_count, - uint not_null_count, - size_t data_size, - uint *column_numbers, + void *column_keys, DYNAMIC_COLUMN_VALUE *values, my_bool new_str) { - uchar *header_end; - uint **columns_order; + struct st_service_funcs *fmt= fmt_data + hdr->format; + void **UNINIT_VAR(columns_order); + uchar *element; uint i; - uint entry_size= COLUMN_NUMBER_SIZE + offset_size; enum enum_dyncol_func_result rc= ER_DYNCOL_RESOURCE; + size_t all_headers_size; - if (!(columns_order= malloc(sizeof(uint*)*column_count))) + if (column_count && !(columns_order= malloc(sizeof(void*)*column_count))) return ER_DYNCOL_RESOURCE; - if (new_str) + if (new_str || str->str == 0) { - if (dynamic_column_init_str(str, - data_size + header_size + DYNCOL_SYZERESERVE)) - goto err; + if (column_count) + { + if (dynamic_column_init_named(str, + fmt->fixed_hdr + + hdr->header_size + + hdr->nmpool_size + + hdr->data_size + + DYNCOL_SYZERESERVE)) + goto err; + } + else + { + mariadb_dyncol_init(str); + } } else { str->length= 0; - if (dynstr_realloc(str, data_size + header_size + DYNCOL_SYZERESERVE)) + if (dynstr_realloc(str, + fmt->fixed_hdr + + hdr->header_size + + hdr->nmpool_size + + hdr->data_size + + DYNCOL_SYZERESERVE)) goto err; - bzero(str->str, FIXED_HEADER_SIZE); - str->length= FIXED_HEADER_SIZE; } + if (!column_count) + return ER_DYNCOL_OK; + + bzero(str->str, fmt->fixed_hdr); + str->length= fmt->fixed_hdr; /* sort columns for the header */ - for (i= 0; i < column_count; i++) - columns_order[i]= column_numbers + i; - qsort(columns_order, (size_t)column_count, sizeof(uint*), &column_sort); + for (i= 0, element= (uchar *) column_keys; + i < column_count; + i++, element+= fmt->key_size_in_array) + columns_order[i]= (void *)element; + qsort(columns_order, (size_t)column_count, sizeof(void*), fmt->column_sort); /* For now we don't allow creating two columns with the same number @@ -1076,38 +1666,43 @@ dynamic_new_column_store(DYNAMIC_COLUMN *str, */ for (i= 0; i < column_count - 1; i++) { - if (columns_order[i][0] > UINT_MAX16 || - columns_order[i][0] == columns_order[i + 1][0]) + if ((*fmt->check_limit)(&columns_order[i]) || + (*fmt->column_sort)(&columns_order[i], &columns_order[i + 1]) == 0) { rc= ER_DYNCOL_DATA; goto err; } } - if (columns_order[i][0] > UINT_MAX16) + if ((*fmt->check_limit)(&columns_order[i])) { rc= ER_DYNCOL_DATA; goto err; } - DBUG_ASSERT(str->max_length >= str->length + header_size); - set_fixed_header(str, offset_size, not_null_count); - str->length+= header_size; /* reserve place for header */ - header_end= (uchar *)str->str + FIXED_HEADER_SIZE; + (*fmt->set_fixed_hdr)(str, hdr); + /* reserve place for header and name pool */ + str->length+= hdr->header_size + hdr->nmpool_size; + + hdr->entry= hdr->header; + hdr->name= hdr->nmpool; + all_headers_size= fmt->fixed_hdr + hdr->header_size + hdr->nmpool_size; for (i= 0; i < column_count; i++) { - uint ord= columns_order[i] - column_numbers; + uint ord= (uint)(((uchar*)columns_order[i] - (uchar*)column_keys) / + fmt->key_size_in_array); if (values[ord].type != DYN_COL_NULL) { /* Store header first in the str */ - int2store(header_end, column_numbers[ord]); - type_and_offset_store(header_end, offset_size, - values[ord].type, - str->length - header_size - FIXED_HEADER_SIZE); + if ((*fmt->put_header_entry)(hdr, columns_order[i], values + ord, + str->length - all_headers_size)) + { + rc= ER_DYNCOL_FORMAT; + goto err; + } /* Store value in 'str + str->length' and increase str->length */ - if ((rc= data_store(str, values + ord))) + if ((rc= data_store(str, values + ord, hdr->format))) goto err; - header_end+= entry_size; } } rc= ER_DYNCOL_OK; @@ -1117,61 +1712,92 @@ err: } /** - Create packed string which contains given columns (internal) + Calculate size of header, name pool and data pool - @param str String where to write the data + @param hdr descriptor of dynamic column record + @param column_count number of elements in arrays @param column_count Number of columns in the arrays - @param column_numbers Array of columns numbers + @param column_keys Array of columns keys (uint or LEX_STRING) @param values Array of columns values - @param new_str True if we need allocate new string @return ER_DYNCOL_* return code */ static enum enum_dyncol_func_result -dynamic_column_create_many_internal(DYNAMIC_COLUMN *str, - uint column_count, - uint *column_numbers, - DYNAMIC_COLUMN_VALUE *values, - my_bool new_str) -{ - size_t data_size= 0; - size_t header_size, offset_size; +calc_var_sizes(DYN_HEADER *hdr, + uint column_count, + void *column_keys, + DYNAMIC_COLUMN_VALUE *values) +{ + struct st_service_funcs *fmt= fmt_data + hdr->format; uint i; - int not_null_column_count= 0; - - if (new_str) - { - /* to make dynstr_free() working in case of errors */ - bzero(str, sizeof(DYNAMIC_COLUMN)); - } - + hdr->nmpool_size= hdr->data_size= 0; + hdr->column_count= 0; for (i= 0; i < column_count; i++) { if (values[i].type != DYN_COL_NULL) { size_t tmp; - not_null_column_count++; - data_size+= (tmp=dynamic_column_value_len(values + i)); + hdr->column_count++; + hdr->data_size+= (tmp= dynamic_column_value_len(values + i, + hdr->format)); if (tmp == (size_t) ~0) return ER_DYNCOL_DATA; + hdr->nmpool_size+= (*fmt->name_size)(column_keys, i); } } - - /* We can handle data up to 1fffffff = 536870911 bytes now */ - if ((offset_size= dynamic_column_offset_bytes(data_size)) >= - MAX_OFFSET_LENGTH) + /* + We can handle data up to 0x1fffffff (old format) and + 0xfffffffff (new format) bytes now. + */ + if ((hdr->offset_size= fmt->dynamic_column_offset_bytes(hdr->data_size)) >= + fmt->max_offset_size) return ER_DYNCOL_LIMIT; - /* header entry is column number + offset & type */ - header_size= not_null_column_count * (offset_size + 2); + /* header entry is column number or string pointer + offset & type */ + hdr->entry_size= fmt->fixed_hdr_entry + hdr->offset_size; + hdr->header_size= hdr->column_count * hdr->entry_size; + return ER_DYNCOL_OK; +} + +/** + Create packed string which contains given columns (internal multi format) + + @param str String where to write the data + @param column_count Number of columns in the arrays + @param column_keys Array of columns keys (format dependent) + @param values Array of columns values + @param new_str True if we need allocate new string + @param string_keys keys are strings - return dynamic_new_column_store(str, - header_size, offset_size, + @return ER_DYNCOL_* return code +*/ + +static enum enum_dyncol_func_result +dynamic_column_create_many_internal_fmt(DYNAMIC_COLUMN *str, + uint column_count, + void *column_keys, + DYNAMIC_COLUMN_VALUE *values, + my_bool new_str, + my_bool string_keys) +{ + DYN_HEADER header; + enum enum_dyncol_func_result rc; + bzero(&header, sizeof(header)); + header.format= (string_keys ? 1 : 0); + + if (new_str) + { + /* to make dynstr_free() working in case of errors */ + mariadb_dyncol_init(str); + } + + if ((rc= calc_var_sizes(&header, column_count, column_keys, values)) < 0) + return rc; + + return dynamic_new_column_store(str, &header, column_count, - not_null_column_count, - data_size, - column_numbers, values, + column_keys, values, new_str); } @@ -1194,11 +1820,60 @@ dynamic_column_create_many(DYNAMIC_COLUMN *str, DYNAMIC_COLUMN_VALUE *values) { DBUG_ENTER("dynamic_column_create_many"); - DBUG_RETURN(dynamic_column_create_many_internal(str, column_count, - column_numbers, values, - TRUE)); + DBUG_RETURN(dynamic_column_create_many_internal_fmt(str, column_count, + column_numbers, values, + TRUE, FALSE)); } +/** + Create packed string which contains given columns + + @param str String where to write the data + @param column_count Number of columns in the arrays + @param column_numbers Array of columns numbers + @param values Array of columns values + @param new_string True if we need allocate new string + + @return ER_DYNCOL_* return code +*/ + +enum enum_dyncol_func_result +mariadb_dyncol_create_many_num(DYNAMIC_COLUMN *str, + uint column_count, + uint *column_numbers, + DYNAMIC_COLUMN_VALUE *values, + my_bool new_string) +{ + DBUG_ENTER("mariadb_dyncol_create_many_num"); + DBUG_RETURN(dynamic_column_create_many_internal_fmt(str, column_count, + column_numbers, values, + new_string, FALSE)); +} + +/** + Create packed string which contains given columns + + @param str String where to write the data + @param column_count Number of columns in the arrays + @param column_keys Array of columns keys + @param values Array of columns value + @param new_string True if we need allocate new string + + @return ER_DYNCOL_* return code +*/ + +enum enum_dyncol_func_result +mariadb_dyncol_create_many_named(DYNAMIC_COLUMN *str, + uint column_count, + LEX_STRING *column_keys, + DYNAMIC_COLUMN_VALUE *values, + my_bool new_string) +{ + DBUG_ENTER("mariadb_dyncol_create_many_named"); + DBUG_RETURN(dynamic_column_create_many_internal_fmt(str, column_count, + column_keys, values, + new_string, TRUE)); +} /** Create packed string which contains given column @@ -1227,50 +1902,63 @@ dynamic_column_create(DYNAMIC_COLUMN *str, uint column_nr, @param header_end Pointer to the header end @param offset_size Size of offset field in bytes @param last_offset Size of the data segment - @param error Set in case of error @return number of bytes */ static size_t get_length_interval(uchar *entry, uchar *entry_next, uchar *header_end, size_t offset_size, - size_t last_offset, my_bool *error) + size_t last_offset) { size_t offset, offset_next; DYNAMIC_COLUMN_TYPE type, type_next; DBUG_ASSERT(entry < entry_next); - type_and_offset_read(&type, &offset, entry, offset_size); + if (type_and_offset_read_num(&type, &offset, entry + COLUMN_NUMBER_SIZE, + offset_size)) + return DYNCOL_OFFSET_ERROR; if (entry_next >= header_end) - { - *error= 0; return (last_offset - offset); - } - type_and_offset_read(&type_next, &offset_next, entry_next, offset_size); - *error= (offset_next > last_offset); + if (type_and_offset_read_num(&type_next, &offset_next, + entry_next + COLUMN_NUMBER_SIZE, offset_size) || + (offset_next > last_offset)) + return DYNCOL_OFFSET_ERROR; return (offset_next - offset); } -/* - Calculate length of data of one column +/** + Calculate length of data between given hdr->entry and next_entry - @param entry Pointer to the first entry - @param header_end Pointer to the header end - @param offset_size Size of offset field in bytes - @param last_offset Size of the data segment - @param error Set in case of error + @param hdr descriptor of dynamic column record + @param next_entry next header entry (can point just after last header + entry) @return number of bytes */ -static size_t get_length(uchar *entry, uchar *header_end, - size_t offset_size, - size_t last_offset, my_bool *error) +static size_t hdr_interval_length(DYN_HEADER *hdr, uchar *next_entry) { - return get_length_interval(entry, - entry + offset_size + COLUMN_NUMBER_SIZE, - header_end, offset_size, last_offset, error); + struct st_service_funcs *fmt= fmt_data + hdr->format; + size_t next_entry_offset; + DYNAMIC_COLUMN_TYPE next_entry_type; + DBUG_ASSERT(hdr->entry < next_entry); + DBUG_ASSERT(hdr->entry >= hdr->header); + DBUG_ASSERT(next_entry <= hdr->header + hdr->header_size); + + if ((*fmt->type_and_offset_read)(&hdr->type, &hdr->offset, + hdr->entry + fmt->fixed_hdr_entry, + hdr->offset_size) || + hdr->data_size < hdr->offset) + return DYNCOL_OFFSET_ERROR; + if (next_entry == hdr->header + hdr->header_size) + return hdr->data_size - hdr->offset; + if ((*fmt->type_and_offset_read)(&next_entry_type, &next_entry_offset, + next_entry + fmt->fixed_hdr_entry, + hdr->offset_size) || + hdr->data_size < next_entry_offset) + return DYNCOL_OFFSET_ERROR; + return (next_entry_offset - hdr->offset); } @@ -1278,7 +1966,7 @@ static size_t get_length(uchar *entry, uchar *header_end, Comparator function for references to header entries for qsort */ -static int header_compar(const void *a, const void *b) +static int header_compar_num(const void *a, const void *b) { uint va= uint2korr((uchar*)a), vb= uint2korr((uchar*)b); return (va > vb ? 1 : (va < vb ? -1 : 0)); @@ -1286,71 +1974,183 @@ static int header_compar(const void *a, const void *b) /** + Find entry in the numeric format header by the column number + + @param hdr descriptor of dynamic column record + @param key number to find + + @return pointer to the entry or NULL +*/ + +static uchar *find_entry_num(DYN_HEADER *hdr, uint key) +{ + uchar header_entry[2+4]; + DBUG_ASSERT(hdr->format == dyncol_fmt_num); + int2store(header_entry, key); + return hdr->entry= bsearch(header_entry, hdr->header, + (size_t)hdr->column_count, + hdr->entry_size, &header_compar_num); +} + + +/** + Read name from header entry + + @param hdr descriptor of dynamic column record + @param entry pointer to the header entry + @param name where to put name + + @return 0 ok + @return 1 error in data +*/ + +static my_bool read_name(DYN_HEADER *hdr, uchar *entry, LEX_STRING *name) +{ + size_t nmoffset= uint2korr(entry); + uchar *next_entry= entry + hdr->entry_size; + + if (nmoffset > hdr->nmpool_size) + return 1; + + name->str= (char *)hdr->nmpool + nmoffset; + if (next_entry == hdr->header + hdr->header_size) + name->length= hdr->nmpool_size - nmoffset; + else + { + size_t next_nmoffset= uint2korr(next_entry); + if (next_nmoffset > hdr->nmpool_size) + return 1; + name->length= next_nmoffset - nmoffset; + } + return 0; +} + + +/** + Find entry in the names format header by the column number + + @param hdr descriptor of dynamic column record + @param key name to find + + @return pointer to the entry or NULL +*/ +static uchar *find_entry_named(DYN_HEADER *hdr, LEX_STRING *key) +{ + uchar *min= hdr->header; + uchar *max= hdr->header + (hdr->column_count - 1) * hdr->entry_size; + uchar *mid; + DBUG_ASSERT(hdr->format == dyncol_fmt_str); + DBUG_ASSERT(hdr->nmpool != NULL); + while (max >= min) + { + LEX_STRING name; + int cmp; + mid= hdr->header + ((min - hdr->header) + + (max - hdr->header)) / + 2 / + hdr->entry_size * hdr->entry_size; + if (read_name(hdr, mid, &name)) + return NULL; + cmp= mariadb_dyncol_column_cmp_named(&name, key); + if (cmp < 0) + min= mid + hdr->entry_size; + else if (cmp > 0) + max= mid - hdr->entry_size; + else + return mid; + } + return NULL; +} + + +/** + Write number in the buffer (backward direction - starts from the buffer end) + + @return pointer on the number beginning +*/ + +static char *backwritenum(char *chr, uint numkey) +{ + if (numkey == 0) + *(--chr)= '0'; + else + while (numkey > 0) + { + *(--chr)= '0' + numkey % 10; + numkey/= 10; + } + return chr; +} + + +/** Find column and fill information about it - @param type Returns type of the column - @param data Returns a pointer to the data - @param length Returns length of the data - @param offset_size Size of offset field in bytes - @param column_count Number of column in the packed string - @param data_end Pointer to the data end - @param num Number of the column we want to fetch - @param entry_pos NULL or place where to put reference to the entry + @param hdr descriptor of dynamic column record + @param numkey Number of the column to fetch (if strkey is NULL) + @param strkey Name of the column to fetch (or NULL) @return 0 ok @return 1 error in data */ static my_bool -find_column(DYNAMIC_COLUMN_TYPE *type, uchar **data, size_t *length, - uchar *header, size_t offset_size, uint column_count, - uchar *data_end, uint num, uchar **entry_pos) +find_column(DYN_HEADER *hdr, uint numkey, LEX_STRING *strkey) { - uchar *entry; - size_t offset, total_data, header_size, entry_size; - uchar key[2+4]; - my_bool error; + LEX_STRING nmkey; + char nmkeybuff[DYNCOL_NUM_CHAR]; /* to fit max 2 bytes number */ + DBUG_ASSERT(hdr->header != NULL); - if (!entry_pos) - entry_pos= &entry; - - calc_param(&entry_size, &header_size, offset_size, column_count); + if (hdr->header + hdr->header_size > hdr->data_end) + return TRUE; - if (header + header_size > data_end) - return 1; + /* fix key */ + if (hdr->format == dyncol_fmt_num && strkey != NULL) + { + char *end; + numkey= (uint) strtoul(strkey->str, &end, 10); + if (end != strkey->str + strkey->length) + { + /* we can't find non-numeric key among numeric ones */ + hdr->type= DYN_COL_NULL; + return 0; + } + } + else if (hdr->format == dyncol_fmt_str && strkey == NULL) + { + nmkey.str= backwritenum(nmkeybuff + sizeof(nmkeybuff), numkey); + nmkey.length= (nmkeybuff + sizeof(nmkeybuff)) - nmkey.str; + strkey= &nmkey; + } + if (hdr->format == dyncol_fmt_num) + hdr->entry= find_entry_num(hdr, numkey); + else + hdr->entry= find_entry_named(hdr, strkey); - int2store(key, num); - entry= bsearch(key, header, (size_t)column_count, entry_size, - &header_compar); - if (!entry) + if (!hdr->entry) { /* Column not found */ - *type= DYN_COL_NULL; - *entry_pos= NULL; + hdr->type= DYN_COL_NULL; return 0; } - type_and_offset_read(type, &offset, entry, offset_size); - total_data= data_end - (header + header_size); - if (offset > total_data) - return 1; - *data= header + header_size + offset; - *length= get_length(entry, header + header_size, offset_size, - total_data, &error); + hdr->length= hdr_interval_length(hdr, hdr->entry + hdr->entry_size); + hdr->data= hdr->dtpool + hdr->offset; /* - Check that the found data is withing the ranges. This can happen if + Check that the found data is within the ranges. This can happen if we get data with wrong offsets. */ - if (error || (long) *length < 0 || offset + *length > total_data) + if (hdr->length == DYNCOL_OFFSET_ERROR || + hdr->length > INT_MAX || hdr->offset > hdr->data_size) return 1; - *entry_pos= entry; return 0; } /** - Read and check the header of the dynamic string + Read and check the header of the dynamic string + @param hdr descriptor of dynamic column record @param str Dynamic string @retval FALSE OK @@ -1361,22 +2161,31 @@ find_column(DYNAMIC_COLUMN_TYPE *type, uchar **data, size_t *length, already have handled this case. */ -static inline my_bool read_fixed_header(DYNAMIC_COLUMN *str, - size_t *offset_size, - uint *column_count) +static inline my_bool read_fixed_header(DYN_HEADER *hdr, + DYNAMIC_COLUMN *str) { DBUG_ASSERT(str != NULL && str->length != 0); - if ((str->length < FIXED_HEADER_SIZE) || + if ((str->length < 1) || (str->str[0] & (~DYNCOL_FLG_KNOWN))) + return 1; + hdr->format= ((str->str[0] & DYNCOL_FLG_NAMES) ? + dyncol_fmt_str: + dyncol_fmt_num); + if ((str->length < fmt_data[hdr->format].fixed_hdr)) return 1; /* Wrong header */ - *offset_size= (str->str[0] & DYNCOL_FLG_OFFSET) + 1; - *column_count= uint2korr(str->str + 1); + hdr->offset_size= (str->str[0] & DYNCOL_FLG_OFFSET) + 1 + + (hdr->format == dyncol_fmt_str ? 1 : 0); + hdr->column_count= uint2korr(str->str + 1); + if (hdr->format == dyncol_fmt_str) + hdr->nmpool_size= uint2korr(str->str + 3); // only 2 bytes supported for now + else + hdr->nmpool_size= 0; return 0; } /** - Get dynamic column value + Get dynamic column value by column number @param str The packed string to extract the column @param column_nr Number of column to fetch @@ -1389,256 +2198,232 @@ enum enum_dyncol_func_result dynamic_column_get(DYNAMIC_COLUMN *str, uint column_nr, DYNAMIC_COLUMN_VALUE *store_it_here) { - uchar *data; - size_t offset_size, length; - uint column_count; - enum enum_dyncol_func_result rc= ER_DYNCOL_FORMAT; + return dynamic_column_get_internal(str, store_it_here, column_nr, NULL); +} - if (str->length == 0) - goto null; +enum enum_dyncol_func_result +mariadb_dyncol_get_num(DYNAMIC_COLUMN *str, uint column_nr, + DYNAMIC_COLUMN_VALUE *store_it_here) +{ + return dynamic_column_get_internal(str, store_it_here, column_nr, NULL); +} - if (read_fixed_header(str, &offset_size, &column_count)) - goto err; - if (column_count == 0) - goto null; +/** + Get dynamic column value by name - if (find_column(&store_it_here->type, &data, &length, - (uchar*)str->str + FIXED_HEADER_SIZE, - offset_size, column_count, (uchar*)str->str + str->length, - column_nr, NULL)) - goto err; + @param str The packed string to extract the column + @param name Name of column to fetch + @param store_it_here Where to store the extracted value - switch (store_it_here->type) { + @return ER_DYNCOL_* return code +*/ + +enum enum_dyncol_func_result +mariadb_dyncol_get_named(DYNAMIC_COLUMN *str, LEX_STRING *name, + DYNAMIC_COLUMN_VALUE *store_it_here) +{ + DBUG_ASSERT(name != NULL); + return dynamic_column_get_internal(str, store_it_here, 0, name); +} + + +static enum enum_dyncol_func_result +dynamic_column_get_value(DYN_HEADER *hdr, DYNAMIC_COLUMN_VALUE *store_it_here) +{ + static enum enum_dyncol_func_result rc; + switch ((store_it_here->type= hdr->type)) { case DYN_COL_INT: - rc= dynamic_column_sint_read(store_it_here, data, length); + rc= dynamic_column_sint_read(store_it_here, hdr->data, hdr->length); break; case DYN_COL_UINT: - rc= dynamic_column_uint_read(store_it_here, data, length); + rc= dynamic_column_uint_read(store_it_here, hdr->data, hdr->length); break; case DYN_COL_DOUBLE: - rc= dynamic_column_double_read(store_it_here, data, length); + rc= dynamic_column_double_read(store_it_here, hdr->data, hdr->length); break; case DYN_COL_STRING: - rc= dynamic_column_string_read(store_it_here, data, length); + rc= dynamic_column_string_read(store_it_here, hdr->data, hdr->length); break; case DYN_COL_DECIMAL: - rc= dynamic_column_decimal_read(store_it_here, data, length); + rc= dynamic_column_decimal_read(store_it_here, hdr->data, hdr->length); break; case DYN_COL_DATETIME: - rc= dynamic_column_date_time_read(store_it_here, data, length); + rc= dynamic_column_date_time_read(store_it_here, hdr->data, + hdr->length); break; case DYN_COL_DATE: - rc= dynamic_column_date_read(store_it_here, data, length); + rc= dynamic_column_date_read(store_it_here, hdr->data, hdr->length); break; case DYN_COL_TIME: - rc= dynamic_column_time_read(store_it_here, data, length); + rc= dynamic_column_time_read(store_it_here, hdr->data, hdr->length); break; case DYN_COL_NULL: rc= ER_DYNCOL_OK; break; + case DYN_COL_DYNCOL: + rc= dynamic_column_dyncol_read(store_it_here, hdr->data, hdr->length); + break; default: - goto err; + rc= ER_DYNCOL_FORMAT; + store_it_here->type= DYN_COL_NULL; + break; } return rc; - -null: - rc= ER_DYNCOL_OK; -err: - store_it_here->type= DYN_COL_NULL; - return rc; } /** - Delete column with given number from the packed string + Get dynamic column value by number or name - @param str The packed string to delete the column - @param column_nr Number of column to delete + @param str The packed string to extract the column + @param store_it_here Where to store the extracted value + @param numkey Number of the column to fetch (if strkey is NULL) + @param strkey Name of the column to fetch (or NULL) @return ER_DYNCOL_* return code */ -enum enum_dyncol_func_result -dynamic_column_delete(DYNAMIC_COLUMN *str, uint column_nr) +static enum enum_dyncol_func_result +dynamic_column_get_internal(DYNAMIC_COLUMN *str, + DYNAMIC_COLUMN_VALUE *store_it_here, + uint num_key, LEX_STRING *str_key) { - uchar *data, *header_entry, *read, *write; - size_t offset_size, new_offset_size, length, entry_size, new_entry_size, - header_size, new_header_size, data_size, new_data_size, - deleted_entry_offset; - uint column_count, i; - DYNAMIC_COLUMN_TYPE type; + DYN_HEADER header; + enum enum_dyncol_func_result rc= ER_DYNCOL_FORMAT; + bzero(&header, sizeof(header)); if (str->length == 0) - return ER_DYNCOL_OK; /* no columns */ + goto null; - if (read_fixed_header(str, &offset_size, &column_count)) - return ER_DYNCOL_FORMAT; + if ((rc= init_read_hdr(&header, str)) < 0) + goto err; - if (column_count == 0) - { - str->length= 0; - return ER_DYNCOL_OK; /* no columns */ - } + if (header.column_count == 0) + goto null; - if (find_column(&type, &data, &length, (uchar*)str->str + FIXED_HEADER_SIZE, - offset_size, column_count, (uchar*)str->str + str->length, - column_nr, &header_entry)) - return ER_DYNCOL_FORMAT; + if (find_column(&header, num_key, str_key)) + goto err; - if (type == DYN_COL_NULL) - return ER_DYNCOL_OK; /* no such column */ + rc= dynamic_column_get_value(&header, store_it_here); + return rc; - if (column_count == 1) - { - /* delete the only column; Return empty string */ - str->length= 0; - return ER_DYNCOL_OK; - } +null: + rc= ER_DYNCOL_OK; +err: + store_it_here->type= DYN_COL_NULL; + return rc; +} - /* Calculate entry_size and header_size */ - calc_param(&entry_size, &header_size, offset_size, column_count); - data_size= str->length - FIXED_HEADER_SIZE - header_size; - new_data_size= data_size - length; - if ((new_offset_size= dynamic_column_offset_bytes(new_data_size)) >= - MAX_OFFSET_LENGTH) - return ER_DYNCOL_LIMIT; - DBUG_ASSERT(new_offset_size <= offset_size); +/** + Check existence of the column in the packed string (by number) - calc_param(&new_entry_size, &new_header_size, - new_offset_size, column_count - 1); + @param str The packed string to check the column + @param column_nr Number of column to check - deleted_entry_offset= ((data - (uchar*) str->str) - - header_size - FIXED_HEADER_SIZE); + @return ER_DYNCOL_* return code +*/ - /* rewrite header*/ - set_fixed_header(str, new_offset_size, column_count - 1); - for (i= 0, write= read= (uchar *)str->str + FIXED_HEADER_SIZE; - i < column_count; - i++, read+= entry_size, write+= new_entry_size) - { - size_t offs; - uint nm; - DYNAMIC_COLUMN_TYPE tp; - if (read == header_entry) - { -#ifndef DBUG_OFF - nm= uint2korr(read); - type_and_offset_read(&tp, &offs, read, - offset_size); - DBUG_ASSERT(nm == column_nr); - DBUG_ASSERT(offs == deleted_entry_offset); -#endif - write-= new_entry_size; /* do not move writer */ - continue; /* skip removed field */ - } +enum enum_dyncol_func_result +dynamic_column_exists(DYNAMIC_COLUMN *str, uint column_nr) +{ + return dynamic_column_exists_internal(str, column_nr, NULL); +} - nm= uint2korr(read), - type_and_offset_read(&tp, &offs, read, - offset_size); +enum enum_dyncol_func_result +mariadb_dyncol_exists_num(DYNAMIC_COLUMN *str, uint column_nr) +{ + return dynamic_column_exists_internal(str, column_nr, NULL); +} - if (offs > deleted_entry_offset) - offs-= length; /* data stored after removed data */ +/** + Check existence of the column in the packed string (by name) - int2store(write, nm); - type_and_offset_store(write, new_offset_size, tp, offs); - } + @param str The packed string to check the column + @param name Name of column to check - /* move data */ - { - size_t first_chunk_len= ((data - (uchar *)str->str) - - FIXED_HEADER_SIZE - header_size); - size_t second_chunk_len= new_data_size - first_chunk_len; - if (first_chunk_len) - memmove(str->str + FIXED_HEADER_SIZE + new_header_size, - str->str + FIXED_HEADER_SIZE + header_size, - first_chunk_len); - if (second_chunk_len) - memmove(str->str + - FIXED_HEADER_SIZE + new_header_size + first_chunk_len, - str->str + - FIXED_HEADER_SIZE + header_size + first_chunk_len + length, - second_chunk_len); - } - - /* fix str length */ - DBUG_ASSERT(str->length >= - FIXED_HEADER_SIZE + new_header_size + new_data_size); - str->length= FIXED_HEADER_SIZE + new_header_size + new_data_size; + @return ER_DYNCOL_* return code +*/ - return ER_DYNCOL_OK; +enum enum_dyncol_func_result +mariadb_dyncol_exists_named(DYNAMIC_COLUMN *str, LEX_STRING *name) +{ + DBUG_ASSERT(name != NULL); + return dynamic_column_exists_internal(str, 0, name); } /** - Check existence of the column in the packed string + Check existence of the column in the packed string (by name of number) @param str The packed string to check the column - @param column_nr Number of column to check + @param num_key Number of the column to fetch (if strkey is NULL) + @param str_key Name of the column to fetch (or NULL) @return ER_DYNCOL_* return code */ -enum enum_dyncol_func_result -dynamic_column_exists(DYNAMIC_COLUMN *str, uint column_nr) +static enum enum_dyncol_func_result +dynamic_column_exists_internal(DYNAMIC_COLUMN *str, uint num_key, + LEX_STRING *str_key) { - uchar *data; - size_t offset_size, length; - uint column_count; - DYNAMIC_COLUMN_TYPE type; + DYN_HEADER header; + enum enum_dyncol_func_result rc; + bzero(&header, sizeof(header)); if (str->length == 0) return ER_DYNCOL_NO; /* no columns */ - if (read_fixed_header(str, &offset_size, &column_count)) - return ER_DYNCOL_FORMAT; + if ((rc= init_read_hdr(&header, str)) < 0) + return rc; - if (column_count == 0) + if (header.column_count == 0) return ER_DYNCOL_NO; /* no columns */ - if (find_column(&type, &data, &length, (uchar*)str->str + FIXED_HEADER_SIZE, - offset_size, column_count, (uchar*)str->str + str->length, - column_nr, NULL)) + if (find_column(&header, num_key, str_key)) return ER_DYNCOL_FORMAT; - return (type != DYN_COL_NULL ? ER_DYNCOL_YES : ER_DYNCOL_NO); + return (header.type != DYN_COL_NULL ? ER_DYNCOL_YES : ER_DYNCOL_NO); } /** - List not-null columns in the packed string + List not-null columns in the packed string (only numeric format) @param str The packed string @param array_of_uint Where to put reference on created array @return ER_DYNCOL_* return code */ - enum enum_dyncol_func_result dynamic_column_list(DYNAMIC_COLUMN *str, DYNAMIC_ARRAY *array_of_uint) { + DYN_HEADER header; uchar *read; - size_t offset_size, entry_size; - uint column_count, i; + uint i; + enum enum_dyncol_func_result rc; bzero(array_of_uint, sizeof(*array_of_uint)); /* In case of errors */ if (str->length == 0) return ER_DYNCOL_OK; /* no columns */ - if (read_fixed_header(str, &offset_size, &column_count)) - return ER_DYNCOL_FORMAT; + if ((rc= init_read_hdr(&header, str)) < 0) + return rc; - entry_size= COLUMN_NUMBER_SIZE + offset_size; + if (header.format != dyncol_fmt_num) + return ER_DYNCOL_FORMAT; - if (entry_size * column_count + FIXED_HEADER_SIZE > str->length) + if (header.entry_size * header.column_count + FIXED_HEADER_SIZE > + str->length) return ER_DYNCOL_FORMAT; - if (init_dynamic_array(array_of_uint, sizeof(uint), column_count, 0)) + if (my_init_dynamic_array(array_of_uint, sizeof(uint), header.column_count, + 0, MYF(0))) return ER_DYNCOL_RESOURCE; - for (i= 0, read= (uchar *)str->str + FIXED_HEADER_SIZE; - i < column_count; - i++, read+= entry_size) + for (i= 0, read= header.header; + i < header.column_count; + i++, read+= header.entry_size) { uint nm= uint2korr(read); /* Insert can't never fail as it's pre-allocated above */ @@ -1647,73 +2432,247 @@ dynamic_column_list(DYNAMIC_COLUMN *str, DYNAMIC_ARRAY *array_of_uint) return ER_DYNCOL_OK; } +/** + List not-null columns in the packed string (only numeric format) + + @param str The packed string + @param array_of_uint Where to put reference on created array + + @return ER_DYNCOL_* return code +*/ +enum enum_dyncol_func_result +mariadb_dyncol_list_num(DYNAMIC_COLUMN *str, uint *count, uint **nums) +{ + DYN_HEADER header; + uchar *read; + uint i; + enum enum_dyncol_func_result rc; + + (*nums)= 0; (*count)= 0; /* In case of errors */ + + if (str->length == 0) + return ER_DYNCOL_OK; /* no columns */ + + if ((rc= init_read_hdr(&header, str)) < 0) + return rc; + + if (header.format != dyncol_fmt_num) + return ER_DYNCOL_FORMAT; + + if (header.entry_size * header.column_count + FIXED_HEADER_SIZE > + str->length) + return ER_DYNCOL_FORMAT; + + if (!((*nums)= my_malloc(sizeof(uint) * header.column_count, MYF(0)))) + return ER_DYNCOL_RESOURCE; + + for (i= 0, read= header.header; + i < header.column_count; + i++, read+= header.entry_size) + { + (*nums)[i]= uint2korr(read); + } + (*count)= header.column_count; + return ER_DYNCOL_OK; +} + +/** + List not-null columns in the packed string (any format) + + @param str The packed string + @param count Number of names in the list + @param names Where to put names list (should be freed) + + @return ER_DYNCOL_* return code +*/ + +enum enum_dyncol_func_result +mariadb_dyncol_list_named(DYNAMIC_COLUMN *str, uint *count, LEX_STRING **names) +{ + DYN_HEADER header; + uchar *read; + char *pool; + struct st_service_funcs *fmt; + uint i; + enum enum_dyncol_func_result rc; + + (*names)= 0; (*count)= 0; + + if (str->length == 0) + return ER_DYNCOL_OK; /* no columns */ + + if ((rc= init_read_hdr(&header, str)) < 0) + return rc; + + fmt= fmt_data + header.format; + + if (header.entry_size * header.column_count + fmt->fixed_hdr > + str->length) + return ER_DYNCOL_FORMAT; + + if (header.format == dyncol_fmt_num) + *names= my_malloc(sizeof(LEX_STRING) * header.column_count + + DYNCOL_NUM_CHAR * header.column_count, MYF(0)); + else + *names= my_malloc(sizeof(LEX_STRING) * header.column_count + + header.nmpool_size + header.column_count, MYF(0)); + if (!(*names)) + return ER_DYNCOL_RESOURCE; + pool= ((char *)(*names)) + sizeof(LEX_STRING) * header.column_count; + + for (i= 0, read= header.header; + i < header.column_count; + i++, read+= header.entry_size) + { + if (header.format == dyncol_fmt_num) + { + uint nm= uint2korr(read); + (*names)[i].str= pool; + pool+= DYNCOL_NUM_CHAR; + (*names)[i].length= + longlong2str(nm, (*names)[i].str, 10) - (*names)[i].str; + } + else + { + LEX_STRING tmp; + if (read_name(&header, read, &tmp)) + return ER_DYNCOL_FORMAT; + (*names)[i].length= tmp.length; + (*names)[i].str= pool; + pool+= tmp.length + 1; + memcpy((*names)[i].str, (const void *)tmp.str, tmp.length); + (*names)[i].str[tmp.length]= '\0'; // just for safety + } + } + (*count)= header.column_count; + return ER_DYNCOL_OK; +} /** Find the place of the column in the header or place where it should be put - @param num Number of the column - @param header Pointer to the header - @param entry_size Size of a header entry - @param column_count Number of columns in the packed string - @param entry Return pointer to the entry or next entry + @param hdr descriptor of dynamic column record + @param key Name or number of column to fetch + (depends on string_key) + @param string_key True if we gave pointer to LEX_STRING. @retval TRUE found @retval FALSE pointer set to the next row */ static my_bool -find_place(uint num, uchar *header, size_t entry_size, - uint column_count, uchar **entry) +find_place(DYN_HEADER *hdr, void *key, my_bool string_keys) { uint mid, start, end, val; - int flag; - LINT_INIT(flag); /* 100 % safe */ + int UNINIT_VAR(flag); + LEX_STRING str; + char buff[DYNCOL_NUM_CHAR]; + my_bool need_conversion= ((string_keys ? dyncol_fmt_str : dyncol_fmt_num) != + hdr->format); + /* new format can't be numeric if the old one is names */ + DBUG_ASSERT(string_keys || + hdr->format == dyncol_fmt_num); start= 0; - end= column_count -1; + end= hdr->column_count -1; mid= 1; while (start != end) { - uint val; - mid= (start + end) / 2; - val= uint2korr(header + mid * entry_size); - if ((flag= CMP_NUM(num, val)) <= 0) - end= mid; - else - start= mid + 1; + mid= (start + end) / 2; + hdr->entry= hdr->header + mid * hdr->entry_size; + if (!string_keys) + { + val= uint2korr(hdr->entry); + flag= CMP_NUM(*((uint *)key), val); + } + else + { + if (need_conversion) + { + str.str= backwritenum(buff + sizeof(buff), uint2korr(hdr->entry)); + str.length= (buff + sizeof(buff)) - str.str; + } + else + { + DBUG_ASSERT(hdr->format == dyncol_fmt_str); + if (read_name(hdr, hdr->entry, &str)) + return 0; + } + flag= mariadb_dyncol_column_cmp_named((LEX_STRING *)key, &str); + } + if (flag <= 0) + end= mid; + else + start= mid + 1; } + hdr->entry= hdr->header + start * hdr->entry_size; if (start != mid) { - val= uint2korr(header + start * entry_size); - flag= CMP_NUM(num, val); + if (!string_keys) + { + val= uint2korr(hdr->entry); + flag= CMP_NUM(*((uint *)key), val); + } + else + { + if (need_conversion) + { + str.str= backwritenum(buff + sizeof(buff), uint2korr(hdr->entry)); + str.length= (buff + sizeof(buff)) - str.str; + } + else + { + DBUG_ASSERT(hdr->format == dyncol_fmt_str); + if (read_name(hdr, hdr->entry, &str)) + return 0; + } + flag= mariadb_dyncol_column_cmp_named((LEX_STRING *)key, &str); + } } - *entry= header + start * entry_size; if (flag > 0) - *entry+= entry_size; /* Point at next bigger key */ + hdr->entry+= hdr->entry_size; /* Point at next bigger key */ return flag == 0; } /* - Description of plan of adding/removing/updating a packed string + It is internal structure which describes a plan of changing the record + of dynamic columns */ typedef enum {PLAN_REPLACE, PLAN_ADD, PLAN_DELETE, PLAN_NOP} PLAN_ACT; struct st_plan { DYNAMIC_COLUMN_VALUE *val; - uint *num; + void *key; uchar *place; size_t length; - int hdelta, ddelta; + long long hdelta, ddelta, ndelta; + long long mv_offset, mv_length; + uint mv_end; PLAN_ACT act; }; typedef struct st_plan PLAN; -static int plan_sort(const void *a, const void *b) +/** + Sort function for plan by column number +*/ + +static int plan_sort_num(const void *a, const void *b) { - return ((PLAN *)a)->num[0] - ((PLAN *)b)->num[0]; + return *((uint *)((PLAN *)a)->key) - *((uint *)((PLAN *)b)->key); +} + + +/** + Sort function for plan by column name +*/ + +static int plan_sort_named(const void *a, const void *b) +{ + return mariadb_dyncol_column_cmp_named((LEX_STRING *)((PLAN *)a)->key, + (LEX_STRING *)((PLAN *)b)->key); } #define DELTA_CHECK(S, D, C) \ @@ -1723,9 +2682,558 @@ static int plan_sort(const void *a, const void *b) ((S) < 0 && (D) > 0)) \ { \ (C)= TRUE; \ - break; \ - } \ + } +/** + Update dynamic column by copying in a new record (string). + + @param str Dynamic column record to change + @param plan Plan of changing the record + @param add_column_count number of records in the plan array. + @param hdr descriptor of old dynamic column record + @param new_hdr descriptor of new dynamic column record + @param convert need conversion from numeric to names format + + @return ER_DYNCOL_* return code +*/ + +static enum enum_dyncol_func_result +dynamic_column_update_copy(DYNAMIC_COLUMN *str, PLAN *plan, + uint add_column_count, + DYN_HEADER *hdr, DYN_HEADER *new_hdr, + my_bool convert) +{ + DYNAMIC_COLUMN tmp; + struct st_service_funcs *fmt= fmt_data + hdr->format, + *new_fmt= fmt_data + new_hdr->format; + uint i, j, k; + size_t all_headers_size; + + if (dynamic_column_init_named(&tmp, + (new_fmt->fixed_hdr + new_hdr->header_size + + new_hdr->nmpool_size + + new_hdr->data_size + DYNCOL_SYZERESERVE))) + { + return ER_DYNCOL_RESOURCE; + } + bzero(tmp.str, new_fmt->fixed_hdr); + (*new_fmt->set_fixed_hdr)(&tmp, new_hdr); + /* Adjust tmp to contain whole the future header */ + tmp.length= new_fmt->fixed_hdr + new_hdr->header_size + new_hdr->nmpool_size; + + + /* + Copy data to the new string + i= index in array of changes + j= index in packed string header index + */ + new_hdr->entry= new_hdr->header; + new_hdr->name= new_hdr->nmpool; + all_headers_size= new_fmt->fixed_hdr + + new_hdr->header_size + new_hdr->nmpool_size; + for (i= 0, j= 0; i < add_column_count || j < hdr->column_count; i++) + { + size_t UNINIT_VAR(first_offset); + uint start= j, end; + + /* + Search in i and j for the next column to add from i and where to + add. + */ + + while (i < add_column_count && plan[i].act == PLAN_NOP) + i++; /* skip NOP */ + + if (i == add_column_count) + j= end= hdr->column_count; + else + { + /* + old data portion. We don't need to check that j < column_count + as plan[i].place is guaranteed to have a pointer inside the + data. + */ + while (hdr->header + j * hdr->entry_size < plan[i].place) + j++; + end= j; + if ((plan[i].act == PLAN_REPLACE || plan[i].act == PLAN_DELETE)) + j++; /* data at 'j' will be removed */ + } + + /* + Adjust all headers since last loop. + We have to do this as the offset for data has moved + */ + for (k= start; k < end; k++) + { + uchar *read= hdr->header + k * hdr->entry_size; + void *key; + LEX_STRING name; + size_t offs; + uint nm; + DYNAMIC_COLUMN_TYPE tp; + char buff[DYNCOL_NUM_CHAR]; + + if (hdr->format == dyncol_fmt_num) + { + if (convert) + { + name.str= backwritenum(buff + sizeof(buff), uint2korr(read)); + name.length= (buff + sizeof(buff)) - name.str; + key= &name; + } + else + { + nm= uint2korr(read); /* Column nummber */ + key= &nm; + } + } + else + { + if (read_name(hdr, read, &name)) + goto err; + key= &name; + } + if (fmt->type_and_offset_read(&tp, &offs, + read + fmt->fixed_hdr_entry, + hdr->offset_size)) + goto err; + if (k == start) + first_offset= offs; + else if (offs < first_offset) + goto err; + + offs+= plan[i].ddelta; + { + DYNAMIC_COLUMN_VALUE val; + val.type= tp; // only the type used in the header + if ((*new_fmt->put_header_entry)(new_hdr, key, &val, offs)) + goto err; + } + } + + /* copy first the data that was not replaced in original packed data */ + if (start < end) + { + size_t data_size; + /* Add old data last in 'tmp' */ + hdr->entry= hdr->header + start * hdr->entry_size; + data_size= + hdr_interval_length(hdr, hdr->header + end * hdr->entry_size); + if (data_size == DYNCOL_OFFSET_ERROR || + (long) data_size < 0 || + data_size > hdr->data_size - first_offset) + goto err; + + memcpy(tmp.str + tmp.length, (char *)hdr->dtpool + first_offset, + data_size); + tmp.length+= data_size; + } + + /* new data adding */ + if (i < add_column_count) + { + if( plan[i].act == PLAN_ADD || plan[i].act == PLAN_REPLACE) + { + if ((*new_fmt->put_header_entry)(new_hdr, plan[i].key, + plan[i].val, + tmp.length - all_headers_size)) + goto err; + data_store(&tmp, plan[i].val, new_hdr->format); /* Append new data */ + } + } + } + mariadb_dyncol_free(str); + *str= tmp; + return ER_DYNCOL_OK; +err: + mariadb_dyncol_free(&tmp); + return ER_DYNCOL_FORMAT; +} + +static enum enum_dyncol_func_result +dynamic_column_update_move_left(DYNAMIC_COLUMN *str, PLAN *plan, + size_t offset_size, + size_t entry_size, + size_t header_size, + size_t new_offset_size, + size_t new_entry_size, + size_t new_header_size, + uint column_count, + uint new_column_count, + uint add_column_count, + uchar *header_end, + size_t max_offset) +{ + uchar *write; + uchar *header_base= (uchar *)str->str + FIXED_HEADER_SIZE; + uint i, j, k; + size_t curr_offset; + + write= (uchar *)str->str + FIXED_HEADER_SIZE; + set_fixed_header(str, (uint)new_offset_size, new_column_count); + + /* + Move headers first. + i= index in array of changes + j= index in packed string header index + */ + for (curr_offset= 0, i= 0, j= 0; + i < add_column_count || j < column_count; + i++) + { + size_t UNINIT_VAR(first_offset); + uint start= j, end; + + /* + Search in i and j for the next column to add from i and where to + add. + */ + + while (i < add_column_count && plan[i].act == PLAN_NOP) + i++; /* skip NOP */ + + if (i == add_column_count) + j= end= column_count; + else + { + /* + old data portion. We don't need to check that j < column_count + as plan[i].place is guaranteed to have a pointer inside the + data. + */ + while (header_base + j * entry_size < plan[i].place) + j++; + end= j; + if ((plan[i].act == PLAN_REPLACE || plan[i].act == PLAN_DELETE)) + j++; /* data at 'j' will be removed */ + } + plan[i].mv_end= end; + + { + DYNAMIC_COLUMN_TYPE tp; + if (type_and_offset_read_num(&tp, &first_offset, + header_base + start * entry_size + + COLUMN_NUMBER_SIZE, offset_size)) + return ER_DYNCOL_FORMAT; + } + /* find data to be moved */ + if (start < end) + { + size_t data_size= + get_length_interval(header_base + start * entry_size, + header_base + end * entry_size, + header_end, offset_size, max_offset); + if (data_size == DYNCOL_OFFSET_ERROR || + (long) data_size < 0 || + data_size > max_offset - first_offset) + { + str->length= 0; // just something valid + return ER_DYNCOL_FORMAT; + } + DBUG_ASSERT(curr_offset == first_offset + plan[i].ddelta); + plan[i].mv_offset= first_offset; + plan[i].mv_length= data_size; + curr_offset+= data_size; + } + else + { + plan[i].mv_length= 0; + plan[i].mv_offset= curr_offset; + } + + if (plan[i].ddelta == 0 && offset_size == new_offset_size && + plan[i].act != PLAN_DELETE) + write+= entry_size * (end - start); + else + { + /* + Adjust all headers since last loop. + We have to do this as the offset for data has moved + */ + for (k= start; k < end; k++) + { + uchar *read= header_base + k * entry_size; + size_t offs; + uint nm; + DYNAMIC_COLUMN_TYPE tp; + + nm= uint2korr(read); /* Column nummber */ + if (type_and_offset_read_num(&tp, &offs, read + COLUMN_NUMBER_SIZE, + offset_size)) + return ER_DYNCOL_FORMAT; + + if (k > start && offs < first_offset) + { + str->length= 0; // just something valid + return ER_DYNCOL_FORMAT; + } + + offs+= plan[i].ddelta; + int2store(write, nm); + /* write rest of data at write + COLUMN_NUMBER_SIZE */ + type_and_offset_store_num(write, new_offset_size, tp, offs); + write+= new_entry_size; + } + } + + /* new data adding */ + if (i < add_column_count) + { + if( plan[i].act == PLAN_ADD || plan[i].act == PLAN_REPLACE) + { + int2store(write, *((uint *)plan[i].key)); + type_and_offset_store_num(write, new_offset_size, + plan[i].val[0].type, + curr_offset); + write+= new_entry_size; + curr_offset+= plan[i].length; + } + } + } + + /* + Move data. + i= index in array of changes + j= index in packed string header index + */ + str->length= (FIXED_HEADER_SIZE + new_header_size); + for (i= 0, j= 0; + i < add_column_count || j < column_count; + i++) + { + uint start= j, end; + + /* + Search in i and j for the next column to add from i and where to + add. + */ + + while (i < add_column_count && plan[i].act == PLAN_NOP) + i++; /* skip NOP */ + + j= end= plan[i].mv_end; + if (i != add_column_count && + (plan[i].act == PLAN_REPLACE || plan[i].act == PLAN_DELETE)) + j++; + + /* copy first the data that was not replaced in original packed data */ + if (start < end && plan[i].mv_length) + { + memmove((header_base + new_header_size + + plan[i].mv_offset + plan[i].ddelta), + header_base + header_size + plan[i].mv_offset, + plan[i].mv_length); + } + str->length+= plan[i].mv_length; + + /* new data adding */ + if (i < add_column_count) + { + if( plan[i].act == PLAN_ADD || plan[i].act == PLAN_REPLACE) + { + data_store(str, plan[i].val, dyncol_fmt_num);/* Append new data */ + } + } + } + return ER_DYNCOL_OK; +} + +#ifdef UNUSED +static enum enum_dyncol_func_result +dynamic_column_update_move_right(DYNAMIC_COLUMN *str, PLAN *plan, + size_t offset_size, + size_t entry_size, + size_t header_size, + size_t new_offset_size, + size_t new_entry_size, + size_t new_header_size, + uint column_count, + uint new_column_count, + uint add_column_count, + uchar *header_end, + size_t max_offset) +{ + uchar *write; + uchar *header_base= (uchar *)str->str + FIXED_HEADER_SIZE; + uint i, j, k; + size_t curr_offset; + + write= (uchar *)str->str + FIXED_HEADER_SIZE; + set_fixed_header(str, new_offset_size, new_column_count); + + /* + Move data first. + i= index in array of changes + j= index in packed string header index + */ + for (curr_offset= 0, i= 0, j= 0; + i < add_column_count || j < column_count; + i++) + { + size_t UNINIT_VAR(first_offset); + uint start= j, end; + + /* + Search in i and j for the next column to add from i and where to + add. + */ + + while (i < add_column_count && plan[i].act == PLAN_NOP) + i++; /* skip NOP */ + + if (i == add_column_count) + j= end= column_count; + else + { + /* + old data portion. We don't need to check that j < column_count + as plan[i].place is guaranteed to have a pointer inside the + data. + */ + while (header_base + j * entry_size < plan[i].place) + j++; + end= j; + if ((plan[i].act == PLAN_REPLACE || plan[i].act == PLAN_DELETE)) + j++; /* data at 'j' will be removed */ + } + plan[i].mv_end= end; + + { + DYNAMIC_COLUMN_TYPE tp; + type_and_offset_read_num(&tp, &first_offset, + header_base + + start * entry_size + COLUMN_NUMBER_SIZE, + offset_size); + } + /* find data to be moved */ + if (start < end) + { + size_t data_size= + get_length_interval(header_base + start * entry_size, + header_base + end * entry_size, + header_end, offset_size, max_offset); + if (data_size == DYNCOL_OFFSET_ERROR || + (long) data_size < 0 || + data_size > max_offset - first_offset) + { + str->length= 0; // just something valid + return ER_DYNCOL_FORMAT; + } + DBUG_ASSERT(curr_offset == first_offset + plan[i].ddelta); + plan[i].mv_offset= first_offset; + plan[i].mv_length= data_size; + curr_offset+= data_size; + } + else + { + plan[i].mv_length= 0; + plan[i].mv_offset= curr_offset; + } + + if (plan[i].ddelta == 0 && offset_size == new_offset_size && + plan[i].act != PLAN_DELETE) + write+= entry_size * (end - start); + else + { + /* + Adjust all headers since last loop. + We have to do this as the offset for data has moved + */ + for (k= start; k < end; k++) + { + uchar *read= header_base + k * entry_size; + size_t offs; + uint nm; + DYNAMIC_COLUMN_TYPE tp; + + nm= uint2korr(read); /* Column nummber */ + type_and_offset_read_num(&tp, &offs, read + COLUMN_NUMBER_SIZE, + offset_size); + if (k > start && offs < first_offset) + { + str->length= 0; // just something valid + return ER_DYNCOL_FORMAT; + } + + offs+= plan[i].ddelta; + int2store(write, nm); + /* write rest of data at write + COLUMN_NUMBER_SIZE */ + if (type_and_offset_store_num(write, new_offset_size, tp, offs)) + { + str->length= 0; // just something valid + return ER_DYNCOL_FORMAT; + } + write+= new_entry_size; + } + } + + /* new data adding */ + if (i < add_column_count) + { + if( plan[i].act == PLAN_ADD || plan[i].act == PLAN_REPLACE) + { + int2store(write, *((uint *)plan[i].key)); + if (type_and_offset_store_num(write, new_offset_size, + plan[i].val[0].type, + curr_offset)) + { + str->length= 0; // just something valid + return ER_DYNCOL_FORMAT; + } + write+= new_entry_size; + curr_offset+= plan[i].length; + } + } + } + + /* + Move headers. + i= index in array of changes + j= index in packed string header index + */ + str->length= (FIXED_HEADER_SIZE + new_header_size); + for (i= 0, j= 0; + i < add_column_count || j < column_count; + i++) + { + uint start= j, end; + + /* + Search in i and j for the next column to add from i and where to + add. + */ + + while (i < add_column_count && plan[i].act == PLAN_NOP) + i++; /* skip NOP */ + + j= end= plan[i].mv_end; + if (i != add_column_count && + (plan[i].act == PLAN_REPLACE || plan[i].act == PLAN_DELETE)) + j++; + + /* copy first the data that was not replaced in original packed data */ + if (start < end && plan[i].mv_length) + { + memmove((header_base + new_header_size + + plan[i].mv_offset + plan[i].ddelta), + header_base + header_size + plan[i].mv_offset, + plan[i].mv_length); + } + str->length+= plan[i].mv_length; + + /* new data adding */ + if (i < add_column_count) + { + if( plan[i].act == PLAN_ADD || plan[i].act == PLAN_REPLACE) + { + data_store(str, plan[i].val, dyncol_fmt_num); /* Append new data */ + } + } + } + return ER_DYNCOL_OK; +} +#endif /** Update the packed string with the given columns @@ -1737,6 +3245,8 @@ static int plan_sort(const void *a, const void *b) @return ER_DYNCOL_* return code */ +/* plan allocated on the stack */ +#define IN_PLACE_PLAN 4 enum enum_dyncol_func_result dynamic_column_update_many(DYNAMIC_COLUMN *str, @@ -1744,39 +3254,95 @@ dynamic_column_update_many(DYNAMIC_COLUMN *str, uint *column_numbers, DYNAMIC_COLUMN_VALUE *values) { - PLAN *plan; - uchar *header_end; - long data_delta= 0; - uint i, j, k; - uint new_column_count, column_count, not_null; + return dynamic_column_update_many_fmt(str, add_column_count, column_numbers, + values, FALSE); +} + +enum enum_dyncol_func_result +mariadb_dyncol_update_many_num(DYNAMIC_COLUMN *str, + uint add_column_count, + uint *column_numbers, + DYNAMIC_COLUMN_VALUE *values) +{ + return dynamic_column_update_many_fmt(str, add_column_count, column_numbers, + values, FALSE); +} + +enum enum_dyncol_func_result +mariadb_dyncol_update_many_named(DYNAMIC_COLUMN *str, + uint add_column_count, + LEX_STRING *column_names, + DYNAMIC_COLUMN_VALUE *values) +{ + return dynamic_column_update_many_fmt(str, add_column_count, column_names, + values, TRUE); +} + +static uint numlen(uint val) +{ + uint res; + if (val == 0) + return 1; + res= 0; + while(val) + { + res++; + val/=10; + } + return res; +} + +static enum enum_dyncol_func_result +dynamic_column_update_many_fmt(DYNAMIC_COLUMN *str, + uint add_column_count, + void *column_keys, + DYNAMIC_COLUMN_VALUE *values, + my_bool string_keys) +{ + PLAN *plan, *alloc_plan= NULL, in_place_plan[IN_PLACE_PLAN]; + uchar *element; + DYN_HEADER header, new_header; + struct st_service_funcs *fmt, *new_fmt; + long long data_delta= 0, name_delta= 0; + uint i; + uint not_null; + long long header_delta= 0; + long long header_delta_sign, data_delta_sign; + int copy= FALSE; enum enum_dyncol_func_result rc; - int header_delta; - size_t offset_size, entry_size, header_size, data_size; - size_t new_offset_size, new_entry_size, new_header_size, new_data_size; - size_t max_offset; + my_bool convert; if (add_column_count == 0) return ER_DYNCOL_OK; + bzero(&header, sizeof(header)); + bzero(&new_header, sizeof(new_header)); + new_header.format= (string_keys ? dyncol_fmt_str : dyncol_fmt_num); + new_fmt= fmt_data + new_header.format; + /* Get columns in column order. As the data in 'str' is already in column order this allows to replace all columns in one loop. */ - - if (!(plan= my_malloc(sizeof(PLAN) * (add_column_count + 1), MYF(0)))) + if (IN_PLACE_PLAN > add_column_count) + plan= in_place_plan; + else if (!(alloc_plan= plan= + my_malloc(sizeof(PLAN) * (add_column_count + 1), MYF(0)))) return ER_DYNCOL_RESOURCE; not_null= add_column_count; - for (i= 0; i < add_column_count; i++) + for (i= 0, element= (uchar *) column_keys; + i < add_column_count; + i++, element+= new_fmt->key_size_in_array) { - if (column_numbers[i] > UINT_MAX16) + if ((*new_fmt->check_limit)(&element)) { rc= ER_DYNCOL_DATA; goto end; } plan[i].val= values + i; - plan[i].num= column_numbers + i; + plan[i].key= element; if (values[i].type == DYN_COL_NULL) not_null--; @@ -1792,22 +3358,32 @@ dynamic_column_update_many(DYNAMIC_COLUMN *str, } /* Check that header is ok */ - if (read_fixed_header(str, &offset_size, &column_count)) - { - rc= ER_DYNCOL_FORMAT; + if ((rc= init_read_hdr(&header, str)) < 0) goto end; - } - if (column_count == 0) + fmt= fmt_data + header.format; + /* new format can't be numeric if the old one is names */ + DBUG_ASSERT(new_header.format == dyncol_fmt_str || + header.format == dyncol_fmt_num); + if (header.column_count == 0) goto create_new_string; - qsort(plan, (size_t)add_column_count, sizeof(PLAN), &plan_sort); + qsort(plan, (size_t)add_column_count, sizeof(PLAN), new_fmt->plan_sort); - new_column_count= column_count; - calc_param(&entry_size, &header_size, offset_size, column_count); - max_offset= str->length - (FIXED_HEADER_SIZE + header_size); - header_end= (uchar*) str->str + FIXED_HEADER_SIZE + header_size; + new_header.column_count= header.column_count; + new_header.nmpool_size= header.nmpool_size; + if ((convert= (new_header.format == dyncol_fmt_str && + header.format == dyncol_fmt_num))) + { + DBUG_ASSERT(new_header.nmpool_size == 0); + for(i= 0, header.entry= header.header; + i < header.column_count; + i++, header.entry+= header.entry_size) + { + new_header.nmpool_size+= numlen(uint2korr(header.entry)); + } + } - if (header_size + FIXED_HEADER_SIZE > str->length) + if (fmt->fixed_hdr + header.header_size + header.nmpool_size > str->length) { rc= ER_DYNCOL_FORMAT; goto end; @@ -1817,17 +3393,15 @@ dynamic_column_update_many(DYNAMIC_COLUMN *str, Calculate how many columns and data is added/deleted and make a 'plan' for each of them. */ - header_delta= 0; for (i= 0; i < add_column_count; i++) { - uchar *entry; - /* For now we don't allow creating two columns with the same number at the time of create. This can be fixed later to just use the later by comparing the pointers. */ - if (i < add_column_count - 1 && plan[i].num[0] == plan[i + 1].num[0]) + if (i < add_column_count - 1 && + new_fmt->column_sort(&plan[i].key, &plan[i + 1].key) == 0) { rc= ER_DYNCOL_DATA; goto end; @@ -1835,27 +3409,42 @@ dynamic_column_update_many(DYNAMIC_COLUMN *str, /* Set common variables for all plans */ plan[i].ddelta= data_delta; + plan[i].ndelta= name_delta; /* get header delta in entries */ plan[i].hdelta= header_delta; plan[i].length= 0; /* Length if NULL */ - if (find_place(plan[i].num[0], - (uchar *)str->str + FIXED_HEADER_SIZE, - entry_size, column_count, &entry)) + if (find_place(&header, plan[i].key, string_keys)) { - size_t entry_data_size; - my_bool error; + size_t entry_data_size, entry_name_size= 0; /* Data existed; We have to replace or delete it */ - entry_data_size= get_length(entry, header_end, - offset_size, max_offset, &error); - if (error || (long) entry_data_size < 0) + entry_data_size= hdr_interval_length(&header, header.entry + + header.entry_size); + if (entry_data_size == DYNCOL_OFFSET_ERROR || + (long) entry_data_size < 0) { rc= ER_DYNCOL_FORMAT; goto end; } + if (new_header.format == dyncol_fmt_str) + { + if (header.format == dyncol_fmt_str) + { + LEX_STRING name; + if (read_name(&header, header.entry, &name)) + { + rc= ER_DYNCOL_FORMAT; + goto end; + } + entry_name_size= name.length; + } + else + entry_name_size= numlen(uint2korr(header.entry)); + } + if (plan[i].val->type == DYN_COL_NULL) { /* Inserting a NULL means delete the old data */ @@ -1863,6 +3452,7 @@ dynamic_column_update_many(DYNAMIC_COLUMN *str, plan[i].act= PLAN_DELETE; /* Remove old value */ header_delta--; /* One row less in header */ data_delta-= entry_data_size; /* Less data to store */ + name_delta-= entry_name_size; } else { @@ -1870,13 +3460,18 @@ dynamic_column_update_many(DYNAMIC_COLUMN *str, plan[i].act= PLAN_REPLACE; /* get data delta in bytes */ - if ((plan[i].length= dynamic_column_value_len(plan[i].val)) == + if ((plan[i].length= dynamic_column_value_len(plan[i].val, + new_header.format)) == (size_t) ~0) { rc= ER_DYNCOL_DATA; goto end; } data_delta+= plan[i].length - entry_data_size; + if (new_header.format == dyncol_fmt_str) + { + name_delta+= ((LEX_STRING *)(plan[i].key))->length - entry_name_size; + } } } else @@ -1885,7 +3480,7 @@ dynamic_column_update_many(DYNAMIC_COLUMN *str, if (plan[i].val->type == DYN_COL_NULL) { - plan[i].act= PLAN_NOP; /* Mark entry to be skiped */ + plan[i].act= PLAN_NOP; /* Mark entry to be skipped */ } else { @@ -1894,228 +3489,931 @@ dynamic_column_update_many(DYNAMIC_COLUMN *str, plan[i].act= PLAN_ADD; header_delta++; /* One more row in header */ /* get data delta in bytes */ - if ((plan[i].length= dynamic_column_value_len(plan[i].val)) == + if ((plan[i].length= dynamic_column_value_len(plan[i].val, + new_header.format)) == (size_t) ~0) { rc= ER_DYNCOL_DATA; goto end; } data_delta+= plan[i].length; + if (new_header.format == dyncol_fmt_str) + name_delta+= ((LEX_STRING *)plan[i].key)->length; } } - plan[i].place= entry; + plan[i].place= header.entry; } plan[add_column_count].hdelta= header_delta; plan[add_column_count].ddelta= data_delta; - new_column_count= column_count + header_delta; + plan[add_column_count].act= PLAN_NOP; + plan[add_column_count].place= header.dtpool; + + new_header.column_count= (uint)(header.column_count + header_delta); /* Check if it is only "increasing" or only "decreasing" plan for (header and data separately). */ - data_size= str->length - header_size - FIXED_HEADER_SIZE; - new_data_size= data_size + data_delta; - if ((new_offset_size= dynamic_column_offset_bytes(new_data_size)) >= - MAX_OFFSET_LENGTH) + new_header.data_size= header.data_size + data_delta; + new_header.nmpool_size= new_header.nmpool_size + name_delta; + DBUG_ASSERT(new_header.format != dyncol_fmt_num || + new_header.nmpool_size == 0); + if ((new_header.offset_size= + new_fmt->dynamic_column_offset_bytes(new_header.data_size)) >= + new_fmt->max_offset_size) { rc= ER_DYNCOL_LIMIT; goto end; } -#ifdef NOT_IMPLEMENTED - /* if (new_offset_size != offset_size) then we have to rewrite header */ - header_delta_sign= new_offset_size - offset_size; + copy= ((header.format != new_header.format) || + (new_header.format == dyncol_fmt_str)); + /* if (new_header.offset_size!=offset_size) then we have to rewrite header */ + header_delta_sign= + ((int)new_header.offset_size + new_fmt->fixed_hdr_entry) - + ((int)header.offset_size + fmt->fixed_hdr_entry); data_delta_sign= 0; - for (i= 0; i < add_column_count; i++) + // plan[add_column_count] contains last deltas. + for (i= 0; i <= add_column_count && !copy; i++) { /* This is the check for increasing/decreasing */ DELTA_CHECK(header_delta_sign, plan[i].hdelta, copy); DELTA_CHECK(data_delta_sign, plan[i].ddelta, copy); } -#endif - calc_param(&new_entry_size, &new_header_size, - new_offset_size, new_column_count); + calc_param(&new_header.entry_size, &new_header.header_size, + new_fmt->fixed_hdr_entry, + new_header.offset_size, new_header.column_count); /* - The following code always make a copy. In future we can do a more - optimized version when data is only increasing / decreasing. + Need copy because: + 1, Header/data parts moved in different directions. + 2. There is no enough allocated space in the string. + 3. Header and data moved in different directions. */ + if (copy || /*1.*/ + str->max_length < str->length + header_delta + data_delta || /*2.*/ + ((header_delta_sign < 0 && data_delta_sign > 0) || + (header_delta_sign > 0 && data_delta_sign < 0))) /*3.*/ + rc= dynamic_column_update_copy(str, plan, add_column_count, + &header, &new_header, + convert); + else + if (header_delta_sign < 0) + rc= dynamic_column_update_move_left(str, plan, header.offset_size, + header.entry_size, + header.header_size, + new_header.offset_size, + new_header.entry_size, + new_header.header_size, + header.column_count, + new_header.column_count, + add_column_count, header.dtpool, + header.data_size); + else + /* + rc= dynamic_column_update_move_right(str, plan, offset_size, + entry_size, header_size, + new_header.offset_size, + new_header.entry_size, + new_heder.header_size, column_count, + new_header.column_count, + add_column_count, header_end, + header.data_size); + */ + rc= dynamic_column_update_copy(str, plan, add_column_count, + &header, &new_header, + convert); +end: + my_free(alloc_plan); + return rc; + +create_new_string: + /* There is no columns from before, so let's just add the new ones */ + rc= ER_DYNCOL_OK; + if (not_null != 0) + rc= dynamic_column_create_many_internal_fmt(str, add_column_count, + (uint*)column_keys, values, + str->str == NULL, + string_keys); + goto end; +} + + +/** + Update the packed string with the given column + + @param str String where to write the data + @param column_number Array of columns number + @param values Array of columns values + + @return ER_DYNCOL_* return code +*/ + - /*if (copy) */ +enum enum_dyncol_func_result +dynamic_column_update(DYNAMIC_COLUMN *str, uint column_nr, + DYNAMIC_COLUMN_VALUE *value) +{ + return dynamic_column_update_many(str, 1, &column_nr, value); +} + + +enum enum_dyncol_func_result +mariadb_dyncol_check(DYNAMIC_COLUMN *str) +{ + struct st_service_funcs *fmt; + enum enum_dyncol_func_result rc= ER_DYNCOL_FORMAT; + DYN_HEADER header; + uint i; + size_t data_offset= 0, name_offset= 0; + size_t prev_data_offset= 0, prev_name_offset= 0; + LEX_STRING name= {0,0}, prev_name= {0,0}; + uint num= 0, prev_num= 0; + void *key, *prev_key; + enum enum_dynamic_column_type type= DYN_COL_NULL, prev_type= DYN_COL_NULL; + + DBUG_ENTER("dynamic_column_check"); + + if (str->length == 0) { - DYNAMIC_COLUMN tmp; - uchar *header_base= (uchar *)str->str + FIXED_HEADER_SIZE, - *write; - if (dynamic_column_init_str(&tmp, - (FIXED_HEADER_SIZE + new_header_size + - new_data_size + DYNCOL_SYZERESERVE))) - { - rc= ER_DYNCOL_RESOURCE; - goto end; - } - write= (uchar *)tmp.str + FIXED_HEADER_SIZE; - /* Adjust tmp to contain whole the future header */ - tmp.length= FIXED_HEADER_SIZE + new_header_size; - set_fixed_header(&tmp, new_offset_size, new_column_count); - data_delta= 0; + DBUG_PRINT("info", ("empty string is OK")); + DBUG_RETURN(ER_DYNCOL_OK); + } - /* - Copy data to the new string - i= index in array of changes - j= index in packed string header index - */ + bzero(&header, sizeof(header)); - for (i= 0, j= 0; i < add_column_count || j < column_count; i++) - { - size_t first_offset; - uint start= j, end; - LINT_INIT(first_offset); + /* Check that header is OK */ + if (read_fixed_header(&header, str)) + { + DBUG_PRINT("info", ("Reading fixed string header failed")); + goto end; + } + fmt= fmt_data + header.format; + calc_param(&header.entry_size, &header.header_size, + fmt->fixed_hdr_entry, header.offset_size, + header.column_count); + /* headers are out of string length (no space for data and part of headers) */ + if (fmt->fixed_hdr + header.header_size + header.nmpool_size > str->length) + { + DBUG_PRINT("info", ("Fixed header: %u Header size: %u " + "Name pool size: %u but Strig length: %u", + (uint)fmt->fixed_hdr, + (uint)header.header_size, + (uint)header.nmpool_size, + (uint)str->length)); + goto end; + } + header.header= (uchar*)str->str + fmt->fixed_hdr; + header.nmpool= header.header + header.header_size; + header.dtpool= header.nmpool + header.nmpool_size; + header.data_size= str->length - fmt->fixed_hdr - + header.header_size - header.nmpool_size; + + /* read and check headers */ + if (header.format == dyncol_fmt_num) + { + key= # + prev_key= &prev_num; + } + else + { + key= &name; + prev_key= &prev_name; + } + for (i= 0, header.entry= header.header; + i < header.column_count; + i++, header.entry+= header.entry_size) + { - /* - Search in i and j for the next column to add from i and where to - add. - */ + if (header.format == dyncol_fmt_num) + { + num= uint2korr(header.entry); + } + else + { + DBUG_ASSERT(header.format == dyncol_fmt_str); + if (read_name(&header, header.entry, &name)) + { + DBUG_PRINT("info", ("Reading name failed: Field order: %u" + " Name offset: %u" + " Name pool size: %u", + (uint) i, + uint2korr(header.entry), + (uint)header.nmpool_size)); + goto end; + } + name_offset= name.str - (char *)header.nmpool; + } + if ((*fmt->type_and_offset_read)(&type, &data_offset, + header.entry + fmt->fixed_hdr_entry, + header.offset_size)) + goto end; - while (i < add_column_count && plan[i].act == PLAN_NOP) - i++; /* skip NOP */ - if (i == add_column_count) - j= end= column_count; - else + DBUG_ASSERT(type != DYN_COL_NULL); + if (data_offset > header.data_size) + { + DBUG_PRINT("info", ("Field order: %u Data offset: %u" + " > Data pool size: %u", + (uint)i, + (uint)data_offset, + (uint)header.data_size)); + goto end; + } + if (prev_type != DYN_COL_NULL) + { + /* It is not first entry */ + if (prev_data_offset > data_offset || + ((prev_type != DYN_COL_INT && + prev_type != DYN_COL_UINT && + prev_type != DYN_COL_DECIMAL) && prev_data_offset == data_offset)) + { + DBUG_PRINT("info", ("Field order: %u Previous data offset: %u" + " >(=) Current data offset: %u", + (uint)i, + (uint)prev_data_offset, + (uint)data_offset)); + goto end; + } + if (prev_name_offset > name_offset) { - /* - old data portion. We don't need to check that j < column_count - as plan[i].place is guaranteed to have a pointer inside the - data. - */ - while (header_base + j * entry_size < plan[i].place) - j++; - end= j; - if ((plan[i].act == PLAN_REPLACE || plan[i].act == PLAN_DELETE)) - j++; /* data at 'j' will be removed */ + DBUG_PRINT("info", ("Field order: %u Previous name offset: %u" + " > Current name offset: %u", + (uint)i, + (uint)prev_data_offset, + (uint)data_offset)); + goto end; } + if ((*fmt->column_sort)(&prev_key, &key) >= 0) + { + DBUG_PRINT("info", ("Field order: %u Previous key >= Current key", + (uint)i)); + goto end; + } + } + prev_num= num; + prev_name= name; + prev_data_offset= data_offset; + prev_name_offset= name_offset; + prev_type= type; + } - if (plan[i].ddelta == 0 && offset_size == new_offset_size) + /* check data, which we can */ + for (i= 0, header.entry= header.header; + i < header.column_count; + i++, header.entry+= header.entry_size) + { + DYNAMIC_COLUMN_VALUE store; + // already checked by previouse pass + (*fmt->type_and_offset_read)(&header.type, &header.offset, + header.entry + fmt->fixed_hdr_entry, + header.offset_size); + header.length= + hdr_interval_length(&header, header.entry + header.entry_size); + header.data= header.dtpool + header.offset; + switch ((header.type)) { + case DYN_COL_INT: + rc= dynamic_column_sint_read(&store, header.data, header.length); + break; + case DYN_COL_UINT: + rc= dynamic_column_uint_read(&store, header.data, header.length); + break; + case DYN_COL_DOUBLE: + rc= dynamic_column_double_read(&store, header.data, header.length); + break; + case DYN_COL_STRING: + rc= dynamic_column_string_read(&store, header.data, header.length); + break; + case DYN_COL_DECIMAL: + rc= dynamic_column_decimal_read(&store, header.data, header.length); + break; + case DYN_COL_DATETIME: + rc= dynamic_column_date_time_read(&store, header.data, + header.length); + break; + case DYN_COL_DATE: + rc= dynamic_column_date_read(&store, header.data, header.length); + break; + case DYN_COL_TIME: + rc= dynamic_column_time_read(&store, header.data, header.length); + break; + case DYN_COL_DYNCOL: + rc= dynamic_column_dyncol_read(&store, header.data, header.length); + break; + case DYN_COL_NULL: + default: + rc= ER_DYNCOL_FORMAT; + goto end; + } + if (rc != ER_DYNCOL_OK) + { + DBUG_ASSERT(rc < 0); + DBUG_PRINT("info", ("Field order: %u Can't read data: %i", + (uint)i, (int) rc)); + goto end; + } + } + + rc= ER_DYNCOL_OK; +end: + DBUG_RETURN(rc); +} + +static +my_bool dynstr_append_json_quoted(DYNAMIC_STRING *str, + const char *append, size_t len) +{ + uint additional= ((str->alloc_increment && str->alloc_increment > 6) ? + str->alloc_increment : + 10); + uint lim= additional; + uint i; + if (dynstr_realloc(str, len + additional + 2)) + return TRUE; + str->str[str->length++]= '"'; + for (i= 0; i < len; i++) + { + register char c= append[i]; + if (unlikely(((uchar)c) <= 0x1F)) + { + if (lim < 5) + { + if (dynstr_realloc(str, additional)) + return TRUE; + lim+= additional; + } + lim-= 5; + str->str[str->length++]= '\\'; + str->str[str->length++]= 'u'; + str->str[str->length++]= '0'; + str->str[str->length++]= '0'; + str->str[str->length++]= (c < 0x10 ? '0' : '1'); + c%= 0x10; + str->str[str->length++]= (c < 0xA ? '0' + c : 'A' + (c - 0xA)); + } + else + { + if (c == '"' || c == '\\') { - uchar *read= header_base + start * entry_size; - DYNAMIC_COLUMN_TYPE tp; - /* - It's safe to copy the header unchanged. This is usually the - case for the first header block before any changed data. - */ - if (start < end) /* Avoid memcpy with 0 */ + if (!lim) { - size_t length= entry_size * (end - start); - memcpy(write, read, length); - write+= length; + if (dynstr_realloc(str, additional)) + return TRUE; + lim= additional; } - /* Read first_offset */ - type_and_offset_read(&tp, &first_offset, read, offset_size); + lim--; + str->str[str->length++]= '\\'; } - else + str->str[str->length++]= c; + } + } + str->str[str->length++]= '"'; + return FALSE; +} + + +enum enum_dyncol_func_result +mariadb_dyncol_val_str(DYNAMIC_STRING *str, DYNAMIC_COLUMN_VALUE *val, + CHARSET_INFO *cs, char quote) +{ + char buff[40]; + size_t len; + switch (val->type) { + case DYN_COL_INT: + len= snprintf(buff, sizeof(buff), "%lld", val->x.long_value); + if (dynstr_append_mem(str, buff, len)) + return ER_DYNCOL_RESOURCE; + break; + case DYN_COL_UINT: + len= snprintf(buff, sizeof(buff), "%llu", val->x.ulong_value); + if (dynstr_append_mem(str, buff, len)) + return ER_DYNCOL_RESOURCE; + break; + case DYN_COL_DOUBLE: + + len= my_gcvt(val->x.double_value, MY_GCVT_ARG_DOUBLE, + sizeof(buff) - 1, buff, NULL); + if (dynstr_realloc(str, len + (quote ? 2 : 0))) + return ER_DYNCOL_RESOURCE; + dynstr_append_mem(str, buff, len); + break; + case DYN_COL_DYNCOL: + case DYN_COL_STRING: { - /* - Adjust all headers since last loop. - We have to do this as the offset for data has moved - */ - for (k= start; k < end; k++) + char *alloc= NULL; + char *from= val->x.string.value.str; + ulong bufflen; + my_bool conv= !my_charset_same(val->x.string.charset, cs); + my_bool rc; + len= val->x.string.value.length; + bufflen= (ulong)(len * (conv ? cs->mbmaxlen : 1)); + if (dynstr_realloc(str, bufflen)) + return ER_DYNCOL_RESOURCE; + + // guaranty UTF-8 string for value + if (!my_charset_same(val->x.string.charset, cs)) { - uchar *read= header_base + k * entry_size; - size_t offs; - uint nm; - DYNAMIC_COLUMN_TYPE tp; - - nm= uint2korr(read); /* Column nummber */ - type_and_offset_read(&tp, &offs, read, offset_size); - if (k == start) - first_offset= offs; - else if (offs < first_offset) + uint dummy_errors; + if (!quote) { - dynamic_column_column_free(&tmp); - rc= ER_DYNCOL_FORMAT; - goto end; + /* convert to the destination */ + str->length+= my_convert(str->str, bufflen, + cs, + from, (uint32)len, + val->x.string.charset, + &dummy_errors); + return ER_DYNCOL_OK; } - - offs+= plan[i].ddelta; - int2store(write, nm); - /* write rest of data at write + COLUMN_NUMBER_SIZE */ - type_and_offset_store(write, new_offset_size, tp, offs); - write+= new_entry_size; + if ((alloc= (char *)my_malloc(bufflen, MYF(0)))) + { + len= my_convert(alloc, bufflen, cs, + from, (uint32)len, + val->x.string.charset, + &dummy_errors); + from= alloc; + } + else + return ER_DYNCOL_RESOURCE; } + if (quote) + if (quote == DYNCOL_JSON_ESC) + rc= dynstr_append_json_quoted(str, from, len); + else + rc= dynstr_append_quoted(str, from, len, quote); + else + rc= dynstr_append_mem(str, from, len); + if (alloc) + my_free(alloc); + if (rc) + return ER_DYNCOL_RESOURCE; + break; } + case DYN_COL_DECIMAL: + { + int tmp_len= sizeof(buff); + decimal2string(&val->x.decimal.value, buff, &tmp_len, + 0, val->x.decimal.value.frac, + '0'); + if (dynstr_append_mem(str, buff, tmp_len)) + return ER_DYNCOL_RESOURCE; + break; + } + case DYN_COL_DATETIME: + case DYN_COL_DATE: + case DYN_COL_TIME: + len= my_TIME_to_str(&val->x.time_value, buff, AUTO_SEC_PART_DIGITS); + if (dynstr_realloc(str, len + (quote ? 2 : 0))) + return ER_DYNCOL_RESOURCE; + if (quote) + str->str[str->length++]= '"'; + dynstr_append_mem(str, buff, len); + if (quote) + str->str[str->length++]= '"'; + break; + case DYN_COL_NULL: + if (dynstr_append_mem(str, "null", 4)) + return ER_DYNCOL_RESOURCE; + break; + default: + return(ER_DYNCOL_FORMAT); + } + return(ER_DYNCOL_OK); +} + - /* copy first the data that was not replaced in original packed data */ - if (start < end) +enum enum_dyncol_func_result +mariadb_dyncol_val_long(longlong *ll, DYNAMIC_COLUMN_VALUE *val) +{ + enum enum_dyncol_func_result rc= ER_DYNCOL_OK; + *ll= 0; + switch (val->type) { + case DYN_COL_INT: + *ll= val->x.long_value; + break; + case DYN_COL_UINT: + *ll= (longlong)val->x.ulong_value; + if (val->x.ulong_value > ULONGLONG_MAX) + rc= ER_DYNCOL_TRUNCATED; + break; + case DYN_COL_DOUBLE: + *ll= (longlong)val->x.double_value; + if (((double) *ll) != val->x.double_value) + rc= ER_DYNCOL_TRUNCATED; + break; + case DYN_COL_STRING: { - my_bool error; - /* Add old data last in 'tmp' */ - size_t data_size= - get_length_interval(header_base + start * entry_size, - header_base + end * entry_size, - header_end, offset_size, max_offset, &error); - if (error || (long) data_size < 0 || - data_size > max_offset - first_offset) + char *src= val->x.string.value.str; + size_t len= val->x.string.value.length; + longlong i= 0, sign= 1; + + while (len && my_isspace(&my_charset_latin1, *src)) src++,len--; + + if (len) { - dynamic_column_column_free(&tmp); - rc= ER_DYNCOL_FORMAT; - goto end; + if (*src == '-') + { + sign= -1; + src++; + } else if (*src == '+') + src++; + while(len && my_isdigit(&my_charset_latin1, *src)) + { + i= i * 10 + (*src - '0'); + src++; + } } + else + rc= ER_DYNCOL_TRUNCATED; + if (len) + rc= ER_DYNCOL_TRUNCATED; + *ll= i * sign; + break; + } + case DYN_COL_DECIMAL: + if (decimal2longlong(&val->x.decimal.value, ll) != E_DEC_OK) + rc= ER_DYNCOL_TRUNCATED; + break; + case DYN_COL_DATETIME: + *ll= (val->x.time_value.year * 10000000000ull + + val->x.time_value.month * 100000000L + + val->x.time_value.day * 1000000 + + val->x.time_value.hour * 10000 + + val->x.time_value.minute * 100 + + val->x.time_value.second) * + (val->x.time_value.neg ? -1 : 1); + break; + case DYN_COL_DATE: + *ll= (val->x.time_value.year * 10000 + + val->x.time_value.month * 100 + + val->x.time_value.day) * + (val->x.time_value.neg ? -1 : 1); + break; + case DYN_COL_TIME: + *ll= (val->x.time_value.hour * 10000 + + val->x.time_value.minute * 100 + + val->x.time_value.second) * + (val->x.time_value.neg ? -1 : 1); + break; + case DYN_COL_DYNCOL: + case DYN_COL_NULL: + rc= ER_DYNCOL_TRUNCATED; + break; + default: + return(ER_DYNCOL_FORMAT); + } + return(rc); +} + - memcpy(tmp.str + tmp.length, (char *)header_end + first_offset, - data_size); - tmp.length+= data_size; +enum enum_dyncol_func_result +mariadb_dyncol_val_double(double *dbl, DYNAMIC_COLUMN_VALUE *val) +{ + enum enum_dyncol_func_result rc= ER_DYNCOL_OK; + *dbl= 0; + switch (val->type) { + case DYN_COL_INT: + *dbl= (double)val->x.long_value; + if (((longlong) *dbl) != val->x.long_value) + rc= ER_DYNCOL_TRUNCATED; + break; + case DYN_COL_UINT: + *dbl= (double)val->x.ulong_value; + if (((ulonglong) *dbl) != val->x.ulong_value) + rc= ER_DYNCOL_TRUNCATED; + break; + case DYN_COL_DOUBLE: + *dbl= val->x.double_value; + break; + case DYN_COL_STRING: + { + char *str, *end; + if (!(str= malloc(val->x.string.value.length + 1))) + return ER_DYNCOL_RESOURCE; + memcpy(str, val->x.string.value.str, val->x.string.value.length); + str[val->x.string.value.length]= '\0'; + *dbl= strtod(str, &end); + if (*end != '\0') + rc= ER_DYNCOL_TRUNCATED; + free(str); + break; } + case DYN_COL_DECIMAL: + if (decimal2double(&val->x.decimal.value, dbl) != E_DEC_OK) + rc= ER_DYNCOL_TRUNCATED; + break; + case DYN_COL_DATETIME: + *dbl= (double)(val->x.time_value.year * 10000000000ull + + val->x.time_value.month * 100000000L + + val->x.time_value.day * 1000000 + + val->x.time_value.hour * 10000 + + val->x.time_value.minute * 100 + + val->x.time_value.second) * + (val->x.time_value.neg ? -1 : 1); + break; + case DYN_COL_DATE: + *dbl= (double)(val->x.time_value.year * 10000 + + val->x.time_value.month * 100 + + val->x.time_value.day) * + (val->x.time_value.neg ? -1 : 1); + break; + case DYN_COL_TIME: + *dbl= (double)(val->x.time_value.hour * 10000 + + val->x.time_value.minute * 100 + + val->x.time_value.second) * + (val->x.time_value.neg ? -1 : 1); + break; + case DYN_COL_DYNCOL: + case DYN_COL_NULL: + rc= ER_DYNCOL_TRUNCATED; + break; + default: + return(ER_DYNCOL_FORMAT); + } + return(rc); +} - /* new data adding */ - if (i < add_column_count) + +/** + Convert to JSON + + @param str The packed string + @param json Where to put json result + + @return ER_DYNCOL_* return code +*/ + +#define JSON_STACK_PROTECTION 10 + +static enum enum_dyncol_func_result +mariadb_dyncol_json_internal(DYNAMIC_COLUMN *str, DYNAMIC_STRING *json, + uint lvl) +{ + DYN_HEADER header; + uint i; + enum enum_dyncol_func_result rc; + + if (lvl >= JSON_STACK_PROTECTION) + { + rc= ER_DYNCOL_RESOURCE; + goto err; + } + + + if (str->length == 0) + return ER_DYNCOL_OK; /* no columns */ + + if ((rc= init_read_hdr(&header, str)) < 0) + goto err; + + if (header.entry_size * header.column_count + FIXED_HEADER_SIZE > + str->length) + { + rc= ER_DYNCOL_FORMAT; + goto err; + } + + rc= ER_DYNCOL_RESOURCE; + + if (dynstr_append_mem(json, "{", 1)) + goto err; + for (i= 0, header.entry= header.header; + i < header.column_count; + i++, header.entry+= header.entry_size) + { + DYNAMIC_COLUMN_VALUE val; + if (i != 0 && dynstr_append_mem(json, ",", 1)) + goto err; + header.length= + hdr_interval_length(&header, header.entry + header.entry_size); + header.data= header.dtpool + header.offset; + /* + Check that the found data is within the ranges. This can happen if + we get data with wrong offsets. + */ + if (header.length == DYNCOL_OFFSET_ERROR || + header.length > INT_MAX || header.offset > header.data_size) + { + rc= ER_DYNCOL_FORMAT; + goto err; + } + if ((rc= dynamic_column_get_value(&header, &val)) < 0) + goto err; + if (header.format == dyncol_fmt_num) + { + uint nm= uint2korr(header.entry); + if (dynstr_realloc(json, DYNCOL_NUM_CHAR + 3)) + goto err; + json->str[json->length++]= '"'; + json->length+= (snprintf(json->str + json->length, + DYNCOL_NUM_CHAR, "%u", nm)); + } + else + { + LEX_STRING name; + if (read_name(&header, header.entry, &name)) { - if( plan[i].act == PLAN_ADD || plan[i].act == PLAN_REPLACE) - { - int2store(write, plan[i].num[0]); - type_and_offset_store(write, new_offset_size, - plan[i].val[0].type, - tmp.length - - (FIXED_HEADER_SIZE + new_header_size)); - write+= new_entry_size; - data_store(&tmp, plan[i].val); /* Append new data */ - } - data_delta= plan[i].ddelta; + rc= ER_DYNCOL_FORMAT; + goto err; + } + if (dynstr_realloc(json, name.length + 3)) + goto err; + json->str[json->length++]= '"'; + memcpy(json->str + json->length, name.str, name.length); + json->length+= name.length; + } + json->str[json->length++]= '"'; + json->str[json->length++]= ':'; + if (val.type == DYN_COL_DYNCOL) + { + /* here we use it only for read so can cheat a bit */ + DYNAMIC_COLUMN dc; + bzero(&dc, sizeof(dc)); + dc.str= val.x.string.value.str; + dc.length= val.x.string.value.length; + if (mariadb_dyncol_json_internal(&dc, json, lvl + 1) < 0) + { + dc.str= NULL; dc.length= 0; + goto err; } + dc.str= NULL; dc.length= 0; + } + else + { + if ((rc= mariadb_dyncol_val_str(json, &val, DYNCOL_UTF, DYNCOL_JSON_ESC)) + < 0) + goto err; } - dynamic_column_column_free(str); - *str= tmp; } + if (dynstr_append_mem(json, "}", 1)) + { + rc= ER_DYNCOL_RESOURCE; + goto err; + } + return ER_DYNCOL_OK; - rc= ER_DYNCOL_OK; - -end: - my_free(plan); +err: + json->length= 0; return rc; +} -create_new_string: - /* There is no columns from before, so let's just add the new ones */ - rc= ER_DYNCOL_OK; - if (not_null != 0) - rc= dynamic_column_create_many_internal(str, add_column_count, - column_numbers, values, - str->str == NULL); - goto end; +enum enum_dyncol_func_result +mariadb_dyncol_json(DYNAMIC_COLUMN *str, DYNAMIC_STRING *json) +{ + + if (init_dynamic_string(json, NULL, str->length * 2, 100)) + return ER_DYNCOL_RESOURCE; + + return mariadb_dyncol_json_internal(str, json, 1); } /** - Update the packed string with the given column + Convert to DYNAMIC_COLUMN_VALUE values and names (LEX_STING) dynamic array - @param str String where to write the data - @param column_number Array of columns number - @param values Array of columns values + @param str The packed string + @param count number of elements in the arrays + @param names Where to put names (should be free by user) + @param vals Where to put values (should be free by user) @return ER_DYNCOL_* return code */ +enum enum_dyncol_func_result +mariadb_dyncol_unpack(DYNAMIC_COLUMN *str, + uint *count, + LEX_STRING **names, DYNAMIC_COLUMN_VALUE **vals) +{ + DYN_HEADER header; + char *nm; + uint i; + enum enum_dyncol_func_result rc; + + *count= 0; *names= 0; *vals= 0; + + if (str->length == 0) + return ER_DYNCOL_OK; /* no columns */ + + if ((rc= init_read_hdr(&header, str)) < 0) + return rc; + + + if (header.entry_size * header.column_count + FIXED_HEADER_SIZE > + str->length) + return ER_DYNCOL_FORMAT; + + *vals= my_malloc(sizeof(DYNAMIC_COLUMN_VALUE)* header.column_count, MYF(0)); + if (header.format == dyncol_fmt_num) + { + *names= my_malloc(sizeof(LEX_STRING) * header.column_count + + DYNCOL_NUM_CHAR * header.column_count, MYF(0)); + nm= (char *)((*names) + header.column_count); + } + else + { + *names= my_malloc(sizeof(LEX_STRING) * header.column_count, MYF(0)); + nm= 0; + } + if (!(*vals) || !(*names)) + { + rc= ER_DYNCOL_RESOURCE; + goto err; + } + + for (i= 0, header.entry= header.header; + i < header.column_count; + i++, header.entry+= header.entry_size) + { + header.length= + hdr_interval_length(&header, header.entry + header.entry_size); + header.data= header.dtpool + header.offset; + /* + Check that the found data is within the ranges. This can happen if + we get data with wrong offsets. + */ + if (header.length == DYNCOL_OFFSET_ERROR || + header.length > INT_MAX || header.offset > header.data_size) + { + rc= ER_DYNCOL_FORMAT; + goto err; + } + if ((rc= dynamic_column_get_value(&header, (*vals) + i)) < 0) + goto err; + + if (header.format == dyncol_fmt_num) + { + uint num= uint2korr(header.entry); + (*names)[i].str= nm; + (*names)[i].length= snprintf(nm, DYNCOL_NUM_CHAR, "%u", num); + nm+= (*names)[i].length + 1; + } + else + { + if (read_name(&header, header.entry, (*names) + i)) + { + rc= ER_DYNCOL_FORMAT; + goto err; + } + } + } + + *count= header.column_count; + return ER_DYNCOL_OK; + +err: + if (*vals) + { + my_free(*vals); + *vals= 0; + } + if (*names) + { + my_free(*names); + *names= 0; + } + return rc; +} + +/** + Free arrays allocated by mariadb_dyncol_unpack() + + @param names Where to put names (should be free by user) + @param vals Where to put values (should be free by user) +*/ +void mariadb_dyncol_unpack_free(LEX_STRING *names, DYNAMIC_COLUMN_VALUE *vals) +{ + my_free(names); + my_free(vals); +} + +/** + Get not NULL column count + + @param str The packed string + @param column_count Where to put column count + + @return ER_DYNCOL_* return code +*/ enum enum_dyncol_func_result -dynamic_column_update(DYNAMIC_COLUMN *str, uint column_nr, - DYNAMIC_COLUMN_VALUE *value) +mariadb_dyncol_column_count(DYNAMIC_COLUMN *str, uint *column_count) { - return dynamic_column_update_many(str, 1, &column_nr, value); + DYN_HEADER header; + enum enum_dyncol_func_result rc; + + *(column_count)= 0; + if (str->length == 0) + return ER_DYNCOL_OK; + + if ((rc= init_read_hdr(&header, str)) < 0) + return rc; + *column_count= header.column_count; + return rc; +} +/** + Free dynamic column + + @param str The packed string +*/ +void mariadb_dyncol_free(DYNAMIC_COLUMN *str) +{ + dynstr_free(str); } diff --git a/mysys/md5.c b/mysys/md5.c deleted file mode 100644 index aefc23c1b7a..00000000000 --- a/mysys/md5.c +++ /dev/null @@ -1,325 +0,0 @@ -/* Copyright (c) 2000, 2013, Oracle and/or its affiliates. All rights reserved. - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; version 2 of the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA */ - -/* - * This code implements the MD5 message-digest algorithm. - * The algorithm is due to Ron Rivest. This code was - * written by Colin Plumb in 1993, no copyright is claimed. - * This code is in the public domain; do with it what you wish. - * - * Equivalent code is available from RSA Data Security, Inc. - * This code has been tested against that, and is equivalent, - * except that you don't need to include two pages of legalese - * with every copy. - * - * To compute the message digest of a chunk of bytes, declare an - * MD5Context structure, pass it to MD5Init, call MD5Update as - * needed on buffers full of bytes, and then call MD5Final, which - * will fill a supplied 16-byte array with the digest. - */ - -/* This code was modified in 1997 by Jim Kingdon of Cyclic Software to - not require an integer type which is exactly 32 bits. This work - draws on the changes for the same purpose by Tatu Ylonen - <ylo@cs.hut.fi> as part of SSH, but since I didn't actually use - that code, there is no copyright issue. I hereby disclaim - copyright in any changes I have made; this code remains in the - public domain. */ - -#include <my_global.h> -#include <m_string.h> -#include "my_md5.h" - -#include <string.h> /* for memcpy() and memset() */ - - -static void -my_MD5Transform (cvs_uint32 buf[4], const unsigned char in[64]); - -/* Little-endian byte-swapping routines. Note that these do not - depend on the size of datatypes such as uint32, nor do they require - us to detect the endianness of the machine we are running on. It - is possible they should be macros for speed, but I would be - surprised if they were a performance bottleneck for MD5. */ - -static uint32 getu32 (const unsigned char *addr) -{ - return (((((unsigned long)addr[3] << 8) | addr[2]) << 8) - | addr[1]) << 8 | addr[0]; -} - -static void -putu32 (uint32 data, unsigned char *addr) -{ - addr[0] = (unsigned char)data; - addr[1] = (unsigned char)(data >> 8); - addr[2] = (unsigned char)(data >> 16); - addr[3] = (unsigned char)(data >> 24); -} - -/* - Start MD5 accumulation. Set bit count to 0 and buffer to mysterious - initialization constants. -*/ -void -my_MD5Init (my_MD5Context *ctx) -{ - ctx->buf[0] = 0x67452301; - ctx->buf[1] = 0xefcdab89; - ctx->buf[2] = 0x98badcfe; - ctx->buf[3] = 0x10325476; - - ctx->bits[0] = 0; - ctx->bits[1] = 0; -} - -/* - Update context to reflect the concatenation of another buffer full - of bytes. -*/ -void -my_MD5Update (my_MD5Context *ctx, unsigned char const *buf, unsigned len) -{ - uint32 t; - - /* Update bitcount */ - - t = ctx->bits[0]; - if ((ctx->bits[0] = (t + ((uint32)len << 3)) & 0xffffffff) < t) - ctx->bits[1]++; /* Carry from low to high */ - ctx->bits[1] += len >> 29; - - t = (t >> 3) & 0x3f; /* Bytes already in shsInfo->data */ - - /* Handle any leading odd-sized chunks */ - - if ( t ) { - unsigned char *p = ctx->in + t; - - t = 64-t; - if (len < t) { - memcpy(p, buf, len); - return; - } - memcpy(p, buf, t); - my_MD5Transform (ctx->buf, ctx->in); - buf += t; - len -= t; - } - - /* Process data in 64-byte chunks */ - - while (len >= 64) { - memcpy(ctx->in, buf, 64); - my_MD5Transform (ctx->buf, ctx->in); - buf += 64; - len -= 64; - } - - /* Handle any remaining bytes of data. */ - - memcpy(ctx->in, buf, len); -} - -/* - Final wrapup - pad to 64-byte boundary with the bit pattern - 1 0* (64-bit count of bits processed, MSB-first) -*/ -void -my_MD5Final (unsigned char digest[16], my_MD5Context *ctx) -{ - unsigned count; - unsigned char *p; - - /* Compute number of bytes mod 64 */ - count = (ctx->bits[0] >> 3) & 0x3F; - - /* Set the first char of padding to 0x80. This is safe since there is - always at least one byte free */ - p = ctx->in + count; - *p++ = 0x80; - - /* Bytes of padding needed to make 64 bytes */ - count = 64 - 1 - count; - - /* Pad out to 56 mod 64 */ - if (count < 8) { - /* Two lots of padding: Pad the first block to 64 bytes */ - memset(p, 0, count); - my_MD5Transform (ctx->buf, ctx->in); - - /* Now fill the next block with 56 bytes */ - memset(ctx->in, 0, 56); - } else { - /* Pad block to 56 bytes */ - memset(p, 0, count-8); - } - - /* Append length in bits and transform */ - putu32(ctx->bits[0], ctx->in + 56); - putu32(ctx->bits[1], ctx->in + 60); - - my_MD5Transform (ctx->buf, ctx->in); - putu32(ctx->buf[0], digest); - putu32(ctx->buf[1], digest + 4); - putu32(ctx->buf[2], digest + 8); - putu32(ctx->buf[3], digest + 12); - memset(ctx, 0, sizeof(*ctx)); /* In case it's sensitive */ -} - -#ifndef ASM_MD5 - -/* The four core functions - F1 is optimized somewhat */ - -/* #define F1(x, y, z) (x & y | ~x & z) */ -#define F1(x, y, z) (z ^ (x & (y ^ z))) -#define F2(x, y, z) F1(z, x, y) -#define F3(x, y, z) (x ^ y ^ z) -#define F4(x, y, z) (y ^ (x | ~z)) - -/* This is the central step in the MD5 algorithm. */ -#define MD5STEP(f, w, x, y, z, data, s) \ - ( w += f(x, y, z) + data, w &= 0xffffffff, w = w<<s | w>>(32-s), w += x ) - -/* - * The core of the MD5 algorithm, this alters an existing MD5 hash to - * reflect the addition of 16 longwords of new data. MD5Update blocks - * the data and converts bytes into longwords for this routine. - */ -static void -my_MD5Transform (uint32 buf[4], const unsigned char inraw[64]) -{ - register uint32 a, b, c, d; - uint32 in[16]; - int i; - - for (i = 0; i < 16; ++i) - in[i] = getu32 (inraw + 4 * i); - - a = buf[0]; - b = buf[1]; - c = buf[2]; - d = buf[3]; - - MD5STEP(F1, a, b, c, d, in[ 0]+0xd76aa478, 7); - MD5STEP(F1, d, a, b, c, in[ 1]+0xe8c7b756, 12); - MD5STEP(F1, c, d, a, b, in[ 2]+0x242070db, 17); - MD5STEP(F1, b, c, d, a, in[ 3]+0xc1bdceee, 22); - MD5STEP(F1, a, b, c, d, in[ 4]+0xf57c0faf, 7); - MD5STEP(F1, d, a, b, c, in[ 5]+0x4787c62a, 12); - MD5STEP(F1, c, d, a, b, in[ 6]+0xa8304613, 17); - MD5STEP(F1, b, c, d, a, in[ 7]+0xfd469501, 22); - MD5STEP(F1, a, b, c, d, in[ 8]+0x698098d8, 7); - MD5STEP(F1, d, a, b, c, in[ 9]+0x8b44f7af, 12); - MD5STEP(F1, c, d, a, b, in[10]+0xffff5bb1, 17); - MD5STEP(F1, b, c, d, a, in[11]+0x895cd7be, 22); - MD5STEP(F1, a, b, c, d, in[12]+0x6b901122, 7); - MD5STEP(F1, d, a, b, c, in[13]+0xfd987193, 12); - MD5STEP(F1, c, d, a, b, in[14]+0xa679438e, 17); - MD5STEP(F1, b, c, d, a, in[15]+0x49b40821, 22); - - MD5STEP(F2, a, b, c, d, in[ 1]+0xf61e2562, 5); - MD5STEP(F2, d, a, b, c, in[ 6]+0xc040b340, 9); - MD5STEP(F2, c, d, a, b, in[11]+0x265e5a51, 14); - MD5STEP(F2, b, c, d, a, in[ 0]+0xe9b6c7aa, 20); - MD5STEP(F2, a, b, c, d, in[ 5]+0xd62f105d, 5); - MD5STEP(F2, d, a, b, c, in[10]+0x02441453, 9); - MD5STEP(F2, c, d, a, b, in[15]+0xd8a1e681, 14); - MD5STEP(F2, b, c, d, a, in[ 4]+0xe7d3fbc8, 20); - MD5STEP(F2, a, b, c, d, in[ 9]+0x21e1cde6, 5); - MD5STEP(F2, d, a, b, c, in[14]+0xc33707d6, 9); - MD5STEP(F2, c, d, a, b, in[ 3]+0xf4d50d87, 14); - MD5STEP(F2, b, c, d, a, in[ 8]+0x455a14ed, 20); - MD5STEP(F2, a, b, c, d, in[13]+0xa9e3e905, 5); - MD5STEP(F2, d, a, b, c, in[ 2]+0xfcefa3f8, 9); - MD5STEP(F2, c, d, a, b, in[ 7]+0x676f02d9, 14); - MD5STEP(F2, b, c, d, a, in[12]+0x8d2a4c8a, 20); - - MD5STEP(F3, a, b, c, d, in[ 5]+0xfffa3942, 4); - MD5STEP(F3, d, a, b, c, in[ 8]+0x8771f681, 11); - MD5STEP(F3, c, d, a, b, in[11]+0x6d9d6122, 16); - MD5STEP(F3, b, c, d, a, in[14]+0xfde5380c, 23); - MD5STEP(F3, a, b, c, d, in[ 1]+0xa4beea44, 4); - MD5STEP(F3, d, a, b, c, in[ 4]+0x4bdecfa9, 11); - MD5STEP(F3, c, d, a, b, in[ 7]+0xf6bb4b60, 16); - MD5STEP(F3, b, c, d, a, in[10]+0xbebfbc70, 23); - MD5STEP(F3, a, b, c, d, in[13]+0x289b7ec6, 4); - MD5STEP(F3, d, a, b, c, in[ 0]+0xeaa127fa, 11); - MD5STEP(F3, c, d, a, b, in[ 3]+0xd4ef3085, 16); - MD5STEP(F3, b, c, d, a, in[ 6]+0x04881d05, 23); - MD5STEP(F3, a, b, c, d, in[ 9]+0xd9d4d039, 4); - MD5STEP(F3, d, a, b, c, in[12]+0xe6db99e5, 11); - MD5STEP(F3, c, d, a, b, in[15]+0x1fa27cf8, 16); - MD5STEP(F3, b, c, d, a, in[ 2]+0xc4ac5665, 23); - - MD5STEP(F4, a, b, c, d, in[ 0]+0xf4292244, 6); - MD5STEP(F4, d, a, b, c, in[ 7]+0x432aff97, 10); - MD5STEP(F4, c, d, a, b, in[14]+0xab9423a7, 15); - MD5STEP(F4, b, c, d, a, in[ 5]+0xfc93a039, 21); - MD5STEP(F4, a, b, c, d, in[12]+0x655b59c3, 6); - MD5STEP(F4, d, a, b, c, in[ 3]+0x8f0ccc92, 10); - MD5STEP(F4, c, d, a, b, in[10]+0xffeff47d, 15); - MD5STEP(F4, b, c, d, a, in[ 1]+0x85845dd1, 21); - MD5STEP(F4, a, b, c, d, in[ 8]+0x6fa87e4f, 6); - MD5STEP(F4, d, a, b, c, in[15]+0xfe2ce6e0, 10); - MD5STEP(F4, c, d, a, b, in[ 6]+0xa3014314, 15); - MD5STEP(F4, b, c, d, a, in[13]+0x4e0811a1, 21); - MD5STEP(F4, a, b, c, d, in[ 4]+0xf7537e82, 6); - MD5STEP(F4, d, a, b, c, in[11]+0xbd3af235, 10); - MD5STEP(F4, c, d, a, b, in[ 2]+0x2ad7d2bb, 15); - MD5STEP(F4, b, c, d, a, in[ 9]+0xeb86d391, 21); - - buf[0] += a; - buf[1] += b; - buf[2] += c; - buf[3] += d; -} -#endif - -#ifdef TEST -/* - Simple test program. Can use it to manually run the tests from - RFC1321 for example. -*/ -#include <stdio.h> - -int -main (int argc, char **argv) -{ - my_MD5Context context; - unsigned char checksum[16]; - int i; - int j; - - if (argc < 2) - { - fprintf (stderr, "usage: %s string-to-hash\n", argv[0]); - exit (1); - } - for (j = 1; j < argc; ++j) - { - printf ("MD5 (\"%s\") = ", argv[j]); - my_MD5Init (&context); - my_MD5Update (&context, argv[j], strlen (argv[j])); - my_MD5Final (checksum, &context); - for (i = 0; i < 16; i++) - { - printf ("%02x", (unsigned int) checksum[i]); - } - printf ("\n"); - } - return 0; -} -#endif /* TEST */ diff --git a/mysys/mf_cache.c b/mysys/mf_cache.c index 545084ea97f..a3426889a82 100644 --- a/mysys/mf_cache.c +++ b/mysys/mf_cache.c @@ -20,11 +20,12 @@ #include "my_static.h" #include "mysys_err.h" - /* - Remove an open tempfile so that it doesn't survive - if we crash; If the operating system doesn't support - this, just remember the file name for later removal - */ +/** + Remove an open tempfile so that it doesn't survive if we crash + + If the operating system doesn't support this, just remember + the file name for later removal +*/ static my_bool cache_remove_open_tmp(IO_CACHE *cache __attribute__((unused)), const char *name) @@ -49,41 +50,46 @@ static my_bool cache_remove_open_tmp(IO_CACHE *cache __attribute__((unused)), return 0; } - /* - ** Open tempfile cached by IO_CACHE - ** Should be used when no seeks are done (only reinit_io_buff) - ** Return 0 if cache is inited ok - ** The actual file is created when the IO_CACHE buffer gets filled - ** If dir is not given, use TMPDIR. - */ +/** + Open tempfile cached by IO_CACHE + Should be used when no seeks are done (only reinit_io_buff) + Return 0 if cache is inited ok + The actual file is created when the IO_CACHE buffer gets filled + If dir is not given, use TMPDIR. +*/ my_bool open_cached_file(IO_CACHE *cache, const char* dir, const char *prefix, size_t cache_size, myf cache_myflags) { DBUG_ENTER("open_cached_file"); - cache->dir= dir ? my_strdup(dir,MYF(cache_myflags & MY_WME)) : (char*) 0; - cache->prefix= (prefix ? my_strdup(prefix,MYF(cache_myflags & MY_WME)) : - (char*) 0); + cache->dir= dir; + if (prefix) + { + DBUG_ASSERT(strlen(prefix) == 2); + memcpy(cache->prefix, prefix, 3); + } + else + cache->prefix[0]= 0; cache->file_name=0; cache->buffer=0; /* Mark that not open */ - if (!init_io_cache(cache,-1,cache_size,WRITE_CACHE,0L,0, + if (!init_io_cache(cache, -1, cache_size, WRITE_CACHE, 0L, 0, MYF(cache_myflags | MY_NABP))) { DBUG_RETURN(0); } - my_free(cache->dir); - my_free(cache->prefix); DBUG_RETURN(1); } - /* Create the temporary file */ - +/** + Create the temporary file +*/ my_bool real_open_cached_file(IO_CACHE *cache) { char name_buff[FN_REFLEN]; int error=1; DBUG_ENTER("real_open_cached_file"); - if ((cache->file=create_temp_file(name_buff, cache->dir, cache->prefix, + if ((cache->file=create_temp_file(name_buff, cache->dir, + cache->prefix[0] ? cache->prefix : 0, (O_RDWR | O_BINARY | O_TRUNC | O_TEMPORARY | O_SHORT_LIVED), MYF(MY_WME))) >= 0) @@ -114,8 +120,6 @@ void close_cached_file(IO_CACHE *cache) } #endif } - my_free(cache->dir); - my_free(cache->prefix); } DBUG_VOID_RETURN; } diff --git a/mysys/mf_dirname.c b/mysys/mf_dirname.c index 569293f5401..bc827f60d44 100644 --- a/mysys/mf_dirname.c +++ b/mysys/mf_dirname.c @@ -78,7 +78,7 @@ size_t dirname_part(char *to, const char *name, size_t *to_res_length) SYNPOSIS convert_dirname() to Store result here. Must be at least of size - min(FN_REFLEN, strlen(from) + 1) to make room + MY_MIN(FN_REFLEN, strlen(from) + 1) to make room for adding FN_LIBCHAR at the end. from Original filename. May be == to from_end Pointer at end of filename (normally end \0) diff --git a/mysys/mf_fn_ext.c b/mysys/mf_fn_ext.c index b78d73074da..debed8afed3 100644 --- a/mysys/mf_fn_ext.c +++ b/mysys/mf_fn_ext.c @@ -86,7 +86,7 @@ char *fn_ext2(const char *name) if (!(gpos= strrchr(name, FN_LIBCHAR))) gpos= name; #endif - // locate the last occurence of FN_EXTCHAR + // locate the last occurrence of FN_EXTCHAR pos= strrchr(gpos, FN_EXTCHAR); DBUG_RETURN((char*) (pos ? pos : strend(gpos))); } /* fn_ext2 */ diff --git a/mysys/mf_format.c b/mysys/mf_format.c index 996fee68bd1..6672a4386e4 100644 --- a/mysys/mf_format.c +++ b/mysys/mf_format.c @@ -85,7 +85,7 @@ char * fn_format(char * to, const char *name, const char *dir, tmp_length= strlength(startpos); DBUG_PRINT("error",("dev: '%s' ext: '%s' length: %u",dev,ext, (uint) length)); - (void) strmake(to,startpos,min(tmp_length,FN_REFLEN-1)); + (void) strmake(to,startpos,MY_MIN(tmp_length,FN_REFLEN-1)); } else { diff --git a/mysys/mf_iocache.c b/mysys/mf_iocache.c index 9a051cf6a9e..5072a32206b 100644 --- a/mysys/mf_iocache.c +++ b/mysys/mf_iocache.c @@ -1,5 +1,6 @@ /* Copyright (c) 2000, 2011, Oracle and/or its affiliates + Copyright (c) 2010, 2015, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -22,7 +23,6 @@ Possibly use of asyncronic io. macros for read and writes for faster io. Used instead of FILE when reading or writing whole files. - This code makes mf_rec_cache obsolete (currently only used by ISAM) One can change info->pos_in_file to a higher value to skip bytes in file if also info->read_pos is set to info->read_end. If called through open_cached_file(), then the temporary file will @@ -64,6 +64,16 @@ static void my_aiowait(my_aio_result *result); #define IO_ROUND_UP(X) (((X)+IO_SIZE-1) & ~(IO_SIZE-1)) #define IO_ROUND_DN(X) ( (X) & ~(IO_SIZE-1)) +static int _my_b_cache_read(IO_CACHE *info, uchar *Buffer, size_t Count); +static int _my_b_cache_read_r(IO_CACHE *info, uchar *Buffer, size_t Count); +static int _my_b_seq_read(IO_CACHE *info, uchar *Buffer, size_t Count); +static int _my_b_cache_write(IO_CACHE *info, const uchar *Buffer, size_t Count); +static int _my_b_cache_write_r(IO_CACHE *info, const uchar *Buffer, size_t Count); + +int (*_my_b_encr_read)(IO_CACHE *info,uchar *Buffer,size_t Count)= 0; +int (*_my_b_encr_write)(IO_CACHE *info,const uchar *Buffer,size_t Count)= 0; + + /* Setup internal pointers inside IO_CACHE @@ -72,7 +82,7 @@ static void my_aiowait(my_aio_result *result); info IO_CACHE handler NOTES - This is called on automaticly on init or reinit of IO_CACHE + This is called on automatically on init or reinit of IO_CACHE It must be called externally if one moves or copies an IO_CACHE object. */ @@ -97,6 +107,8 @@ static void init_functions(IO_CACHE* info) { enum cache_type type= info->type; + info->read_function = 0; /* Force a core if used */ + info->write_function = 0; /* Force a core if used */ switch (type) { case READ_NET: /* @@ -106,14 +118,35 @@ init_functions(IO_CACHE* info) programs that link against mysys but know nothing about THD, such as myisamchk */ + DBUG_ASSERT(!(info->myflags & MY_ENCRYPT)); break; case SEQ_READ_APPEND: info->read_function = _my_b_seq_read; - info->write_function = 0; /* Force a core if used */ + DBUG_ASSERT(!(info->myflags & MY_ENCRYPT)); + break; + case READ_CACHE: + if (info->myflags & MY_ENCRYPT) + { + DBUG_ASSERT(info->share == 0); + info->read_function = _my_b_encr_read; + break; + } + /* fall through */ + case WRITE_CACHE: + if (info->myflags & MY_ENCRYPT) + { + info->write_function = _my_b_encr_write; + break; + } + /* fall through */ + case READ_FIFO: + DBUG_ASSERT(!(info->myflags & MY_ENCRYPT)); + info->read_function = info->share ? _my_b_cache_read_r : _my_b_cache_read; + info->write_function = info->share ? _my_b_cache_write_r : _my_b_cache_write; + break; + case TYPE_NOT_SET: + DBUG_ASSERT(0); break; - default: - info->read_function = info->share ? _my_b_read_r : _my_b_read; - info->write_function = _my_b_write; } setup_io_cache(info); @@ -133,8 +166,8 @@ init_functions(IO_CACHE* info) If == 0 then use my_default_record_cache_size type Type of cache seek_offset Where cache should start reading/writing - use_async_io Set to 1 of we should use async_io (if avaiable) - cache_myflags Bitmap of differnt flags + use_async_io Set to 1 of we should use async_io (if available) + cache_myflags Bitmap of different flags MY_WME | MY_FAE | MY_NABP | MY_FNABP | MY_DONT_CHECK_FILESIZE @@ -145,7 +178,7 @@ init_functions(IO_CACHE* info) int init_io_cache(IO_CACHE *info, File file, size_t cachesize, enum cache_type type, my_off_t seek_offset, - pbool use_async_io, myf cache_myflags) + my_bool use_async_io, myf cache_myflags) { size_t min_cache; my_off_t pos; @@ -157,14 +190,13 @@ int init_io_cache(IO_CACHE *info, File file, size_t cachesize, info->file= file; info->type= TYPE_NOT_SET; /* Don't set it until mutex are created */ info->pos_in_file= seek_offset; - info->pre_close = info->pre_read = info->post_read = 0; - info->arg = 0; info->alloced_buffer = 0; info->buffer=0; info->seek_not_done= 0; if (file >= 0) { + DBUG_ASSERT(!(cache_myflags & MY_ENCRYPT)); pos= mysql_file_tell(file, MYF(0)); if ((pos == (my_off_t) -1) && (my_errno == ESPIPE)) { @@ -179,8 +211,14 @@ int init_io_cache(IO_CACHE *info, File file, size_t cachesize, DBUG_ASSERT(seek_offset == 0); } else - info->seek_not_done= test(seek_offset != pos); + info->seek_not_done= MY_TEST(seek_offset != pos); } + else + if (type == WRITE_CACHE && _my_b_encr_read) + { + cache_myflags|= MY_ENCRYPT; + DBUG_ASSERT(seek_offset == 0); + } info->disk_writes= 0; info->share=0; @@ -190,6 +228,7 @@ int init_io_cache(IO_CACHE *info, File file, size_t cachesize, min_cache=use_async_io ? IO_SIZE*4 : IO_SIZE*2; if (type == READ_CACHE || type == SEQ_READ_APPEND) { /* Assume file isn't growing */ + DBUG_ASSERT(!(cache_myflags & MY_ENCRYPT)); if (!(cache_myflags & MY_DONT_CHECK_FILESIZE)) { /* Calculate end of file to avoid allocating oversized buffers */ @@ -207,7 +246,7 @@ int init_io_cache(IO_CACHE *info, File file, size_t cachesize, } } cache_myflags &= ~MY_DONT_CHECK_FILESIZE; - if (type != READ_NET && type != WRITE_NET) + if (type != READ_NET) { /* Retry allocating memory in smaller blocks until we get one */ cachesize= ((cachesize + min_cache-1) & ~(min_cache-1)); @@ -225,15 +264,18 @@ int init_io_cache(IO_CACHE *info, File file, size_t cachesize, buffer_block= cachesize; if (type == SEQ_READ_APPEND) buffer_block *= 2; + else if (cache_myflags & MY_ENCRYPT) + buffer_block= 2*(buffer_block + MY_AES_BLOCK_SIZE) + sizeof(IO_CACHE_CRYPT); if (cachesize == min_cache) flags|= (myf) MY_WME; if ((info->buffer= (uchar*) my_malloc(buffer_block, flags)) != 0) { - info->write_buffer=info->buffer; if (type == SEQ_READ_APPEND) - info->write_buffer = info->buffer + cachesize; - info->alloced_buffer=1; + info->write_buffer= info->buffer + cachesize; + else + info->write_buffer= info->buffer; + info->alloced_buffer= 1; break; /* Enough memory found */ } if (cachesize == min_cache) @@ -277,6 +319,7 @@ int init_io_cache(IO_CACHE *info, File file, size_t cachesize, if (use_async_io && ! my_disable_async_io) { DBUG_PRINT("info",("Using async io")); + DBUG_ASSERT(!(cache_myflags & MY_ENCRYPT)); info->read_length/=2; info->read_function=_my_b_async_read; } @@ -304,7 +347,7 @@ static void my_aiowait(my_aio_result *result) if (errno == EINTR) continue; DBUG_PRINT("error",("No aio request, error: %d",errno)); - result->pending=0; /* Assume everythings is ok */ + result->pending=0; /* Assume everything is ok */ break; } ((my_aio_result*) tmp)->pending=0; @@ -326,18 +369,16 @@ static void my_aiowait(my_aio_result *result) my_bool reinit_io_cache(IO_CACHE *info, enum cache_type type, my_off_t seek_offset, - pbool use_async_io __attribute__((unused)), - pbool clear_cache) + my_bool use_async_io __attribute__((unused)), + my_bool clear_cache) { DBUG_ENTER("reinit_io_cache"); DBUG_PRINT("enter",("cache: 0x%lx type: %d seek_offset: %lu clear_cache: %d", (ulong) info, type, (ulong) seek_offset, (int) clear_cache)); - /* One can't do reinit with the following types */ - DBUG_ASSERT(type != READ_NET && info->type != READ_NET && - type != WRITE_NET && info->type != WRITE_NET && - type != SEQ_READ_APPEND && info->type != SEQ_READ_APPEND); + DBUG_ASSERT(type == READ_CACHE || type == WRITE_CACHE); + DBUG_ASSERT(info->type == READ_CACHE || info->type == WRITE_CACHE); /* If the whole file is in memory, avoid flushing to disk */ if (! clear_cache && @@ -395,8 +436,22 @@ my_bool reinit_io_cache(IO_CACHE *info, enum cache_type type, } else { - info->write_end=(info->buffer + info->buffer_length - - (seek_offset & (IO_SIZE-1))); + if (info->myflags & MY_ENCRYPT) + { + info->write_end = info->write_buffer + info->buffer_length; + if (seek_offset && info->file != -1) + { + info->read_end= info->buffer; + _my_b_encr_read(info, 0, 0); /* prefill the buffer */ + info->write_pos= info->read_pos; + info->seek_not_done=1; + } + } + else + { + info->write_end=(info->buffer + info->buffer_length - + (seek_offset & (IO_SIZE-1))); + } info->end_of_file= ~(my_off_t) 0; } } @@ -409,6 +464,7 @@ my_bool reinit_io_cache(IO_CACHE *info, enum cache_type type, ((ulong) info->buffer_length < (ulong) (info->end_of_file - seek_offset))) { + DBUG_ASSERT(!(cache_myflags & MY_ENCRYPT)); info->read_length=info->buffer_length/2; info->read_function=_my_b_async_read; } @@ -418,12 +474,72 @@ my_bool reinit_io_cache(IO_CACHE *info, enum cache_type type, } /* reinit_io_cache */ +int _my_b_read(IO_CACHE *info, uchar *Buffer, size_t Count) +{ + size_t left_length; + int res; + + /* If the buffer is not empty yet, copy what is available. */ + if ((left_length= (size_t) (info->read_end - info->read_pos))) + { + DBUG_ASSERT(Count > left_length); + memcpy(Buffer, info->read_pos, left_length); + Buffer+=left_length; + Count-=left_length; + } + res= info->read_function(info, Buffer, Count); + if (res && info->error >= 0) + info->error+= left_length; /* update number or read bytes */ + return res; +} + +int _my_b_write(IO_CACHE *info, const uchar *Buffer, size_t Count) +{ + size_t rest_length; + int res; + + /* Always use my_b_flush_io_cache() to flush write_buffer! */ + DBUG_ASSERT(Buffer != info->write_buffer); + + if (info->pos_in_file + info->buffer_length > info->end_of_file) + { + my_errno=errno=EFBIG; + return info->error = -1; + } + + rest_length= (size_t) (info->write_end - info->write_pos); + DBUG_ASSERT(Count >= rest_length); + memcpy(info->write_pos, Buffer, (size_t) rest_length); + Buffer+=rest_length; + Count-=rest_length; + info->write_pos+=rest_length; + + if (my_b_flush_io_cache(info, 1)) + return 1; + + if (Count) + { + my_off_t old_pos_in_file= info->pos_in_file; + res= info->write_function(info, Buffer, Count); + Count-= info->pos_in_file - old_pos_in_file; + Buffer+= info->pos_in_file - old_pos_in_file; + } + else + res= 0; + + if (!res && Count) + { + memcpy(info->write_pos, Buffer, Count); + info->write_pos+= Count; + } + return res; +} /* Read buffered. SYNOPSIS - _my_b_read() + _my_b_cache_read() info IO_CACHE pointer Buffer Buffer to retrieve count bytes from file Count Number of bytes to read into Buffer @@ -439,7 +555,7 @@ my_bool reinit_io_cache(IO_CACHE *info, enum cache_type type, types than my_off_t unless you can be sure that their value fits. Same applies to differences of file offsets. - When changing this function, check _my_b_read_r(). It might need the + When changing this function, check _my_b_cache_read_r(). It might need the same change. RETURN @@ -449,20 +565,11 @@ my_bool reinit_io_cache(IO_CACHE *info, enum cache_type type, Otherwise info->error contains the number of bytes in Buffer. */ -int _my_b_read(register IO_CACHE *info, uchar *Buffer, size_t Count) +int _my_b_cache_read(IO_CACHE *info, uchar *Buffer, size_t Count) { - size_t length,diff_length,left_length, max_length; + size_t length= 0, diff_length, left_length= 0, max_length; my_off_t pos_in_file; - DBUG_ENTER("_my_b_read"); - - /* If the buffer is not empty yet, copy what is available. */ - if ((left_length= (size_t) (info->read_end-info->read_pos))) - { - DBUG_ASSERT(Count >= left_length); /* User is not using my_b_read() */ - memcpy(Buffer,info->read_pos, left_length); - Buffer+=left_length; - Count-=left_length; - } + DBUG_ENTER("_my_b_cache_read"); /* pos_in_file always point on where info->buffer was read */ pos_in_file=info->pos_in_file+ (size_t) (info->read_end - info->buffer); @@ -511,6 +618,7 @@ int _my_b_read(register IO_CACHE *info, uchar *Buffer, size_t Count) { /* End of file. Return, what we did copy from the buffer. */ info->error= (int) left_length; + info->seek_not_done=1; DBUG_RETURN(1); } /* @@ -518,7 +626,7 @@ int _my_b_read(register IO_CACHE *info, uchar *Buffer, size_t Count) what we did already read from a block. That way, the read will end aligned with a block. */ - length=(Count & (size_t) ~(IO_SIZE-1))-diff_length; + length= IO_ROUND_DN(Count) - diff_length; if ((read_length= mysql_file_read(info->file,Buffer, length, info->myflags)) != length) { @@ -528,6 +636,7 @@ int _my_b_read(register IO_CACHE *info, uchar *Buffer, size_t Count) */ info->error= (read_length == (size_t) -1 ? -1 : (int) (read_length+left_length)); + info->seek_not_done=1; DBUG_RETURN(1); } Count-=length; @@ -560,7 +669,14 @@ int _my_b_read(register IO_CACHE *info, uchar *Buffer, size_t Count) info->error= (int) left_length; DBUG_RETURN(1); } - length=0; /* Didn't read any chars */ + else + { + info->error= 0; + if (length == 0) /* nothing was read */ + DBUG_RETURN(0); /* EOF */ + + length= 0; /* non-zero size read was done */ + } } else if ((length= mysql_file_read(info->file,info->buffer, max_length, info->myflags)) < Count || @@ -576,6 +692,7 @@ int _my_b_read(register IO_CACHE *info, uchar *Buffer, size_t Count) /* For a read error, return -1, otherwise, what we got in total. */ info->error= length == (size_t) -1 ? -1 : (int) (length+left_length); info->read_pos=info->read_end=info->buffer; + info->seek_not_done=1; DBUG_RETURN(1); } /* @@ -686,12 +803,15 @@ void init_io_cache_share(IO_CACHE *read_cache, IO_CACHE_SHARE *cshare, cshare->source_cache= write_cache; /* Can be NULL. */ read_cache->share= cshare; - read_cache->read_function= _my_b_read_r; + read_cache->read_function= _my_b_cache_read_r; read_cache->current_pos= NULL; read_cache->current_end= NULL; if (write_cache) + { write_cache->share= cshare; + write_cache->write_function= _my_b_cache_write_r; + } DBUG_VOID_RETURN; } @@ -959,7 +1079,7 @@ static void unlock_io_cache(IO_CACHE *cache) Read from IO_CACHE when it is shared between several threads. SYNOPSIS - _my_b_read_r() + _my_b_cache_read_r() cache IO_CACHE pointer Buffer Buffer to retrieve count bytes from file Count Number of bytes to read into Buffer @@ -984,7 +1104,7 @@ static void unlock_io_cache(IO_CACHE *cache) types than my_off_t unless you can be sure that their value fits. Same applies to differences of file offsets. (Bug #11527) - When changing this function, check _my_b_read(). It might need the + When changing this function, check _my_b_cache_read(). It might need the same change. RETURN @@ -992,20 +1112,14 @@ static void unlock_io_cache(IO_CACHE *cache) 1 Error: can't read requested characters */ -int _my_b_read_r(register IO_CACHE *cache, uchar *Buffer, size_t Count) +static int _my_b_cache_read_r(IO_CACHE *cache, uchar *Buffer, size_t Count) { my_off_t pos_in_file; - size_t length, diff_length, left_length; + size_t length, diff_length, left_length= 0; IO_CACHE_SHARE *cshare= cache->share; - DBUG_ENTER("_my_b_read_r"); + DBUG_ENTER("_my_b_cache_read_r"); + DBUG_ASSERT(!(cache->myflags & MY_ENCRYPT)); - if ((left_length= (size_t) (cache->read_end - cache->read_pos))) - { - DBUG_ASSERT(Count >= left_length); /* User is not using my_b_read() */ - memcpy(Buffer, cache->read_pos, left_length); - Buffer+= left_length; - Count-= left_length; - } while (Count) { size_t cnt, len; @@ -1120,21 +1234,22 @@ int _my_b_read_r(register IO_CACHE *cache, uchar *Buffer, size_t Count) */ static void copy_to_read_buffer(IO_CACHE *write_cache, - const uchar *write_buffer, size_t write_length) + const uchar *write_buffer, my_off_t pos_in_file) { + size_t write_length= write_cache->pos_in_file - pos_in_file; IO_CACHE_SHARE *cshare= write_cache->share; DBUG_ASSERT(cshare->source_cache == write_cache); /* write_length is usually less or equal to buffer_length. - It can be bigger if _my_b_write() is called with a big length. + It can be bigger if _my_b_cache_write_r() is called with a big length. */ while (write_length) { - size_t copy_length= min(write_length, write_cache->buffer_length); + size_t copy_length= MY_MIN(write_length, write_cache->buffer_length); int __attribute__((unused)) rc; - rc= lock_io_cache(write_cache, write_cache->pos_in_file); + rc= lock_io_cache(write_cache, pos_in_file); /* The writing thread does always have the lock when it awakes. */ DBUG_ASSERT(rc); @@ -1142,7 +1257,7 @@ static void copy_to_read_buffer(IO_CACHE *write_cache, cshare->error= 0; cshare->read_end= cshare->buffer + copy_length; - cshare->pos_in_file= write_cache->pos_in_file; + cshare->pos_in_file= pos_in_file; /* Mark all threads as running and wake them. */ unlock_io_cache(write_cache); @@ -1166,20 +1281,12 @@ static void copy_to_read_buffer(IO_CACHE *write_cache, 1 Failed to read */ -int _my_b_seq_read(register IO_CACHE *info, uchar *Buffer, size_t Count) +static int _my_b_seq_read(IO_CACHE *info, uchar *Buffer, size_t Count) { - size_t length, diff_length, left_length, save_count, max_length; + size_t length, diff_length, left_length= 0, save_count, max_length; my_off_t pos_in_file; save_count=Count; - /* first, read the regular buffer */ - if ((left_length=(size_t) (info->read_end-info->read_pos))) - { - DBUG_ASSERT(Count > left_length); /* User is not using my_b_read() */ - memcpy(Buffer,info->read_pos, left_length); - Buffer+=left_length; - Count-=left_length; - } lock_append_buffer(info); /* pos_in_file always point on where info->buffer was read */ @@ -1207,7 +1314,7 @@ int _my_b_seq_read(register IO_CACHE *info, uchar *Buffer, size_t Count) /* Fill first intern buffer */ size_t read_length; - length=(Count & (size_t) ~(IO_SIZE-1))-diff_length; + length= IO_ROUND_DN(Count) - diff_length; if ((read_length= mysql_file_read(info->file,Buffer, length, info->myflags)) == (size_t) -1) { @@ -1285,11 +1392,7 @@ read_append_buffer: size_t transfer_len; DBUG_ASSERT(info->append_read_pos <= info->write_pos); - /* - TODO: figure out if the assert below is needed or correct. - */ - DBUG_ASSERT(pos_in_file == info->end_of_file); - copy_len=min(Count, len_in_buff); + copy_len=MY_MIN(Count, len_in_buff); memcpy(Buffer, info->append_read_pos, copy_len); info->append_read_pos += copy_len; Count -= copy_len; @@ -1328,18 +1431,14 @@ read_append_buffer: 1 An error has occurred; IO_CACHE to error state. */ -int _my_b_async_read(register IO_CACHE *info, uchar *Buffer, size_t Count) +int _my_b_async_read(IO_CACHE *info, uchar *Buffer, size_t Count) { - size_t length,read_length,diff_length,left_length,use_length,org_Count; + size_t length, read_length, diff_length, left_length=0, use_length, org_Count; size_t max_length; my_off_t next_pos_in_file; uchar *read_buffer; - memcpy(Buffer,info->read_pos, - (left_length= (size_t) (info->read_end-info->read_pos))); - Buffer+=left_length; org_Count=Count; - Count-=left_length; if (info->inited) { /* wait for read block */ @@ -1398,7 +1497,7 @@ int _my_b_async_read(register IO_CACHE *info, uchar *Buffer, size_t Count) } #endif /* Copy found bytes to buffer */ - length=min(Count,read_length); + length=MY_MIN(Count,read_length); memcpy(Buffer,info->read_pos,(size_t) length); Buffer+=length; Count-=length; @@ -1432,7 +1531,7 @@ int _my_b_async_read(register IO_CACHE *info, uchar *Buffer, size_t Count) if ((read_length=mysql_file_read(info->file,info->request_pos, read_length, info->myflags)) == (size_t) -1) return info->error= -1; - use_length=min(Count,read_length); + use_length=MY_MIN(Count,read_length); memcpy(Buffer,info->request_pos,(size_t) use_length); info->read_pos=info->request_pos+Count; info->read_end=info->request_pos+read_length; @@ -1493,7 +1592,7 @@ int _my_b_async_read(register IO_CACHE *info, uchar *Buffer, size_t Count) info->read_end-=info->read_length; } info->read_length=info->buffer_length; /* Use hole buffer */ - info->read_function=_my_b_read; /* Use normal IO_READ next */ + info->read_function=_my_b_cache_read; /* Use normal IO_READ next */ } else info->inited=info->aio_result.pending=1; @@ -1508,13 +1607,8 @@ int _my_b_async_read(register IO_CACHE *info, uchar *Buffer, size_t Count) int _my_b_get(IO_CACHE *info) { uchar buff; - IO_CACHE_CALLBACK pre_read,post_read; - if ((pre_read = info->pre_read)) - (*pre_read)(info); if ((*(info)->read_function)(info,&buff,1)) return my_b_EOF; - if ((post_read = info->post_read)) - (*post_read)(info); return (int) (uchar) buff; } @@ -1528,70 +1622,61 @@ int _my_b_get(IO_CACHE *info) -1 On error; my_errno contains error code. */ -int _my_b_write(register IO_CACHE *info, const uchar *Buffer, size_t Count) +int _my_b_cache_write(IO_CACHE *info, const uchar *Buffer, size_t Count) { - size_t rest_length,length; - my_off_t pos_in_file= info->pos_in_file; - - DBUG_EXECUTE_IF("simulate_huge_load_data_file", - { - pos_in_file=(my_off_t)(5000000000ULL); - }); - if (pos_in_file+info->buffer_length > info->end_of_file) + if (Buffer != info->write_buffer) { - my_errno=errno=EFBIG; - return info->error = -1; + Count= IO_ROUND_DN(Count); + if (!Count) + return 0; } - rest_length= (size_t) (info->write_end - info->write_pos); - memcpy(info->write_pos,Buffer,(size_t) rest_length); - Buffer+=rest_length; - Count-=rest_length; - info->write_pos+=rest_length; - - if (my_b_flush_io_cache(info,1)) - return 1; - if (Count >= IO_SIZE) - { /* Fill first intern buffer */ - length=Count & (size_t) ~(IO_SIZE-1); - if (info->seek_not_done) + if (info->seek_not_done) + { + /* + Whenever a function which operates on IO_CACHE flushes/writes + some part of the IO_CACHE to disk it will set the property + "seek_not_done" to indicate this to other functions operating + on the IO_CACHE. + */ + if (mysql_file_seek(info->file, info->pos_in_file, MY_SEEK_SET, + MYF(info->myflags & MY_WME)) == MY_FILEPOS_ERROR) { - /* - Whenever a function which operates on IO_CACHE flushes/writes - some part of the IO_CACHE to disk it will set the property - "seek_not_done" to indicate this to other functions operating - on the IO_CACHE. - */ - if (mysql_file_seek(info->file, info->pos_in_file, MY_SEEK_SET, MYF(0))) - { - info->error= -1; - return (1); - } - info->seek_not_done=0; + info->error= -1; + return 1; } - if (mysql_file_write(info->file, Buffer, length, info->myflags | MY_NABP)) - return info->error= -1; + info->seek_not_done=0; + } + if (mysql_file_write(info->file, Buffer, Count, info->myflags | MY_NABP)) + return info->error= -1; - /* - In case of a shared I/O cache with a writer we normally do direct - write cache to read cache copy. Simulate this here by direct - caller buffer to read cache copy. Do it after the write so that - the cache readers actions on the flushed part can go in parallel - with the write of the extra stuff. copy_to_read_buffer() - synchronizes writer and readers so that after this call the - readers can act on the extra stuff while the writer can go ahead - and prepare the next output. copy_to_read_buffer() relies on - info->pos_in_file. - */ - if (info->share) - copy_to_read_buffer(info, Buffer, length); + info->pos_in_file+= Count; + return 0; +} + + +/* + In case of a shared I/O cache with a writer we normally do direct + write cache to read cache copy. Simulate this here by direct + caller buffer to read cache copy. Do it after the write so that + the cache readers actions on the flushed part can go in parallel + with the write of the extra stuff. copy_to_read_buffer() + synchronizes writer and readers so that after this call the + readers can act on the extra stuff while the writer can go ahead + and prepare the next output. copy_to_read_buffer() relies on + info->pos_in_file. +*/ +static int _my_b_cache_write_r(IO_CACHE *info, const uchar *Buffer, size_t Count) +{ + my_off_t old_pos_in_file= info->pos_in_file; + int res= _my_b_cache_write(info, Buffer, Count); + if (res) + return res; + + DBUG_ASSERT(!(info->myflags & MY_ENCRYPT)); + DBUG_ASSERT(info->share); + copy_to_read_buffer(info, Buffer, old_pos_in_file); - Count-=length; - Buffer+=length; - info->pos_in_file+=length; - } - memcpy(info->write_pos,Buffer,(size_t) Count); - info->write_pos+=Count; return 0; } @@ -1602,7 +1687,7 @@ int _my_b_write(register IO_CACHE *info, const uchar *Buffer, size_t Count) the write buffer before we are ready with it. */ -int my_b_append(register IO_CACHE *info, const uchar *Buffer, size_t Count) +int my_b_append(IO_CACHE *info, const uchar *Buffer, size_t Count) { size_t rest_length,length; @@ -1611,6 +1696,7 @@ int my_b_append(register IO_CACHE *info, const uchar *Buffer, size_t Count) day, we might need to add a call to copy_to_read_buffer(). */ DBUG_ASSERT(!info->share); + DBUG_ASSERT(!(info->myflags & MY_ENCRYPT)); lock_append_buffer(info); rest_length= (size_t) (info->write_end - info->write_pos); @@ -1627,7 +1713,7 @@ int my_b_append(register IO_CACHE *info, const uchar *Buffer, size_t Count) } if (Count >= IO_SIZE) { /* Fill first intern buffer */ - length=Count & (size_t) ~(IO_SIZE-1); + length= IO_ROUND_DN(Count); if (mysql_file_write(info->file,Buffer, length, info->myflags | MY_NABP)) { unlock_append_buffer(info); @@ -1666,7 +1752,7 @@ int my_b_safe_write(IO_CACHE *info, const uchar *Buffer, size_t Count) we will never get a seek over the end of the buffer */ -int my_block_write(register IO_CACHE *info, const uchar *Buffer, size_t Count, +int my_block_write(IO_CACHE *info, const uchar *Buffer, size_t Count, my_off_t pos) { size_t length; @@ -1677,6 +1763,7 @@ int my_block_write(register IO_CACHE *info, const uchar *Buffer, size_t Count, day, we might need to add a call to copy_to_read_buffer(). */ DBUG_ASSERT(!info->share); + DBUG_ASSERT(!(info->myflags & MY_ENCRYPT)); if (pos < info->pos_in_file) { @@ -1724,11 +1811,9 @@ int my_block_write(register IO_CACHE *info, const uchar *Buffer, size_t Count, #define UNLOCK_APPEND_BUFFER if (need_append_buffer_lock) \ unlock_append_buffer(info); -int my_b_flush_io_cache(IO_CACHE *info, - int need_append_buffer_lock __attribute__((unused))) +int my_b_flush_io_cache(IO_CACHE *info, int need_append_buffer_lock) { size_t length; - my_off_t pos_in_file; my_bool append_cache= (info->type == SEQ_READ_APPEND); DBUG_ENTER("my_b_flush_io_cache"); DBUG_PRINT("enter", ("cache: 0x%lx", (long) info)); @@ -1747,52 +1832,30 @@ int my_b_flush_io_cache(IO_CACHE *info, if ((length=(size_t) (info->write_pos - info->write_buffer))) { - /* - In case of a shared I/O cache with a writer we do direct write - cache to read cache copy. Do it before the write here so that - the readers can work in parallel with the write. - copy_to_read_buffer() relies on info->pos_in_file. - */ - if (info->share) - copy_to_read_buffer(info, info->write_buffer, length); - - pos_in_file=info->pos_in_file; - /* - If we have append cache, we always open the file with - O_APPEND which moves the pos to EOF automatically on every write - */ - if (!append_cache && info->seek_not_done) - { /* File touched, do seek */ - if (mysql_file_seek(info->file, pos_in_file, MY_SEEK_SET, MYF(info->myflags & MY_WME)) == - MY_FILEPOS_ERROR) - { - UNLOCK_APPEND_BUFFER; - DBUG_RETURN((info->error= -1)); - } - if (!append_cache) - info->seek_not_done=0; - } - if (!append_cache) - info->pos_in_file+=length; - info->write_end= (info->write_buffer+info->buffer_length- - ((pos_in_file+length) & (IO_SIZE-1))); - - if (mysql_file_write(info->file,info->write_buffer,length, - info->myflags | MY_NABP)) - info->error= -1; - else - info->error= 0; - if (!append_cache) + if (append_cache) { - set_if_bigger(info->end_of_file,(pos_in_file+length)); + + if (mysql_file_write(info->file, info->write_buffer, length, + info->myflags | MY_NABP)) + info->error= -1; + else + info->error= 0; + + info->end_of_file+= info->write_pos - info->append_read_pos; + info->append_read_pos= info->write_buffer; + DBUG_ASSERT(info->end_of_file == mysql_file_tell(info->file, MYF(0))); } else { - info->end_of_file+=(info->write_pos-info->append_read_pos); - DBUG_ASSERT(info->end_of_file == mysql_file_tell(info->file, MYF(0))); - } + int res= info->write_function(info, info->write_buffer, length); + if (res) + DBUG_RETURN(res); - info->append_read_pos=info->write_pos=info->write_buffer; + set_if_bigger(info->end_of_file, info->pos_in_file); + } + info->write_end= (info->write_buffer + info->buffer_length - + ((info->pos_in_file + length) & (IO_SIZE - 1))); + info->write_pos= info->write_buffer; ++info->disk_writes; UNLOCK_APPEND_BUFFER; DBUG_RETURN(info->error); @@ -1820,6 +1883,7 @@ int my_b_flush_io_cache(IO_CACHE *info, It's currently safe to call this if one has called init_io_cache() on the 'info' object, even if init_io_cache() failed. This function is also safe to call twice with the same handle. + Note that info->file is not reset as the caller may still use ut for my_close() RETURN 0 ok @@ -1829,7 +1893,6 @@ int my_b_flush_io_cache(IO_CACHE *info, int end_io_cache(IO_CACHE *info) { int error=0; - IO_CACHE_CALLBACK pre_close; DBUG_ENTER("end_io_cache"); DBUG_PRINT("enter",("cache: 0x%lx", (ulong) info)); @@ -1839,11 +1902,6 @@ int end_io_cache(IO_CACHE *info) */ DBUG_ASSERT(!info->share || !info->share->total_threads); - if ((pre_close=info->pre_close)) - { - (*pre_close)(info); - info->pre_close= 0; - } if (info->alloced_buffer) { info->alloced_buffer=0; @@ -1855,10 +1913,12 @@ int end_io_cache(IO_CACHE *info) if (info->type == SEQ_READ_APPEND) { /* Destroy allocated mutex */ - info->type= TYPE_NOT_SET; mysql_mutex_destroy(&info->append_buffer_lock); } info->share= 0; + info->type= TYPE_NOT_SET; /* Ensure that flush_io_cache() does nothing */ + info->write_end= 0; /* Ensure that my_b_write() fails */ + info->write_function= 0; /* my_b_write will crash if used */ DBUG_RETURN(error); } /* end_io_cache */ @@ -1946,7 +2006,7 @@ int main(int argc, char** argv) /* check correctness of tests */ if (total_bytes != status.st_size) { - fprintf(stderr,"Not the same number of bytes acutally in file as bytes \ + fprintf(stderr,"Not the same number of bytes actually in file as bytes \ supposedly written\n"); error=1; } diff --git a/mysys/mf_iocache2.c b/mysys/mf_iocache2.c index a9a237696d2..f3877331664 100644 --- a/mysys/mf_iocache2.c +++ b/mysys/mf_iocache2.c @@ -23,52 +23,56 @@ #include <stdarg.h> #include <m_ctype.h> -/* - Copy contents of an IO_CACHE to a file. - - SYNOPSIS - my_b_copy_to_file() - cache IO_CACHE to copy from - file File to copy to - - DESCRIPTION - Copy the contents of the cache to the file. The cache will be - re-inited to a read cache and will read from the beginning of the - cache. - - If a failure to write fully occurs, the cache is only copied - partially. +/** + Copy the cache to the file. Copying can be constrained to @c count + number of bytes when the parameter is less than SIZE_T_MAX. The + cache will be optionally re-inited to a read cache and will read + from the beginning of the cache. If a failure to write fully + occurs, the cache is only copied partially. TODO - Make this function solid by handling partial reads from the cache - in a correct manner: it should be atomic. - - RETURN VALUE - 0 All OK - 1 An error occured + Make this function solid by handling partial reads from the cache + in a correct manner: it should be atomic. + + @param cache IO_CACHE to copy from + @param file File to copy to + @param count the copied size or the max of the type + when the whole cache is to be copied. + @return + 0 All OK + 1 An error occurred */ int -my_b_copy_to_file(IO_CACHE *cache, FILE *file) +my_b_copy_to_file(IO_CACHE *cache, FILE *file, + size_t count) { - size_t bytes_in_cache; + size_t curr_write, bytes_in_cache; DBUG_ENTER("my_b_copy_to_file"); - /* Reinit the cache to read from the beginning of the cache */ - if (reinit_io_cache(cache, READ_CACHE, 0L, FALSE, FALSE)) - DBUG_RETURN(1); bytes_in_cache= my_b_bytes_in_cache(cache); do { - if (my_fwrite(file, cache->read_pos, bytes_in_cache, + curr_write= MY_MIN(bytes_in_cache, count); + if (my_fwrite(file, cache->read_pos, curr_write, MYF(MY_WME | MY_NABP)) == (size_t) -1) DBUG_RETURN(1); - cache->read_pos= cache->read_end; - } while ((bytes_in_cache= my_b_fill(cache))); + + cache->read_pos += curr_write; + count -= curr_write; + } while (count && (bytes_in_cache= my_b_fill(cache))); if(cache->error == -1) DBUG_RETURN(1); DBUG_RETURN(0); } +int my_b_copy_all_to_file(IO_CACHE *cache, FILE *file) +{ + DBUG_ENTER("my_b_copy_all_to_file"); + /* Reinit the cache to read from the beginning of the cache */ + if (reinit_io_cache(cache, READ_CACHE, 0L, FALSE, FALSE)) + DBUG_RETURN(1); + DBUG_RETURN(my_b_copy_to_file(cache, file, SIZE_T_MAX)); +} my_off_t my_b_append_tell(IO_CACHE* info) { @@ -103,14 +107,14 @@ my_off_t my_b_append_tell(IO_CACHE* info) */ { volatile my_off_t save_pos; - save_pos = mysql_file_tell(info->file,MYF(0)); - mysql_file_seek(info->file,(my_off_t)0,MY_SEEK_END,MYF(0)); + save_pos= mysql_file_tell(info->file, MYF(0)); + mysql_file_seek(info->file, 0, MY_SEEK_END, MYF(0)); /* Save the value of my_tell in res so we can see it when studying coredump */ DBUG_ASSERT(info->end_of_file - (info->append_read_pos-info->write_buffer) - == (res=mysql_file_tell(info->file,MYF(0)))); - mysql_file_seek(info->file,save_pos,MY_SEEK_SET,MYF(0)); + == (res= mysql_file_tell(info->file, MYF(0)))); + mysql_file_seek(info->file, save_pos, MY_SEEK_SET, MYF(0)); } #endif res = info->end_of_file + (info->write_pos-info->append_read_pos); @@ -182,60 +186,19 @@ void my_b_seek(IO_CACHE *info,my_off_t pos) DBUG_VOID_RETURN; } - -/* - Fill buffer of the cache. - - NOTES - This assumes that you have already used all characters in the CACHE, - independent of the read_pos value! - - RETURN - 0 On error or EOF (info->error = -1 on error) - # Number of characters -*/ - - -size_t my_b_fill(IO_CACHE *info) +int my_b_pread(IO_CACHE *info, uchar *Buffer, size_t Count, my_off_t pos) { - my_off_t pos_in_file=(info->pos_in_file+ - (size_t) (info->read_end - info->buffer)); - size_t diff_length, length, max_length; - - if (info->seek_not_done) - { /* File touched, do seek */ - if (mysql_file_seek(info->file,pos_in_file,MY_SEEK_SET,MYF(0)) == - MY_FILEPOS_ERROR) - { - info->error= 0; - return 0; - } - info->seek_not_done=0; - } - diff_length=(size_t) (pos_in_file & (IO_SIZE-1)); - max_length=(info->read_length-diff_length); - if (max_length >= (info->end_of_file - pos_in_file)) - max_length= (size_t) (info->end_of_file - pos_in_file); - - if (!max_length) - { - info->error= 0; - return 0; /* EOF */ - } - DBUG_EXECUTE_IF ("simulate_my_b_fill_error", - {DBUG_SET("+d,simulate_file_read_error");}); - if ((length= mysql_file_read(info->file,info->buffer,max_length, - info->myflags)) == (size_t) -1) + if (info->myflags & MY_ENCRYPT) { - info->error= -1; - return 0; + my_b_seek(info, pos); + return my_b_read(info, Buffer, Count); } - info->read_pos=info->buffer; - info->read_end=info->buffer+length; - info->pos_in_file=pos_in_file; - return length; -} + /* backward compatibility behavior. XXX remove it? */ + if (mysql_file_pread(info->file, Buffer, Count, pos, info->myflags | MY_NABP)) + return info->error= -1; + return 0; +} /* Read a string ended by '\n' into a buffer of 'max_length' size. @@ -288,7 +251,7 @@ my_off_t my_b_filelength(IO_CACHE *info) return my_b_tell(info); info->seek_not_done= 1; - return mysql_file_seek(info->file, 0L, MY_SEEK_END, MYF(0)); + return mysql_file_seek(info->file, 0, MY_SEEK_END, MYF(0)); } @@ -505,7 +468,7 @@ process_flags: if (my_b_write(info, (uchar*) buff, length2)) goto err; } - else if ((*fmt == 'l' && fmt[1] == 'd') || fmt[1] == 'u') + else if ((*fmt == 'l' && (fmt[1] == 'd' || fmt[1] == 'u'))) /* long parameter */ { register long iarg; @@ -534,4 +497,3 @@ process_flags: err: return (size_t) -1; } - diff --git a/mysys/mf_keycache.c b/mysys/mf_keycache.c index 8191e92bb27..9356f9154f0 100644 --- a/mysys/mf_keycache.c +++ b/mysys/mf_keycache.c @@ -58,11 +58,11 @@ When a new block is required it is first tried to pop one from the stack. If the stack is empty, it is tried to get a never-used block from the pool. If this is empty too, then a block is taken from the LRU ring, flushing it - to disk, if neccessary. This is handled in find_key_block(). + to disk, if necessary. This is handled in find_key_block(). With the new free list, the blocks can have three temperatures: hot, warm and cold (which is free). This is remembered in the block header by the enum BLOCK_TEMPERATURE temperature variable. Remembering the - temperature is neccessary to correctly count the number of warm blocks, + temperature is necessary to correctly count the number of warm blocks, which is required to decide when blocks are allowed to become hot. Whenever a block is inserted to another (sub-)chain, we take the old and new temperature into account to decide if we got one more or less warm block. @@ -149,7 +149,8 @@ typedef struct st_keycache_wqueue struct st_my_thread_var *last_thread; /* circular list of waiting threads */ } KEYCACHE_WQUEUE; -#define CHANGED_BLOCKS_HASH 128 /* must be power of 2 */ +/* Default size of hash for changed files */ +#define MIN_CHANGED_BLOCKS_HASH_SIZE 128 /* Control block for a simple (non-partitioned) key cache */ @@ -165,6 +166,7 @@ typedef struct st_simple_key_cache_cb ulong age_threshold; /* age threshold for hot blocks */ ulonglong keycache_time; /* total number of block link operations */ uint hash_entries; /* max number of entries in the hash table */ + uint changed_blocks_hash_size; /* Number of hash buckets for file blocks */ int hash_links; /* max number of hash links */ int hash_links_used; /* number of hash links currently used */ int disk_blocks; /* max number of blocks in the cache */ @@ -191,8 +193,8 @@ typedef struct st_simple_key_cache_cb KEYCACHE_WQUEUE waiting_for_resize_cnt; KEYCACHE_WQUEUE waiting_for_hash_link; /* waiting for a free hash link */ KEYCACHE_WQUEUE waiting_for_block; /* requests waiting for a free block */ - BLOCK_LINK *changed_blocks[CHANGED_BLOCKS_HASH]; /* hash for dirty file bl.*/ - BLOCK_LINK *file_blocks[CHANGED_BLOCKS_HASH]; /* hash for other file bl.*/ + BLOCK_LINK **changed_blocks; /* hash for dirty file bl.*/ + BLOCK_LINK **file_blocks; /* hash for other file bl.*/ /* Statistics variables. These are reset in reset_key_cache_counters(). */ ulong global_blocks_changed; /* number of currently dirty blocks */ @@ -272,7 +274,7 @@ struct st_hash_link }; /* simple states of a block */ -#define BLOCK_ERROR 1 /* an error occured when performing file i/o */ +#define BLOCK_ERROR 1 /* an error occurred when performing file i/o */ #define BLOCK_READ 2 /* file block is in the block buffer */ #define BLOCK_IN_SWITCH 4 /* block is preparing to read new page */ #define BLOCK_REASSIGNED 8 /* blk does not accept requests for old page */ @@ -331,7 +333,7 @@ static void test_key_cache(SIMPLE_KEY_CACHE_CB *keycache, #define KEYCACHE_HASH(f, pos) \ ((KEYCACHE_BASE_EXPR(f, pos) / keycache->hash_factor) & \ (keycache->hash_entries-1)) -#define FILE_HASH(f) ((uint) (f) & (CHANGED_BLOCKS_HASH-1)) +#define FILE_HASH(f, cache) ((uint) (f) & (cache->changed_blocks_hash_size-1)) #define DEFAULT_KEYCACHE_DEBUG_LOG "keycache_debug.log" @@ -448,7 +450,7 @@ static inline uint next_power(uint value) structure of the type SIMPLE_KEY_CACHE_CB that is used for this key cache. The parameter keycache is supposed to point to this structure. The parameter key_cache_block_size specifies the size of the blocks in - the key cache to be built. The parameters division_limit and age_threshhold + the key cache to be built. The parameters division_limit and age_threshold determine the initial values of those characteristics of the key cache that are used for midpoint insertion strategy. The parameter use_mem specifies the total amount of memory to be allocated for key cache blocks @@ -468,9 +470,10 @@ static inline uint next_power(uint value) */ static -int init_simple_key_cache(SIMPLE_KEY_CACHE_CB *keycache, uint key_cache_block_size, +int init_simple_key_cache(SIMPLE_KEY_CACHE_CB *keycache, + uint key_cache_block_size, size_t use_mem, uint division_limit, - uint age_threshold) + uint age_threshold, uint changed_blocks_hash_size) { ulong blocks, hash_links; size_t length; @@ -515,6 +518,11 @@ int init_simple_key_cache(SIMPLE_KEY_CACHE_CB *keycache, uint key_cache_block_si blocks= (ulong) (use_mem / (sizeof(BLOCK_LINK) + 2 * sizeof(HASH_LINK) + sizeof(HASH_LINK*) * 5/4 + key_cache_block_size)); + + /* Changed blocks hash needs to be a power of 2 */ + changed_blocks_hash_size= my_round_up_to_next_power(MY_MAX(changed_blocks_hash_size, + MIN_CHANGED_BLOCKS_HASH_SIZE)); + /* It doesn't make sense to have too few blocks (less than 8) */ if (blocks >= 8) { @@ -531,8 +539,9 @@ int init_simple_key_cache(SIMPLE_KEY_CACHE_CB *keycache, uint key_cache_block_si while ((length= (ALIGN_SIZE(blocks * sizeof(BLOCK_LINK)) + ALIGN_SIZE(hash_links * sizeof(HASH_LINK)) + ALIGN_SIZE(sizeof(HASH_LINK*) * - keycache->hash_entries))) + - ((size_t) blocks * keycache->key_cache_block_size) > use_mem) + keycache->hash_entries) + + sizeof(BLOCK_LINK*)* (changed_blocks_hash_size*2))) + + ((size_t) blocks * keycache->key_cache_block_size) > use_mem && blocks > 8) blocks--; /* Allocate memory for cache page buffers */ if ((keycache->block_mem= @@ -543,8 +552,21 @@ int init_simple_key_cache(SIMPLE_KEY_CACHE_CB *keycache, uint key_cache_block_si Allocate memory for blocks, hash_links and hash entries; For each block 2 hash links are allocated */ - if ((keycache->block_root= (BLOCK_LINK*) my_malloc(length, - MYF(0)))) + if (my_multi_malloc_large(MYF(MY_ZEROFILL), + &keycache->block_root, + (ulonglong) (blocks * sizeof(BLOCK_LINK)), + &keycache->hash_root, + (ulonglong) (sizeof(HASH_LINK*) * + keycache->hash_entries), + &keycache->hash_link_root, + (ulonglong) (hash_links * sizeof(HASH_LINK)), + &keycache->changed_blocks, + (ulonglong) (sizeof(BLOCK_LINK*) * + changed_blocks_hash_size), + &keycache->file_blocks, + (ulonglong) (sizeof(BLOCK_LINK*) * + changed_blocks_hash_size), + NullS)) break; my_large_free(keycache->block_mem); keycache->block_mem= 0; @@ -561,17 +583,6 @@ int init_simple_key_cache(SIMPLE_KEY_CACHE_CB *keycache, uint key_cache_block_si keycache->blocks_unused= blocks; keycache->disk_blocks= (int) blocks; keycache->hash_links= hash_links; - keycache->hash_root= (HASH_LINK**) ((char*) keycache->block_root + - ALIGN_SIZE(blocks*sizeof(BLOCK_LINK))); - keycache->hash_link_root= (HASH_LINK*) ((char*) keycache->hash_root + - ALIGN_SIZE((sizeof(HASH_LINK*) * - keycache->hash_entries))); - bzero((uchar*) keycache->block_root, - keycache->disk_blocks * sizeof(BLOCK_LINK)); - bzero((uchar*) keycache->hash_root, - keycache->hash_entries * sizeof(HASH_LINK*)); - bzero((uchar*) keycache->hash_link_root, - keycache->hash_links * sizeof(HASH_LINK)); keycache->hash_links_used= 0; keycache->free_hash_list= NULL; keycache->blocks_used= keycache->blocks_changed= 0; @@ -591,7 +602,7 @@ int init_simple_key_cache(SIMPLE_KEY_CACHE_CB *keycache, uint key_cache_block_si keycache->age_threshold= (age_threshold ? blocks * age_threshold / 100 : blocks); - + keycache->changed_blocks_hash_size= changed_blocks_hash_size; keycache->can_be_used= 1; keycache->waiting_for_hash_link.last_thread= NULL; @@ -602,10 +613,6 @@ int init_simple_key_cache(SIMPLE_KEY_CACHE_CB *keycache, uint key_cache_block_si keycache->disk_blocks, (long) keycache->block_root, keycache->hash_entries, (long) keycache->hash_root, keycache->hash_links, (long) keycache->hash_link_root)); - bzero((uchar*) keycache->changed_blocks, - sizeof(keycache->changed_blocks[0]) * CHANGED_BLOCKS_HASH); - bzero((uchar*) keycache->file_blocks, - sizeof(keycache->file_blocks[0]) * CHANGED_BLOCKS_HASH); } else { @@ -832,9 +839,10 @@ void finish_resize_simple_key_cache(SIMPLE_KEY_CACHE_CB *keycache, */ static -int resize_simple_key_cache(SIMPLE_KEY_CACHE_CB *keycache, uint key_cache_block_size, +int resize_simple_key_cache(SIMPLE_KEY_CACHE_CB *keycache, + uint key_cache_block_size, size_t use_mem, uint division_limit, - uint age_threshold) + uint age_threshold, uint changed_blocks_hash_size) { int blocks= 0; DBUG_ENTER("resize_simple_key_cache"); @@ -852,7 +860,8 @@ int resize_simple_key_cache(SIMPLE_KEY_CACHE_CB *keycache, uint key_cache_block_ /* The following will work even if use_mem is 0 */ blocks= init_simple_key_cache(keycache, key_cache_block_size, use_mem, - division_limit, age_threshold); + division_limit, age_threshold, + changed_blocks_hash_size); finish: finish_resize_simple_key_cache(keycache, 0); @@ -1015,11 +1024,11 @@ void end_simple_key_cache(SIMPLE_KEY_CACHE_CB *keycache, my_bool cleanup) */ static void link_into_queue(KEYCACHE_WQUEUE *wqueue, - struct st_my_thread_var *thread) + struct st_my_thread_var *thread) { struct st_my_thread_var *last; - DBUG_ASSERT(!thread->next && !thread->prev); + if (! (last= wqueue->last_thread)) { /* Queue is empty */ @@ -1028,10 +1037,15 @@ static void link_into_queue(KEYCACHE_WQUEUE *wqueue, } else { - thread->prev= last->next->prev; - last->next->prev= &thread->next; - thread->next= last->next; - last->next= thread; + DBUG_ASSERT(last->next->prev == &last->next); + /* Add backlink to previous element */ + thread->prev= last->next->prev; + /* Fix first in list to point backwords to current */ + last->next->prev= &thread->next; + /* Next should point to the first element in list */ + thread->next= last->next; + /* Fix old element to point to new one */ + last->next= thread; } wqueue->last_thread= thread; } @@ -1052,17 +1066,22 @@ static void link_into_queue(KEYCACHE_WQUEUE *wqueue, */ static void unlink_from_queue(KEYCACHE_WQUEUE *wqueue, - struct st_my_thread_var *thread) + struct st_my_thread_var *thread) { KEYCACHE_DBUG_PRINT("unlink_from_queue", ("thread %ld", thread->id)); DBUG_ASSERT(thread->next && thread->prev); + if (thread->next == thread) + { /* The queue contains only one member */ wqueue->last_thread= NULL; + } else { + /* Remove current element from list */ thread->next->prev= thread->prev; - *thread->prev=thread->next; + *thread->prev= thread->next; + /* If first element, change list pointer to point to previous element */ if (wqueue->last_thread == thread) wqueue->last_thread= STRUCT_PTR(struct st_my_thread_var, next, thread->prev); @@ -1106,10 +1125,11 @@ static void wait_on_queue(KEYCACHE_WQUEUE *wqueue, { struct st_my_thread_var *last; struct st_my_thread_var *thread= my_thread_var; - - /* Add to queue. */ DBUG_ASSERT(!thread->next); DBUG_ASSERT(!thread->prev); /* Not required, but must be true anyway. */ + mysql_mutex_assert_owner(mutex); + + /* Add to queue. */ if (! (last= wqueue->last_thread)) thread->next= thread; else @@ -1120,7 +1140,7 @@ static void wait_on_queue(KEYCACHE_WQUEUE *wqueue, wqueue->last_thread= thread; /* - Wait until thread is removed from queue by the signalling thread. + Wait until thread is removed from queue by the signaling thread. The loop protects against stray signals. */ do @@ -1158,17 +1178,19 @@ static void release_whole_queue(KEYCACHE_WQUEUE *wqueue) if (!(last= wqueue->last_thread)) return; - next= last->next; + next= last->next; /* First (oldest) element */ do { thread=next; + DBUG_ASSERT(thread && thread->init == 1); KEYCACHE_DBUG_PRINT("release_whole_queue: signal", ("thread %ld", thread->id)); - /* Signal the thread. */ - keycache_pthread_cond_signal(&thread->suspend); /* Take thread from queue. */ - next=thread->next; + next= thread->next; thread->next= NULL; + + /* Signal the thread. */ + keycache_pthread_cond_signal(&thread->suspend); } while (thread != last); @@ -1248,7 +1270,7 @@ static void link_to_file_list(SIMPLE_KEY_CACHE_CB *keycache, DBUG_ASSERT(block->hash_link->file == file); if (unlink_block) unlink_changed(block); - link_changed(block, &keycache->file_blocks[FILE_HASH(file)]); + link_changed(block, &keycache->file_blocks[FILE_HASH(file, keycache)]); if (block->status & BLOCK_CHANGED) { block->status&= ~BLOCK_CHANGED; @@ -1289,7 +1311,7 @@ static void link_to_changed_list(SIMPLE_KEY_CACHE_CB *keycache, unlink_changed(block); link_changed(block, - &keycache->changed_blocks[FILE_HASH(block->hash_link->file)]); + &keycache->changed_blocks[FILE_HASH(block->hash_link->file, keycache)]); block->status|=BLOCK_CHANGED; keycache->blocks_changed++; keycache->global_blocks_changed++; @@ -1354,7 +1376,7 @@ static void link_block(SIMPLE_KEY_CACHE_CB *keycache, BLOCK_LINK *block, keycache->waiting_for_block.last_thread; struct st_my_thread_var *first_thread= last_thread->next; struct st_my_thread_var *next_thread= first_thread; - HASH_LINK *hash_link= (HASH_LINK *) first_thread->opt_info; + HASH_LINK *hash_link= (HASH_LINK *) first_thread->keycache_link; struct st_my_thread_var *thread; do { @@ -1364,7 +1386,7 @@ static void link_block(SIMPLE_KEY_CACHE_CB *keycache, BLOCK_LINK *block, We notify about the event all threads that ask for the same page as the first thread in the queue */ - if ((HASH_LINK *) thread->opt_info == hash_link) + if ((HASH_LINK *) thread->keycache_link == hash_link) { KEYCACHE_DBUG_PRINT("link_block: signal", ("thread %ld", thread->id)); keycache_pthread_cond_signal(&thread->suspend); @@ -1698,7 +1720,7 @@ static void unlink_hash(SIMPLE_KEY_CACHE_CB *keycache, HASH_LINK *hash_link) keycache->waiting_for_hash_link.last_thread; struct st_my_thread_var *first_thread= last_thread->next; struct st_my_thread_var *next_thread= first_thread; - KEYCACHE_PAGE *first_page= (KEYCACHE_PAGE *) (first_thread->opt_info); + KEYCACHE_PAGE *first_page= (KEYCACHE_PAGE *) (first_thread->keycache_link); struct st_my_thread_var *thread; hash_link->file= first_page->file; @@ -1707,7 +1729,7 @@ static void unlink_hash(SIMPLE_KEY_CACHE_CB *keycache, HASH_LINK *hash_link) { KEYCACHE_PAGE *page; thread= next_thread; - page= (KEYCACHE_PAGE *) thread->opt_info; + page= (KEYCACHE_PAGE *) thread->keycache_link; next_thread= thread->next; /* We notify about the event all threads that ask @@ -1796,13 +1818,13 @@ restart: KEYCACHE_DBUG_PRINT("get_hash_link", ("waiting")); page.file= file; page.filepos= filepos; - thread->opt_info= (void *) &page; + thread->keycache_link= (void *) &page; link_into_queue(&keycache->waiting_for_hash_link, thread); KEYCACHE_DBUG_PRINT("get_hash_link: wait", ("suspend thread %ld", thread->id)); keycache_pthread_cond_wait(&thread->suspend, &keycache->cache_lock); - thread->opt_info= NULL; + thread->keycache_link= NULL; goto restart; } hash_link->file= file; @@ -1960,7 +1982,7 @@ restart: for another file/pos. */ thread= my_thread_var; - thread->opt_info= (void *) hash_link; + thread->keycache_link= (void *) hash_link; link_into_queue(&keycache->waiting_for_block, thread); do { @@ -1969,7 +1991,7 @@ restart: keycache_pthread_cond_wait(&thread->suspend, &keycache->cache_lock); } while (thread->next); - thread->opt_info= NULL; + thread->keycache_link= NULL; /* A block should now be assigned to the hash_link. But it may still need to be evicted. Anyway, we should re-check the @@ -2307,7 +2329,7 @@ restart: */ struct st_my_thread_var *thread= my_thread_var; - thread->opt_info= (void *) hash_link; + thread->keycache_link= (void *) hash_link; link_into_queue(&keycache->waiting_for_block, thread); do { @@ -2317,7 +2339,7 @@ restart: &keycache->cache_lock); } while (thread->next); - thread->opt_info= NULL; + thread->keycache_link= NULL; /* Assert that block has a request registered. */ DBUG_ASSERT(hash_link->block->requests); /* Assert that block is not in LRU ring. */ @@ -3901,7 +3923,7 @@ static int flush_key_blocks_int(SIMPLE_KEY_CACHE_CB *keycache, to flush all dirty pages with minimum seek moves */ count= 0; - for (block= keycache->changed_blocks[FILE_HASH(file)] ; + for (block= keycache->changed_blocks[FILE_HASH(file, keycache)] ; block ; block= block->next_changed) { @@ -3934,7 +3956,7 @@ restart: last_in_flush= NULL; last_for_update= NULL; end= (pos= cache)+count; - for (block= keycache->changed_blocks[FILE_HASH(file)] ; + for (block= keycache->changed_blocks[FILE_HASH(file, keycache)] ; block ; block= next) { @@ -4144,10 +4166,10 @@ restart: if (! (type == FLUSH_KEEP || type == FLUSH_FORCE_WRITE)) { - BLOCK_LINK *last_for_update= NULL; BLOCK_LINK *last_in_switch= NULL; uint total_found= 0; uint found; + last_for_update= NULL; /* Finally free all clean blocks for this file. @@ -4156,7 +4178,7 @@ restart: do { found= 0; - for (block= keycache->file_blocks[FILE_HASH(file)] ; + for (block= keycache->file_blocks[FILE_HASH(file, keycache)] ; block ; block= next) { @@ -4397,6 +4419,7 @@ static int flush_all_key_blocks(SIMPLE_KEY_CACHE_CB *keycache) uint total_found; uint found; uint idx; + uint changed_blocks_hash_size= keycache->changed_blocks_hash_size; DBUG_ENTER("flush_all_key_blocks"); do @@ -4412,7 +4435,7 @@ static int flush_all_key_blocks(SIMPLE_KEY_CACHE_CB *keycache) { found= 0; /* Step over the whole changed_blocks hash array. */ - for (idx= 0; idx < CHANGED_BLOCKS_HASH; idx++) + for (idx= 0; idx < changed_blocks_hash_size; idx++) { /* If an array element is non-empty, use the first block from its @@ -4423,7 +4446,7 @@ static int flush_all_key_blocks(SIMPLE_KEY_CACHE_CB *keycache) same hash bucket, one of them will be flushed per iteration of the outer loop of phase 1. */ - if ((block= keycache->changed_blocks[idx])) + while ((block= keycache->changed_blocks[idx])) { found++; /* @@ -4435,7 +4458,6 @@ static int flush_all_key_blocks(SIMPLE_KEY_CACHE_CB *keycache) DBUG_RETURN(1); } } - } while (found); /* @@ -4450,7 +4472,7 @@ static int flush_all_key_blocks(SIMPLE_KEY_CACHE_CB *keycache) { found= 0; /* Step over the whole file_blocks hash array. */ - for (idx= 0; idx < CHANGED_BLOCKS_HASH; idx++) + for (idx= 0; idx < changed_blocks_hash_size; idx++) { /* If an array element is non-empty, use the first block from its @@ -4460,7 +4482,7 @@ static int flush_all_key_blocks(SIMPLE_KEY_CACHE_CB *keycache) same hash bucket, one of them will be flushed per iteration of the outer loop of phase 2. */ - if ((block= keycache->file_blocks[idx])) + while ((block= keycache->file_blocks[idx])) { total_found++; found++; @@ -4469,7 +4491,6 @@ static int flush_all_key_blocks(SIMPLE_KEY_CACHE_CB *keycache) DBUG_RETURN(1); } } - } while (found); /* @@ -4482,7 +4503,7 @@ static int flush_all_key_blocks(SIMPLE_KEY_CACHE_CB *keycache) #ifndef DBUG_OFF /* Now there should not exist any block any more. */ - for (idx= 0; idx < CHANGED_BLOCKS_HASH; idx++) + for (idx= 0; idx < changed_blocks_hash_size; idx++) { DBUG_ASSERT(!keycache->changed_blocks[idx]); DBUG_ASSERT(!keycache->file_blocks[idx]); @@ -4573,7 +4594,7 @@ static void keycache_dump(SIMPLE_KEY_CACHE_CB *keycache) do { thread=thread->next; - page= (KEYCACHE_PAGE *) thread->opt_info; + page= (KEYCACHE_PAGE *) thread->keycache_link; fprintf(keycache_dump_file, "thread:%u, (file,filepos)=(%u,%lu)\n", thread->id,(uint) page->file,(ulong) page->filepos); @@ -4589,7 +4610,7 @@ static void keycache_dump(SIMPLE_KEY_CACHE_CB *keycache) do { thread=thread->next; - hash_link= (HASH_LINK *) thread->opt_info; + hash_link= (HASH_LINK *) thread->keycache_link; fprintf(keycache_dump_file, "thread:%u hash_link:%u (file,filepos)=(%u,%lu)\n", thread->id, (uint) HASH_LINK_NUMBER(hash_link), @@ -5028,15 +5049,18 @@ static SIMPLE_KEY_CACHE_CB age_threshold age threshold (may be zero) DESCRIPTION - This function is the implementation of the init_key_cache interface function - that is employed by partitioned key caches. - The function builds and initializes an array of simple key caches, and then - initializes the control block structure of the type PARTITIONED_KEY_CACHE_CB - that is used for a partitioned key cache. The parameter keycache is - supposed to point to this structure. The number of partitions in the - partitioned key cache to be built must be passed through the field - 'partitions' of this structure. The parameter key_cache_block_size specifies - the size of the blocks in the the simple key caches to be built. + This function is the implementation of the init_key_cache + interface function that is employed by partitioned key caches. + + The function builds and initializes an array of simple key caches, + and then initializes the control block structure of the type + PARTITIONED_KEY_CACHE_CB that is used for a partitioned key + cache. The parameter keycache is supposed to point to this + structure. The number of partitions in the partitioned key cache + to be built must be passed through the field 'partitions' of this + structure. + The parameter key_cache_block_size specifies the size of the + blocks in the the simple key caches to be built. The parameters division_limit and age_threshold determine the initial values of those characteristics of the simple key caches that are used for midpoint insertion strategy. The parameter use_mem specifies the total @@ -5059,7 +5083,7 @@ static int init_partitioned_key_cache(PARTITIONED_KEY_CACHE_CB *keycache, uint key_cache_block_size, size_t use_mem, uint division_limit, - uint age_threshold) + uint age_threshold, uint changed_blocks_hash_size) { int i; size_t mem_per_cache; @@ -5103,7 +5127,8 @@ int init_partitioned_key_cache(PARTITIONED_KEY_CACHE_CB *keycache, } cnt= init_simple_key_cache(partition, key_cache_block_size, mem_per_cache, - division_limit, age_threshold); + division_limit, age_threshold, + changed_blocks_hash_size); if (cnt <= 0) { end_simple_key_cache(partition, 1); @@ -5222,7 +5247,8 @@ static int resize_partitioned_key_cache(PARTITIONED_KEY_CACHE_CB *keycache, uint key_cache_block_size, size_t use_mem, uint division_limit, - uint age_threshold) + uint age_threshold, + uint changed_blocks_hash_size) { uint i; uint partitions= keycache->partitions; @@ -5241,7 +5267,8 @@ int resize_partitioned_key_cache(PARTITIONED_KEY_CACHE_CB *keycache, } if (!err) blocks= init_partitioned_key_cache(keycache, key_cache_block_size, - use_mem, division_limit, age_threshold); + use_mem, division_limit, age_threshold, + changed_blocks_hash_size); if (blocks > 0) { for (i= 0; i < partitions; i++) @@ -5650,7 +5677,7 @@ int flush_partitioned_key_cache_blocks(PARTITIONED_KEY_CACHE_CB *keycache, if ((type == FLUSH_KEEP || type == FLUSH_FORCE_WRITE) && !((*dirty_part_map) & ((ulonglong) 1 << i))) continue; - err|= test(flush_simple_key_cache_blocks(partition, file, 0, type)); + err|= MY_TEST(flush_simple_key_cache_blocks(partition, file, 0, type)); } *dirty_part_map= 0; @@ -5816,6 +5843,7 @@ static int repartition_key_cache_internal(KEY_CACHE *keycache, uint key_cache_block_size, size_t use_mem, uint division_limit, uint age_threshold, + uint changed_blocks_hash_size, uint partitions, my_bool use_op_lock); /* @@ -5828,8 +5856,11 @@ int repartition_key_cache_internal(KEY_CACHE *keycache, use_mem total memory to use for cache buffers/structures division_limit division limit (may be zero) age_threshold age threshold (may be zero) - partitions number of partitions in the key cache - use_op_lock if TRUE use keycache->op_lock, otherwise - ignore it + changed_blocks_hash_size Number of hash buckets to hold a link of different + files. Should be proportional to number of different + files sused. + partitions Number of partitions in the key cache + use_op_lock if TRUE use keycache->op_lock, otherwise - ignore it DESCRIPTION The function performs the actions required from init_key_cache(). @@ -5850,7 +5881,8 @@ int repartition_key_cache_internal(KEY_CACHE *keycache, static int init_key_cache_internal(KEY_CACHE *keycache, uint key_cache_block_size, size_t use_mem, uint division_limit, - uint age_threshold, uint partitions, + uint age_threshold, uint changed_blocks_hash_size, + uint partitions, my_bool use_op_lock) { void *keycache_cb; @@ -5901,7 +5933,7 @@ int init_key_cache_internal(KEY_CACHE *keycache, uint key_cache_block_size, keycache->can_be_used= 0; blocks= keycache->interface_funcs->init(keycache_cb, key_cache_block_size, use_mem, division_limit, - age_threshold); + age_threshold, changed_blocks_hash_size); keycache->partitions= partitions ? ((PARTITIONED_KEY_CACHE_CB *) keycache_cb)->partitions : 0; @@ -5956,10 +5988,12 @@ int init_key_cache_internal(KEY_CACHE *keycache, uint key_cache_block_size, int init_key_cache(KEY_CACHE *keycache, uint key_cache_block_size, size_t use_mem, uint division_limit, - uint age_threshold, uint partitions) + uint age_threshold, uint changed_blocks_hash_size, + uint partitions) { return init_key_cache_internal(keycache, key_cache_block_size, use_mem, - division_limit, age_threshold, partitions, 1); + division_limit, age_threshold, + changed_blocks_hash_size, partitions, 1); } @@ -5998,7 +6032,8 @@ int init_key_cache(KEY_CACHE *keycache, uint key_cache_block_size, */ int resize_key_cache(KEY_CACHE *keycache, uint key_cache_block_size, - size_t use_mem, uint division_limit, uint age_threshold) + size_t use_mem, uint division_limit, uint age_threshold, + uint changed_blocks_hash_size) { int blocks= -1; if (keycache->key_cache_inited) @@ -6008,6 +6043,7 @@ int resize_key_cache(KEY_CACHE *keycache, uint key_cache_block_size, blocks= repartition_key_cache_internal(keycache, key_cache_block_size, use_mem, division_limit, age_threshold, + changed_blocks_hash_size, (uint) keycache->param_partitions, 0); else @@ -6015,7 +6051,8 @@ int resize_key_cache(KEY_CACHE *keycache, uint key_cache_block_size, blocks= keycache->interface_funcs->resize(keycache->keycache_cb, key_cache_block_size, use_mem, division_limit, - age_threshold); + age_threshold, + changed_blocks_hash_size); if (keycache->partitions) keycache->partitions= @@ -6453,6 +6490,7 @@ static int repartition_key_cache_internal(KEY_CACHE *keycache, uint key_cache_block_size, size_t use_mem, uint division_limit, uint age_threshold, + uint changed_blocks_hash_size, uint partitions, my_bool use_op_lock) { uint blocks= -1; @@ -6462,10 +6500,12 @@ int repartition_key_cache_internal(KEY_CACHE *keycache, pthread_mutex_lock(&keycache->op_lock); keycache->interface_funcs->resize(keycache->keycache_cb, key_cache_block_size, 0, - division_limit, age_threshold); + division_limit, age_threshold, + changed_blocks_hash_size); end_key_cache_internal(keycache, 1, 0); blocks= init_key_cache_internal(keycache, key_cache_block_size, use_mem, - division_limit, age_threshold, partitions, + division_limit, age_threshold, + changed_blocks_hash_size, partitions, 0); if (use_op_lock) pthread_mutex_unlock(&keycache->op_lock); @@ -6510,10 +6550,12 @@ int repartition_key_cache_internal(KEY_CACHE *keycache, int repartition_key_cache(KEY_CACHE *keycache, uint key_cache_block_size, size_t use_mem, uint division_limit, - uint age_threshold, uint partitions) + uint age_threshold, uint changed_blocks_hash_size, + uint partitions) { return repartition_key_cache_internal(keycache, key_cache_block_size, use_mem, division_limit, age_threshold, + changed_blocks_hash_size, partitions, 1); } diff --git a/mysys/mf_radix.c b/mysys/mf_radix.c index b0ed419e6c1..11c4ac45a93 100644 --- a/mysys/mf_radix.c +++ b/mysys/mf_radix.c @@ -26,6 +26,11 @@ /* Radixsort */ +my_bool radixsort_is_appliccable(uint n_items, size_t size_of_element) +{ + return size_of_element <= 20 && n_items >= 1000 && n_items < 100000; +} + void radixsort_for_str_ptr(uchar **base, uint number_of_elements, size_t size_of_element, uchar **buffer) { uchar **end,**ptr,**buffer_ptr; diff --git a/mysys/mf_sort.c b/mysys/mf_sort.c index 9ef02787716..b2c58c26624 100644 --- a/mysys/mf_sort.c +++ b/mysys/mf_sort.c @@ -23,7 +23,7 @@ void my_string_ptr_sort(uchar *base, uint items, size_t size) #if INT_MAX > 65536L uchar **ptr=0; - if (size <= 20 && items >= 1000 && items < 100000 && + if (radixsort_is_appliccable(items, size) && (ptr= (uchar**) my_malloc(items*sizeof(char*),MYF(0)))) { radixsort_for_str_ptr((uchar**) base,items,size,ptr); diff --git a/mysys/mf_tempdir.c b/mysys/mf_tempdir.c index 34b469b4066..2fbbedc4e89 100644 --- a/mysys/mf_tempdir.c +++ b/mysys/mf_tempdir.c @@ -30,7 +30,7 @@ my_bool init_tmpdir(MY_TMPDIR *tmpdir, const char *pathlist) DBUG_PRINT("enter", ("pathlist: %s", pathlist ? pathlist : "NULL")); mysql_mutex_init(key_TMPDIR_mutex, &tmpdir->mutex, MY_MUTEX_INIT_FAST); - if (my_init_dynamic_array(&tmpdir->full_list, sizeof(char*), 1, 5)) + if (my_init_dynamic_array(&tmpdir->full_list, sizeof(char*), 1, 5, MYF(0))) goto err; if (!pathlist || !pathlist[0]) { diff --git a/mysys/mulalloc.c b/mysys/mulalloc.c index 9384ed744ad..fceecdc1dc7 100644 --- a/mysys/mulalloc.c +++ b/mysys/mulalloc.c @@ -62,3 +62,47 @@ void* my_multi_malloc(myf myFlags, ...) va_end(args); DBUG_RETURN((void*) start); } + + +/* + Same as my_multi_malloc, but each entry can be over 4G + + SYNOPSIS + my_multi_malloc() + myFlags Flags + ptr1, length1 Multiple arguments terminated by null ptr + ptr2, length2 ... + ... + NULL +*/ + +void *my_multi_malloc_large(myf myFlags, ...) +{ + va_list args; + char **ptr,*start,*res; + size_t tot_length,length; + DBUG_ENTER("my_multi_malloc"); + + va_start(args,myFlags); + tot_length=0; + while ((ptr=va_arg(args, char **))) + { + length=va_arg(args,ulonglong); + tot_length+=ALIGN_SIZE(length); + } + va_end(args); + + if (!(start=(char *) my_malloc(tot_length, myFlags))) + DBUG_RETURN(0); /* purecov: inspected */ + + va_start(args,myFlags); + res=start; + while ((ptr=va_arg(args, char **))) + { + *ptr=res; + length=va_arg(args,ulonglong); + res+=ALIGN_SIZE(length); + } + va_end(args); + DBUG_RETURN((void*) start); +} diff --git a/mysys/my_access.c b/mysys/my_access.c index 22f145a75c9..75774240406 100644 --- a/mysys/my_access.c +++ b/mysys/my_access.c @@ -150,11 +150,80 @@ int check_if_legal_tablename(const char *name) } +#ifdef __WIN__ +/** + Checks if the drive letter supplied is valid or not. Valid drive + letters are A to Z, both lower case and upper case. + + @param drive_letter : The drive letter to validate. + + @return TRUE if the drive exists, FALSE otherwise. +*/ +static my_bool does_drive_exists(char drive_letter) +{ + DWORD drive_mask= GetLogicalDrives(); + drive_letter= toupper(drive_letter); + + return (drive_letter >= 'A' && drive_letter <= 'Z') && + (drive_mask & (0x1 << (drive_letter - 'A'))); +} + +/** + Verifies if the file name supplied is allowed or not. On Windows + file names with a colon (:) are not allowed because such file names + store data in Alternate Data Streams which can be used to hide + the data. + Apart from colon, other characters that are not allowed in filenames + on Windows are greater/less sign, double quotes, forward slash, backslash, + pipe and star characters. + + See MSDN documentation on filename restrictions. + + @param name contains the file name with or without path + @param length contains the length of file name + @param allow_current_dir TRUE if paths like C:foobar are allowed, + FALSE otherwise + + @return TRUE if the file name is allowed, FALSE otherwise. +*/ +#define ILLEGAL_FILENAME_CHARS "<>:\"/\\|?*" + +my_bool is_filename_allowed(const char *name __attribute__((unused)), + size_t length __attribute__((unused)), + my_bool allow_current_dir __attribute__((unused))) +{ + /* + For Windows, check if the file name contains : character. + Start from end of path and search if the file name contains : + */ + const char* ch = NULL; + for (ch= name + length - 1; ch >= name; --ch) + { + if (FN_LIBCHAR == *ch || '/' == *ch) + break; + else if (':' == *ch) + { + /* + File names like C:foobar.txt are allowed since the syntax means + file foobar.txt in current directory of C drive. However file + names likes CC:foobar are not allowed since this syntax means ADS + foobar in file CC. + */ + return (allow_current_dir && (ch - name == 1) && + does_drive_exists(*name)); + } + else if (strchr(ILLEGAL_FILENAME_CHARS, *ch)) + return FALSE; + } + return TRUE; +} /* is_filename_allowed */ +#endif /* __WIN__ */ + #if defined(__WIN__) || defined(__EMX__) /* - Check if a path will access a reserverd file name that may cause problems + Check if a path will access a reserved file name that may cause problems SYNOPSIS check_if_legal_filename @@ -171,6 +240,9 @@ int check_if_legal_filename(const char *path) const char **reserved_name; DBUG_ENTER("check_if_legal_filename"); + if (!is_filename_allowed(path, strlen(path), TRUE)) + DBUG_RETURN(1); + path+= dirname_length(path); /* To start of filename */ if (!(end= strchr(path, FN_EXTCHAR))) end= strend(path); diff --git a/mysys/my_addr_resolve.c b/mysys/my_addr_resolve.c index 02f71fd72bd..b82792fc4b5 100644 --- a/mysys/my_addr_resolve.c +++ b/mysys/my_addr_resolve.c @@ -126,8 +126,8 @@ err: */ #elif defined(MY_ADDR_RESOLVE_FORK) /* - yet another - just execute addr2line or eu-addr2line, whatever available, - pipe the addresses to it, and parse the output + yet another - just execute addr2line pipe the addresses to it, and parse the + output */ #include <m_string.h> @@ -181,9 +181,19 @@ int start_addr2line_fork(const char *binary_path) int my_addr_resolve(void *ptr, my_addr_loc *loc) { - char input[32], *s; + char input[32]; size_t len; + ssize_t total_bytes_read = 0; + ssize_t extra_bytes_read = 0; + ssize_t parsed = 0; + + fd_set set; + struct timeval timeout; + + int filename_start = -1; + int line_number_start = -1; + Dl_info info; void *offset; @@ -207,24 +217,58 @@ int my_addr_resolve(void *ptr, my_addr_loc *loc) len= my_snprintf(input, sizeof(input), "%08x\n", (ulonglong)(ptr - offset)); if (write(in[1], input, len) <= 0) return 1; - if (read(out[0], output, sizeof(output)) <= 0) - return 1; - loc->func= s= output; - while (*s != '\n') - s++; - *s++= 0; - loc->file= s; - while (*s != ':') - s++; - *s++= 0; + FD_ZERO(&set); + FD_SET(out[0], &set); + + /* 100 ms should be plenty of time for addr2line to issue a response. */ + timeout.tv_sec = 0; + timeout.tv_usec = 100000; + /* Read in a loop till all the output from addr2line is complete. */ + while (parsed == total_bytes_read && + select(out[0] + 1, &set, NULL, NULL, &timeout) > 0) + { + extra_bytes_read= read(out[0], output + total_bytes_read, + sizeof(output) - total_bytes_read); + if (extra_bytes_read < 0) + return 1; + /* Timeout or max bytes read. */ + if (extra_bytes_read == 0) + break; + + total_bytes_read += extra_bytes_read; + + /* Go through the addr2line response and get the required data. + The response is structured in 2 lines. The first line contains the function + name, while the second one contains <filename>:<line number> */ + for (; parsed < total_bytes_read; parsed++) + { + if (output[parsed] == '\n') + { + filename_start = parsed + 1; + output[parsed] = '\0'; + } + if (filename_start != -1 && output[parsed] == ':') + { + line_number_start = parsed + 1; + output[parsed] = '\0'; + break; + } + } + } + + /* Response is malformed. */ + if (filename_start == -1 || line_number_start == -1) + return 1; + + loc->func= output; + loc->file= output + filename_start; + loc->line= atoi(output + line_number_start); + + /* Addr2line was unable to extract any meaningful information. */ if (strcmp(loc->file, "??") == 0) return 1; - loc->line= 0; - while (isdigit(*s)) - loc->line = loc->line * 10 + (*s++ - '0'); - *s = 0; loc->file= strip_path(loc->file); return 0; diff --git a/mysys/my_aes.c b/mysys/my_aes.c deleted file mode 100644 index 7074f700413..00000000000 --- a/mysys/my_aes.c +++ /dev/null @@ -1,228 +0,0 @@ -/* Copyright (c) 2002, 2006 MySQL AB - Use is subject to license terms - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; version 2 of the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA */ - - -/* - Implementation of AES Encryption for MySQL - Initial version by Peter Zaitsev June 2002 -*/ - - -#include <my_global.h> -#include <m_string.h> -#include "my_aes.h" - -enum encrypt_dir { AES_ENCRYPT, AES_DECRYPT }; - -#define AES_BLOCK_SIZE 16 /* Block size in bytes */ - -#define AES_BAD_DATA -1 /* If bad data discovered during decoding */ - - -/* The structure for key information */ -typedef struct { - int nr; /* Number of rounds */ - uint32 rk[4*(AES_MAXNR + 1)]; /* key schedule */ -} KEYINSTANCE; - - -/* - This is internal function just keeps joint code of Key generation - - SYNOPSIS - my_aes_create_key() - aes_key Address of Key Instance to be created - direction Direction (are we encoding or decoding) - key Key to use for real key creation - key_length Length of the key - - DESCRIPTION - - RESULT - 0 ok - -1 Error Note: The current impementation never returns this -*/ - -static int my_aes_create_key(KEYINSTANCE *aes_key, - enum encrypt_dir direction, const char *key, - int key_length) -{ - uint8 rkey[AES_KEY_LENGTH/8]; /* The real key to be used for encryption */ - uint8 *rkey_end=rkey+AES_KEY_LENGTH/8; /* Real key boundary */ - uint8 *ptr; /* Start of the real key*/ - const char *sptr; /* Start of the working key */ - const char *key_end=key+key_length; /* Working key boundary*/ - - bzero((char*) rkey,AES_KEY_LENGTH/8); /* Set initial key */ - - for (ptr= rkey, sptr= key; sptr < key_end; ptr++,sptr++) - { - if (ptr == rkey_end) - ptr= rkey; /* Just loop over tmp_key until we used all key */ - *ptr^= (uint8) *sptr; - } -#ifdef AES_USE_KEY_BITS - /* - This block is intended to allow more weak encryption if application - build with libmysqld needs to correspond to export regulations - It should be never used in normal distribution as does not give - any speed improvement. - To get worse security define AES_USE_KEY_BITS to number of bits - you want key to be. It should be divisible by 8 - - WARNING: Changing this value results in changing of enryption for - all key lengths so altering this value will result in impossibility - to decrypt data encrypted with previous value - */ -#define AES_USE_KEY_BYTES (AES_USE_KEY_BITS/8) - /* - To get weaker key we use first AES_USE_KEY_BYTES bytes of created key - and cyclically copy them until we created all required key length - */ - for (ptr= rkey+AES_USE_KEY_BYTES, sptr=rkey ; ptr < rkey_end; - ptr++,sptr++) - { - if (sptr == rkey+AES_USE_KEY_BYTES) - sptr=rkey; - *ptr=*sptr; - } -#endif - if (direction == AES_DECRYPT) - aes_key->nr = rijndaelKeySetupDec(aes_key->rk, rkey, AES_KEY_LENGTH); - else - aes_key->nr = rijndaelKeySetupEnc(aes_key->rk, rkey, AES_KEY_LENGTH); - return 0; -} - - -/* - Crypt buffer with AES encryption algorithm. - - SYNOPSIS - my_aes_encrypt() - source Pointer to data for encryption - source_length Size of encryption data - dest Buffer to place encrypted data (must be large enough) - key Key to be used for encryption - key_length Length of the key. Will handle keys of any length - - RETURN - >= 0 Size of encrypted data - < 0 Error -*/ - -int my_aes_encrypt(const char* source, int source_length, char* dest, - const char* key, int key_length) -{ - KEYINSTANCE aes_key; - uint8 block[AES_BLOCK_SIZE]; /* 128 bit block used for padding */ - int rc; /* result codes */ - int num_blocks; /* number of complete blocks */ - char pad_len; /* pad size for the last block */ - int i; - - if ((rc= my_aes_create_key(&aes_key,AES_ENCRYPT,key,key_length))) - return rc; - - num_blocks = source_length/AES_BLOCK_SIZE; - - for (i = num_blocks; i > 0; i--) /* Encode complete blocks */ - { - rijndaelEncrypt(aes_key.rk, aes_key.nr, (const uint8*) source, - (uint8*) dest); - source+= AES_BLOCK_SIZE; - dest+= AES_BLOCK_SIZE; - } - - /* Encode the rest. We always have incomplete block */ - pad_len = AES_BLOCK_SIZE - (source_length - AES_BLOCK_SIZE*num_blocks); - memcpy(block, source, 16 - pad_len); - bfill(block + AES_BLOCK_SIZE - pad_len, pad_len, pad_len); - rijndaelEncrypt(aes_key.rk, aes_key.nr, block, (uint8*) dest); - return AES_BLOCK_SIZE*(num_blocks + 1); -} - - -/* - DeCrypt buffer with AES encryption algorithm. - - SYNOPSIS - my_aes_decrypt() - source Pointer to data for decryption - source_length Size of encrypted data - dest Buffer to place decrypted data (must be large enough) - key Key to be used for decryption - key_length Length of the key. Will handle keys of any length - - RETURN - >= 0 Size of encrypted data - < 0 Error -*/ - -int my_aes_decrypt(const char *source, int source_length, char *dest, - const char *key, int key_length) -{ - KEYINSTANCE aes_key; - uint8 block[AES_BLOCK_SIZE]; /* 128 bit block used for padding */ - int rc; /* Result codes */ - int num_blocks; /* Number of complete blocks */ - uint pad_len; /* Pad size for the last block */ - int i; - - if ((rc=my_aes_create_key(&aes_key,AES_DECRYPT,key,key_length))) - return rc; - - num_blocks = source_length/AES_BLOCK_SIZE; - - if ((source_length != num_blocks*AES_BLOCK_SIZE) || num_blocks ==0 ) - return AES_BAD_DATA; /* Input size has to be even and at least one block */ - - for (i = num_blocks-1; i > 0; i--) /* Decode all but last blocks */ - { - rijndaelDecrypt(aes_key.rk, aes_key.nr, (const uint8*) source, - (uint8*) dest); - source+= AES_BLOCK_SIZE; - dest+= AES_BLOCK_SIZE; - } - - rijndaelDecrypt(aes_key.rk, aes_key.nr, (const uint8*) source, block); - /* Use last char in the block as size */ - pad_len = (uint) (uchar) block[AES_BLOCK_SIZE-1]; - - if (pad_len > AES_BLOCK_SIZE) - return AES_BAD_DATA; - /* We could also check whole padding but we do not really need this */ - - memcpy(dest, block, AES_BLOCK_SIZE - pad_len); - return AES_BLOCK_SIZE*num_blocks - pad_len; -} - - -/* - Get size of buffer which will be large enough for encrypted data - - SYNOPSIS - my_aes_get_size() - source_length Length of data to be encrypted - - RETURN - Size of buffer required to store encrypted data -*/ - -int my_aes_get_size(int source_length) -{ - return AES_BLOCK_SIZE*(source_length/AES_BLOCK_SIZE)+AES_BLOCK_SIZE; -} diff --git a/mysys/my_alloc.c b/mysys/my_alloc.c index e727f46be6d..276ad080520 100644 --- a/mysys/my_alloc.c +++ b/mysys/my_alloc.c @@ -22,6 +22,10 @@ #undef EXTRA_DEBUG #define EXTRA_DEBUG +/* data packed in MEM_ROOT -> min_malloc */ + +#define MALLOC_FLAG(A) ((A & 1) ? MY_THREAD_SPECIFIC : 0) + #define TRASH_MEM(X) TRASH_FREE(((char*)(X) + ((X)->size-(X)->left)), (X)->left) /* @@ -36,24 +40,33 @@ should be no less than ALLOC_ROOT_MIN_BLOCK_SIZE) pre_alloc_size - if non-0, then size of block that should be pre-allocated during memory root initialization. + my_flags MY_THREAD_SPECIFIC flag for my_malloc DESCRIPTION This function prepares memory root for further use, sets initial size of chunk for memory allocation and pre-allocates first block if specified. - Altough error can happen during execution of this function if + Although error can happen during execution of this function if pre_alloc_size is non-0 it won't be reported. Instead it will be reported as error in first alloc_root() on this memory root. + + We don't want to change the structure size for MEM_ROOT. + Because of this, we store in MY_THREAD_SPECIFIC as bit 1 in block_size */ void init_alloc_root(MEM_ROOT *mem_root, size_t block_size, - size_t pre_alloc_size __attribute__((unused))) + size_t pre_alloc_size __attribute__((unused)), + myf my_flags) { DBUG_ENTER("init_alloc_root"); - DBUG_PRINT("enter",("root: 0x%lx", (long) mem_root)); + DBUG_PRINT("enter",("root: %p prealloc: %zu", mem_root, + pre_alloc_size)); mem_root->free= mem_root->used= mem_root->pre_alloc= 0; mem_root->min_malloc= 32; - mem_root->block_size= block_size - ALLOC_ROOT_MIN_BLOCK_SIZE; + mem_root->block_size= (block_size - ALLOC_ROOT_MIN_BLOCK_SIZE) & ~1; + if (MY_TEST(my_flags & MY_THREAD_SPECIFIC)) + mem_root->block_size|= 1; + mem_root->error_handler= 0; mem_root->block_num= 4; /* We shift this with >>2 */ mem_root->first_block_usage= 0; @@ -62,8 +75,8 @@ void init_alloc_root(MEM_ROOT *mem_root, size_t block_size, if (pre_alloc_size) { if ((mem_root->free= mem_root->pre_alloc= - (USED_MEM*) my_malloc(pre_alloc_size + ALIGN_SIZE(sizeof(USED_MEM)), - MYF(0)))) + (USED_MEM*) my_malloc(pre_alloc_size + ALIGN_SIZE(sizeof(USED_MEM)), + MYF(my_flags)))) { mem_root->free->size= pre_alloc_size+ALIGN_SIZE(sizeof(USED_MEM)); mem_root->free->left= pre_alloc_size; @@ -75,7 +88,6 @@ void init_alloc_root(MEM_ROOT *mem_root, size_t block_size, DBUG_VOID_RETURN; } - /* SYNOPSIS reset_root_defaults() @@ -96,9 +108,11 @@ void init_alloc_root(MEM_ROOT *mem_root, size_t block_size, void reset_root_defaults(MEM_ROOT *mem_root, size_t block_size, size_t pre_alloc_size __attribute__((unused))) { + DBUG_ENTER("reset_root_defaults"); DBUG_ASSERT(alloc_root_inited(mem_root)); - mem_root->block_size= block_size - ALLOC_ROOT_MIN_BLOCK_SIZE; + mem_root->block_size= (((block_size - ALLOC_ROOT_MIN_BLOCK_SIZE) & ~1) | + (mem_root->block_size & 1)); #if !(defined(HAVE_valgrind) && defined(EXTRA_DEBUG)) if (pre_alloc_size) { @@ -117,7 +131,7 @@ void reset_root_defaults(MEM_ROOT *mem_root, size_t block_size, { /* We found a suitable block, no need to do anything else */ mem_root->pre_alloc= mem; - return; + DBUG_VOID_RETURN; } if (mem->left + ALIGN_SIZE(sizeof(USED_MEM)) == mem->size) { @@ -129,7 +143,9 @@ void reset_root_defaults(MEM_ROOT *mem_root, size_t block_size, prev= &mem->next; } /* Allocate new prealloc block and add it to the end of free list */ - if ((mem= (USED_MEM *) my_malloc(size, MYF(0)))) + if ((mem= (USED_MEM *) my_malloc(size, + MYF(MALLOC_FLAG(mem_root-> + block_size))))) { mem->size= size; mem->left= pre_alloc_size; @@ -146,6 +162,8 @@ void reset_root_defaults(MEM_ROOT *mem_root, size_t block_size, else #endif mem_root->pre_alloc= 0; + + DBUG_VOID_RETURN; } @@ -154,7 +172,7 @@ void *alloc_root(MEM_ROOT *mem_root, size_t length) #if defined(HAVE_valgrind) && defined(EXTRA_DEBUG) reg1 USED_MEM *next; DBUG_ENTER("alloc_root"); - DBUG_PRINT("enter",("root: 0x%lx", (long) mem_root)); + DBUG_PRINT("enter",("root: %p", mem_root)); DBUG_ASSERT(alloc_root_inited(mem_root)); @@ -167,7 +185,9 @@ void *alloc_root(MEM_ROOT *mem_root, size_t length) }); length+=ALIGN_SIZE(sizeof(USED_MEM)); - if (!(next = (USED_MEM*) my_malloc(length,MYF(MY_WME | ME_FATALERROR)))) + if (!(next = (USED_MEM*) my_malloc(length, + MYF(MY_WME | ME_FATALERROR | + MALLOC_FLAG(mem_root->block_size))))) { if (mem_root->error_handler) (*mem_root->error_handler)(); @@ -176,8 +196,8 @@ void *alloc_root(MEM_ROOT *mem_root, size_t length) next->next= mem_root->used; next->size= length; mem_root->used= next; - DBUG_PRINT("exit",("ptr: 0x%lx", (long) (((char*) next)+ - ALIGN_SIZE(sizeof(USED_MEM))))); + DBUG_PRINT("exit",("ptr: %p", (((char*) next)+ + ALIGN_SIZE(sizeof(USED_MEM))))); DBUG_RETURN((uchar*) (((char*) next)+ALIGN_SIZE(sizeof(USED_MEM)))); #else size_t get_size, block_size; @@ -186,7 +206,7 @@ void *alloc_root(MEM_ROOT *mem_root, size_t length) reg2 USED_MEM **prev; size_t original_length = length; DBUG_ENTER("alloc_root"); - DBUG_PRINT("enter",("root: 0x%lx", (long) mem_root)); + DBUG_PRINT("enter",("root: %p", mem_root)); DBUG_ASSERT(alloc_root_inited(mem_root)); DBUG_EXECUTE_IF("simulate_out_of_memory", @@ -215,11 +235,14 @@ void *alloc_root(MEM_ROOT *mem_root, size_t length) } if (! next) { /* Time to alloc new block */ - block_size= mem_root->block_size * (mem_root->block_num >> 2); + block_size= (mem_root->block_size & ~1) * (mem_root->block_num >> 2); get_size= length+ALIGN_SIZE(sizeof(USED_MEM)); - get_size= max(get_size, block_size); + get_size= MY_MAX(get_size, block_size); - if (!(next = (USED_MEM*) my_malloc(get_size,MYF(MY_WME | ME_FATALERROR)))) + if (!(next = (USED_MEM*) my_malloc(get_size, + MYF(MY_WME | ME_FATALERROR | + MALLOC_FLAG(mem_root-> + block_size))))) { if (mem_root->error_handler) (*mem_root->error_handler)(); @@ -243,7 +266,7 @@ void *alloc_root(MEM_ROOT *mem_root, size_t length) mem_root->first_block_usage= 0; } TRASH_ALLOC(point, original_length); - DBUG_PRINT("exit",("ptr: 0x%lx", (ulong) point)); + DBUG_PRINT("exit",("ptr: %p", point)); DBUG_RETURN((void*) point); #endif } @@ -353,7 +376,7 @@ void free_root(MEM_ROOT *root, myf MyFlags) { reg1 USED_MEM *next,*old; DBUG_ENTER("free_root"); - DBUG_PRINT("enter",("root: 0x%lx flags: %u", (long) root, (uint) MyFlags)); + DBUG_PRINT("enter",("root: %p flags: %u", root, (uint) MyFlags)); if (MyFlags & MY_MARK_BLOCKS_FREE) { diff --git a/mysys/my_bitmap.c b/mysys/my_bitmap.c index 851fe2b1026..0eaf1a88aa1 100644 --- a/mysys/my_bitmap.c +++ b/mysys/my_bitmap.c @@ -147,10 +147,30 @@ static inline void bitmap_unlock(MY_BITMAP *map __attribute__((unused))) } -my_bool bitmap_init(MY_BITMAP *map, my_bitmap_map *buf, uint n_bits, - my_bool thread_safe __attribute__((unused))) +static inline uint get_first_set(my_bitmap_map value, uint word_pos) { - DBUG_ENTER("bitmap_init"); + uchar *byte_ptr= (uchar*)&value; + uchar byte_value; + uint byte_pos, bit_pos; + + DBUG_ASSERT(value); + for (byte_pos=0; ; byte_pos++, byte_ptr++) + { + if ((byte_value= *byte_ptr)) + { + for (bit_pos=0; ; bit_pos++) + if (byte_value & (1 << bit_pos)) + return (word_pos*32) + (byte_pos*8) + bit_pos; + } + } + return MY_BIT_NONE; /* Impossible */ +} + + +my_bool my_bitmap_init(MY_BITMAP *map, my_bitmap_map *buf, uint n_bits, + my_bool thread_safe __attribute__((unused))) +{ + DBUG_ENTER("my_bitmap_init"); if (!buf) { uint size_in_bytes= bitmap_buffer_size(n_bits); @@ -182,9 +202,9 @@ my_bool bitmap_init(MY_BITMAP *map, my_bitmap_map *buf, uint n_bits, } -void bitmap_free(MY_BITMAP *map) +void my_bitmap_free(MY_BITMAP *map) { - DBUG_ENTER("bitmap_free"); + DBUG_ENTER("my_bitmap_free"); if (map->bitmap) { if (map->mutex) @@ -286,6 +306,12 @@ uint bitmap_set_next(MY_BITMAP *map) } +/** + Set the specified number of bits in the bitmap buffer. + + @param map [IN] Bitmap + @param prefix_size [IN] Number of bits to be set +*/ void bitmap_set_prefix(MY_BITMAP *map, uint prefix_size) { uint prefix_bytes, prefix_bits, d; @@ -299,11 +325,12 @@ void bitmap_set_prefix(MY_BITMAP *map, uint prefix_size) m+= prefix_bytes; if ((prefix_bits= prefix_size & 7)) { - *m++= (1 << prefix_bits)-1; - prefix_bytes++; + *(m++)= (1 << prefix_bits)-1; + // As the prefix bits are set, lets count this byte too as a prefix byte. + prefix_bytes ++; } if ((d= no_bytes_in_map(map)-prefix_bytes)) - bzero(m, d); + memset(m, 0, d); } @@ -353,6 +380,7 @@ my_bool bitmap_is_clear_all(const MY_BITMAP *map) my_bitmap_map *data_ptr= map->bitmap; my_bitmap_map *end= map->last_word_ptr; + DBUG_ASSERT(map->n_bits > 0); for (; data_ptr < end; data_ptr++) if (*data_ptr) return FALSE; @@ -405,7 +433,7 @@ void bitmap_intersect(MY_BITMAP *map, const MY_BITMAP *map2) DBUG_ASSERT(map->bitmap && map2->bitmap); - end= to+min(len,len2); + end= to+MY_MIN(len,len2); while (to < end) *to++ &= *from++; @@ -597,12 +625,10 @@ void bitmap_copy(MY_BITMAP *map, const MY_BITMAP *map2) uint bitmap_get_first_set(const MY_BITMAP *map) { - uchar *byte_ptr; - uint i,j,k; - my_bitmap_map *data_ptr, *end= map->last_word_ptr; + uint i; + my_bitmap_map *data_ptr= map->bitmap, *end= map->last_word_ptr; DBUG_ASSERT(map->bitmap); - data_ptr= map->bitmap; for (i=0; data_ptr < end; data_ptr++, i++) if (*data_ptr) @@ -611,25 +637,66 @@ uint bitmap_get_first_set(const MY_BITMAP *map) return MY_BIT_NONE; found: + return get_first_set(*data_ptr, i); +} + + +/** + Get the next set bit. + + @param map Bitmap + @param bitmap_bit Bit to start search from + + @return Index to first bit set after bitmap_bit +*/ + +uint bitmap_get_next_set(const MY_BITMAP *map, uint bitmap_bit) +{ + uint word_pos, byte_to_mask, i; + union { my_bitmap_map bitmap ; uchar bitmap_buff[sizeof(my_bitmap_map)]; } + first_word; + uchar *ptr= &first_word.bitmap_buff[0]; + my_bitmap_map *data_ptr, *end= map->last_word_ptr; + + DBUG_ASSERT(map->bitmap); + + /* Look for the next bit */ + bitmap_bit++; + if (bitmap_bit >= map->n_bits) + return MY_BIT_NONE; + word_pos= bitmap_bit / 32; + data_ptr= map->bitmap + word_pos; + first_word.bitmap= *data_ptr; + + /* Mask out previous bits from first_word */ + byte_to_mask= (bitmap_bit % 32) / 8; + for (i= 0; i < byte_to_mask; i++) + ptr[i]= 0; + ptr[byte_to_mask]&= 0xFFU << (bitmap_bit & 7); + + if (data_ptr == end) { - byte_ptr= (uchar*)data_ptr; - for (j=0; ; j++, byte_ptr++) - { - if (*byte_ptr) - { - for (k=0; ; k++) - { - if (*byte_ptr & (1 << k)) - return (i*32) + (j*8) + k; - } - } - } + if (first_word.bitmap & ~map->last_word_mask) + return get_first_set(first_word.bitmap, word_pos); + else + return MY_BIT_NONE; } - DBUG_ASSERT(0); - return MY_BIT_NONE; /* Impossible */ + + if (first_word.bitmap) + return get_first_set(first_word.bitmap, word_pos); + + for (data_ptr++, word_pos++; data_ptr < end; data_ptr++, word_pos++) + if (*data_ptr) + return get_first_set(*data_ptr, word_pos); + + if (!(*end & ~map->last_word_mask)) + return MY_BIT_NONE; + return get_first_set(*end, word_pos); } +/* Get first free bit */ + uint bitmap_get_first(const MY_BITMAP *map) { uchar *byte_ptr; @@ -647,17 +714,15 @@ uint bitmap_get_first(const MY_BITMAP *map) return MY_BIT_NONE; found: + byte_ptr= (uchar*)data_ptr; + for (j=0; ; j++, byte_ptr++) { - byte_ptr= (uchar*)data_ptr; - for (j=0; ; j++, byte_ptr++) + if (*byte_ptr != 0xFF) { - if (*byte_ptr != 0xFF) + for (k=0; ; k++) { - for (k=0; ; k++) - { - if (!(*byte_ptr & (1 << k))) - return (i*32) + (j*8) + k; - } + if (!(*byte_ptr & (1 << k))) + return (i*32) + (j*8) + k; } } } diff --git a/mysys/my_chmod.c b/mysys/my_chmod.c index c60957ae38e..3d0863936a7 100644 --- a/mysys/my_chmod.c +++ b/mysys/my_chmod.c @@ -34,7 +34,7 @@ int my_chmod(const char *name, mode_t mode, myf my_flags) { DBUG_ENTER("my_chmod"); - DBUG_PRINT("my",("name: %s mode: %lu flags: %d", name, (ulong) mode, + DBUG_PRINT("my",("name: %s mode: %lu flags: %lu", name, (ulong) mode, my_flags)); if (chmod(name, mode)) diff --git a/mysys/my_chsize.c b/mysys/my_chsize.c index 63964916d6f..51da6be7935 100644 --- a/mysys/my_chsize.c +++ b/mysys/my_chsize.c @@ -43,7 +43,7 @@ int my_chsize(File fd, my_off_t newlength, int filler, myf MyFlags) my_off_t oldsize; uchar buff[IO_SIZE]; DBUG_ENTER("my_chsize"); - DBUG_PRINT("my",("fd: %d length: %lu MyFlags: %d",fd,(ulong) newlength, + DBUG_PRINT("my",("fd: %d length: %lu MyFlags: %lu",fd,(ulong) newlength, MyFlags)); if ((oldsize= my_seek(fd, 0L, MY_SEEK_END, MYF(MY_WME+MY_FAE))) == newlength) diff --git a/mysys/my_compare.c b/mysys/my_compare.c index 82b30ab3ed3..7fb2ef97916 100644 --- a/mysys/my_compare.c +++ b/mysys/my_compare.c @@ -36,7 +36,7 @@ static int compare_bin(const uchar *a, uint a_length, const uchar *b, uint b_length, my_bool part_key, my_bool skip_end_space) { - uint length= min(a_length,b_length); + uint length= MY_MIN(a_length,b_length); const uchar *end= a+ length; int flag; @@ -99,12 +99,12 @@ static int compare_bin(const uchar *a, uint a_length, Example1: if the function is called for tuples ('aaa','bbb') and ('eee','fff'), then diff_pos[0] = 1 (as 'aaa' != 'eee') - diff_pos[1] = 0 (offset from beggining of tuple b to 'eee' keypart). + diff_pos[1] = 0 (offset from beginning of tuple b to 'eee' keypart). Example2: if the index function is called for tuples ('aaa','bbb') and ('aaa','fff'), diff_pos[0] = 2 (as 'aaa' != 'eee') - diff_pos[1] = 3 (offset from beggining of tuple b to 'fff' keypart, + diff_pos[1] = 3 (offset from beginning of tuple b to 'fff' keypart, here we assume that first key part is CHAR(3) NOT NULL) NOTES @@ -171,7 +171,7 @@ int ha_key_cmp(HA_KEYSEG *keyseg, const uchar *a, continue; /* To next key part */ } } - end= a+ min(keyseg->length,key_length); + end= a+ MY_MIN(keyseg->length,key_length); next_key_length=key_length-keyseg->length; switch ((enum ha_base_keytype) keyseg->type) { diff --git a/mysys/my_compress.c b/mysys/my_compress.c index ea56900db05..0e4db5c3b17 100644 --- a/mysys/my_compress.c +++ b/mysys/my_compress.c @@ -162,7 +162,7 @@ uchar *my_compress_alloc(const uchar *packet, size_t *len, size_t *complen) SYNOPSIS my_uncompress() - packet Compressed data. This is is replaced with the orignal data. + packet Compressed data. This is is replaced with the original data. len Length of compressed data complen Length of the packet buffer (must be enough for the original data) @@ -203,129 +203,4 @@ my_bool my_uncompress(uchar *packet, size_t len, size_t *complen) DBUG_RETURN(0); } -/* - Internal representation of the frm blob is: - - ver 4 bytes - orglen 4 bytes - complen 4 bytes -*/ - -#define BLOB_HEADER 12 - - -/* - packfrm is a method used to compress the frm file for storage in a - handler. This method was developed for the NDB handler and has been moved - here to serve also other uses. - - SYNOPSIS - packfrm() - data Data reference to frm file data. - len Length of frm file data - out:pack_data Reference to the pointer to the packed frm data - out:pack_len Length of packed frm file data - - NOTES - data is replaced with compressed content - - RETURN VALUES - 0 Success - >0 Failure -*/ - -int packfrm(uchar *data, size_t len, - uchar **pack_data, size_t *pack_len) -{ - int error; - size_t org_len, comp_len, blob_len; - uchar *blob; - DBUG_ENTER("packfrm"); - DBUG_PRINT("enter", ("data: 0x%lx len: %lu", (long) data, (ulong) len)); - - error= 1; - org_len= len; - if (my_compress((uchar*)data, &org_len, &comp_len)) - goto err; - - DBUG_PRINT("info", ("org_len: %lu comp_len: %lu", (ulong) org_len, - (ulong) comp_len)); - DBUG_DUMP("compressed", data, org_len); - - error= 2; - blob_len= BLOB_HEADER + org_len; - if (!(blob= (uchar*) my_malloc(blob_len,MYF(MY_WME)))) - goto err; - - /* Store compressed blob in machine independent format */ - int4store(blob, 1); - int4store(blob+4, (uint32) len); - int4store(blob+8, (uint32) org_len); /* compressed length */ - - /* Copy frm data into blob, already in machine independent format */ - memcpy(blob+BLOB_HEADER, data, org_len); - - *pack_data= blob; - *pack_len= blob_len; - error= 0; - - DBUG_PRINT("exit", ("pack_data: 0x%lx pack_len: %lu", - (long) *pack_data, (ulong) *pack_len)); -err: - DBUG_RETURN(error); - -} - -/* - unpackfrm is a method used to decompress the frm file received from a - handler. This method was developed for the NDB handler and has been moved - here to serve also other uses for other clustered storage engines. - - SYNOPSIS - unpackfrm() - pack_data Data reference to packed frm file data - out:unpack_data Reference to the pointer to the unpacked frm data - out:unpack_len Length of unpacked frm file data - - RETURN VALUES¨ - 0 Success - >0 Failure -*/ - -int unpackfrm(uchar **unpack_data, size_t *unpack_len, - const uchar *pack_data) -{ - uchar *data; - size_t complen, orglen; - ulong ver; - DBUG_ENTER("unpackfrm"); - DBUG_PRINT("enter", ("pack_data: 0x%lx", (long) pack_data)); - - ver= uint4korr(pack_data); - orglen= uint4korr(pack_data+4); - complen= uint4korr(pack_data+8); - - DBUG_PRINT("blob",("ver: %lu complen: %lu orglen: %lu", - ver, (ulong) complen, (ulong) orglen)); - DBUG_DUMP("blob->data", pack_data + BLOB_HEADER, complen); - - if (ver != 1) - DBUG_RETURN(1); - if (!(data= my_malloc(max(orglen, complen), MYF(MY_WME)))) - DBUG_RETURN(2); - memcpy(data, pack_data + BLOB_HEADER, complen); - - if (my_uncompress(data, complen, &orglen)) - { - my_free(data); - DBUG_RETURN(3); - } - - *unpack_data= data; - *unpack_len= orglen; - - DBUG_PRINT("exit", ("frmdata: 0x%lx len: %lu", (long) *unpack_data, - (ulong) *unpack_len)); - DBUG_RETURN(0); -} #endif /* HAVE_COMPRESS */ diff --git a/mysys/my_conio.c b/mysys/my_conio.c index 2480c37d039..0af5706cace 100644 --- a/mysys/my_conio.c +++ b/mysys/my_conio.c @@ -166,13 +166,13 @@ char* my_cgets(char *buffer, size_t clen, size_t* plen) though it is known it should not be more than 64K so we cut 64K and try first size of screen buffer if it is still to large we cut half of it and try again - later we may want to cycle from min(clen, 65535) to allowed size + later we may want to cycle from MY_MIN(clen, 65535) to allowed size with small decrement to determine exact allowed buffer */ - clen= min(clen, 65535); + clen= MY_MIN(clen, 65535); do { - clen= min(clen, (size_t) csbi.dwSize.X*csbi.dwSize.Y); + clen= MY_MIN(clen, (size_t) csbi.dwSize.X*csbi.dwSize.Y); if (!ReadConsole((HANDLE)my_coninpfh, (LPVOID)buffer, (DWORD) clen - 1, &plen_res, NULL)) { diff --git a/mysys/my_context.c b/mysys/my_context.c index 6960b0c6c5a..29ba9f79c72 100644 --- a/mysys/my_context.c +++ b/mysys/my_context.c @@ -1,5 +1,5 @@ /* - Copyright 2011 Kristian Nielsen and Monty Program Ab + Copyright 2011, 2012 Kristian Nielsen and Monty Program Ab This file is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public @@ -438,8 +438,6 @@ my_context_destroy(struct my_context *c) int my_context_spawn(struct my_context *c, void (*f)(void *), void *d) { - void (*tmp_f)(void *)= f; - void *tmp_d= d; int ret; DBUG_SWAP_CODE_STATE(&c->dbug_state); @@ -496,8 +494,8 @@ my_context_spawn(struct my_context *c, void (*f)(void *), void *d) "movl $1, %[ret]\n" "4:\n" : [ret] "=a" (ret), - [f] "+c" (tmp_f), - [d] "+d" (tmp_d) + [f] "+c" (f), + [d] "+d" (d) : [stack] "a" (c->stack_top), /* Need this in callee-save register to preserve across function call. */ [save] "D" (&c->save[0]) diff --git a/mysys/my_copy.c b/mysys/my_copy.c index 82cb2cc2f5f..bd23dfc48cd 100644 --- a/mysys/my_copy.c +++ b/mysys/my_copy.c @@ -60,12 +60,12 @@ int my_copy(const char *from, const char *to, myf MyFlags) MY_STAT stat_buff,new_stat_buff; my_bool file_created= 0; DBUG_ENTER("my_copy"); - DBUG_PRINT("my",("from %s to %s MyFlags %d", from, to, MyFlags)); + DBUG_PRINT("my",("from %s to %s MyFlags %lu", from, to, MyFlags)); from_file=to_file= -1; DBUG_ASSERT(!(MyFlags & (MY_FNABP | MY_NABP))); /* for my_read/my_write */ if (MyFlags & MY_HOLD_ORIGINAL_MODES) /* Copy stat if possible */ - new_file_stat= test(my_stat((char*) to, &new_stat_buff, MYF(0))); + new_file_stat= MY_TEST(my_stat((char*) to, &new_stat_buff, MYF(0))); if ((from_file=my_open(from,O_RDONLY | O_SHARE,MyFlags)) >= 0) { diff --git a/mysys/my_create.c b/mysys/my_create.c index e9a1365ca19..014b65c4e14 100644 --- a/mysys/my_create.c +++ b/mysys/my_create.c @@ -38,12 +38,12 @@ File my_create(const char *FileName, int CreateFlags, int access_flags, { int fd; DBUG_ENTER("my_create"); - DBUG_PRINT("my",("Name: '%s' CreateFlags: %d AccessFlags: %d MyFlags: %d", + DBUG_PRINT("my",("Name: '%s' CreateFlags: %d AccessFlags: %d MyFlags: %lu", FileName, CreateFlags, access_flags, MyFlags)); #if defined(_WIN32) fd= my_win_open(FileName, access_flags | O_CREAT); #else - fd= open((char *) FileName, access_flags | O_CREAT, + fd= open((char *) FileName, access_flags | O_CREAT | O_CLOEXEC, CreateFlags ? CreateFlags : my_umask); #endif diff --git a/mysys/default.c b/mysys/my_default.c index e901fb0b593..5c4e33141a6 100644 --- a/mysys/default.c +++ b/mysys/my_default.c @@ -25,7 +25,7 @@ pre- and end 'blank space' are removed from options and values. The following escape sequences are recognized in values: \b \t \n \r \\ - The following arguments are handled automaticly; If used, they must be + The following arguments are handled automatically; If used, they must be first argument on the command line! --no-defaults ; no options are read. --defaults-file=full-path-to-default-file ; Only this file will be read. @@ -35,8 +35,9 @@ ****************************************************************************/ #include "mysys_priv.h" -#include "m_string.h" -#include "m_ctype.h" +#include <my_default.h> +#include <m_string.h> +#include <m_ctype.h> #include <my_dir.h> #ifdef __WIN__ #include <winbase.h> @@ -62,7 +63,7 @@ check the pointer, use "----args-separator----" here to ease debug if someone misused it. - The args seprator will only be added when + The args separator will only be added when my_getopt_use_args_seprator is set to TRUE before calling load_defaults(); @@ -90,7 +91,7 @@ static my_bool defaults_already_read= FALSE; /* Which directories are searched for options (and in which order) */ -#define MAX_DEFAULT_DIRS 6 +#define MAX_DEFAULT_DIRS 7 #define DEFAULT_DIRS_SIZE (MAX_DEFAULT_DIRS + 1) /* Terminate with NULL */ static const char **default_directories = NULL; @@ -102,8 +103,7 @@ static const char *f_extensions[]= { ".cnf", 0 }; #define NEWLINE "\n" #endif -static int handle_default_option(void *in_ctx, const char *group_name, - const char *option); +static int handle_default_option(void *, const char *, const char *); /* This structure defines the context that we pass to callback @@ -138,9 +138,8 @@ static int search_default_file_with_ext(Process_option_func func, - Windows: GetWindowsDirectory() - Windows: C:/ - Windows: Directory above where the executable is located - - Unix: /etc/ - - Unix: /etc/mysql/ - - Unix: --sysconfdir=<path> (compile-time option) + - Unix: /etc/ or the value of DEFAULT_SYSCONFDIR, if defined + - Unix: /etc/mysql/ unless DEFAULT_SYSCONFDIR is defined - ALL: getenv("MYSQL_HOME") - ALL: --defaults-extra-file=<path> (run-time option) - Unix: ~/ @@ -365,7 +364,7 @@ err: RETURN 0 - ok - 1 - error occured + 1 - error occurred */ static int handle_default_option(void *in_ctx, const char *group_name, @@ -411,14 +410,13 @@ int get_defaults_options(int argc, char **argv, char **extra_defaults, char **group_suffix) { - int org_argc= argc, prev_argc= 0; + int org_argc= argc; *defaults= *extra_defaults= *group_suffix= 0; - while (argc >= 2 && argc != prev_argc) + while (argc >= 2) { /* Skip program name or previously handled argument */ argv++; - prev_argc= argc; /* To check if we found */ if (!*defaults && is_prefix(*argv,"--defaults-file=")) { *defaults= *argv + sizeof("--defaults-file=")-1; @@ -437,6 +435,7 @@ int get_defaults_options(int argc, char **argv, argc--; continue; } + break; } return org_argc - argc; } @@ -489,8 +488,7 @@ int load_defaults(const char *conf_file, const char **groups, easily command line options override options in configuration files NOTES - In case of fatal error, the function will print a warning and do - exit(1) + In case of fatal error, the function will print a warning and returns 2 To free used memory one should call free_defaults() with the argument that was put in *argv @@ -521,7 +519,7 @@ int my_load_defaults(const char *conf_file, const char **groups, uint args_sep= my_getopt_use_args_separator ? 1 : 0; DBUG_ENTER("load_defaults"); - init_alloc_root(&alloc,512,0); + init_alloc_root(&alloc, 512, 0, MYF(0)); if ((dirs= init_default_directories(&alloc)) == NULL) goto err; /* @@ -567,7 +565,7 @@ int my_load_defaults(const char *conf_file, const char **groups, for (; *groups ; groups++) group.count++; - if (my_init_dynamic_array(&args, sizeof(char*),*argc, 32)) + if (my_init_dynamic_array(&args, sizeof(char*),*argc, 32, MYF(0))) goto err; ctx.alloc= &alloc; @@ -599,7 +597,7 @@ int my_load_defaults(const char *conf_file, const char **groups, (*argv)+= args_used; /* - Check if we wan't to see the new argument list + Check if we want to see the new argument list This options must always be the last of the default options */ if (*argc >= 2 && !strcmp(argv[0][1],"--print-defaults")) @@ -633,7 +631,7 @@ int my_load_defaults(const char *conf_file, const char **groups, if (!my_getopt_is_args_separator((*argv)[i])) /* skip arguments separator */ printf("%s ", (*argv)[i]); puts(""); - exit(0); + DBUG_RETURN(4); } if (default_directories) @@ -643,8 +641,7 @@ int my_load_defaults(const char *conf_file, const char **groups, err: fprintf(stderr,"Fatal error in defaults handling. Program aborted\n"); - exit(1); - return 0; /* Keep compiler happy */ + DBUG_RETURN(2); } @@ -845,10 +842,10 @@ static int search_default_file_with_ext(Process_option_func opt_handler, ptr, name, line))) goto err; - if (!(search_dir= my_dir(ptr, MYF(MY_WME)))) + if (!(search_dir= my_dir(ptr, MYF(MY_WME | MY_WANT_SORT)))) goto err; - for (i= 0; i < (uint) search_dir->number_off_files; i++) + for (i= 0; i < (uint) search_dir->number_of_files; i++) { search_file= search_dir->dir_entry + i; ext= fn_ext2(search_file->name); @@ -901,7 +898,7 @@ static int search_default_file_with_ext(Process_option_func opt_handler, for ( ; my_isspace(&my_charset_latin1,end[-1]) ; end--) ; end[0]=0; - strmake(curr_gr, ptr, min((size_t) (end-ptr)+1, sizeof(curr_gr)-1)); + strmake(curr_gr, ptr, MY_MIN((size_t) (end-ptr)+1, sizeof(curr_gr)-1)); /* signal that a new group is found */ opt_handler(handler_ctx, curr_gr, NULL); @@ -919,7 +916,7 @@ static int search_default_file_with_ext(Process_option_func opt_handler, end= remove_end_comment(ptr); if ((value= strchr(ptr, '='))) - end= value; /* Option without argument */ + end= value; for ( ; my_isspace(&my_charset_latin1,end[-1]) ; end--) ; if (!value) { @@ -1045,7 +1042,7 @@ void my_print_default_files(const char *conf_file) { const char **dirs; MEM_ROOT alloc; - init_alloc_root(&alloc,512,0); + init_alloc_root(&alloc, 512, 0, MYF(0)); if ((dirs= init_default_directories(&alloc)) == NULL) { @@ -1222,17 +1219,22 @@ static const char **init_default_directories(MEM_ROOT *alloc) errors += add_directory(alloc, "C:/", dirs); if (my_get_module_parent(fname_buffer, sizeof(fname_buffer)) != NULL) + { + errors += add_directory(alloc, fname_buffer, dirs); + + strncat(fname_buffer, "/data", sizeof(fname_buffer)); errors += add_directory(alloc, fname_buffer, dirs); + } } #else - errors += add_directory(alloc, "/etc/", dirs); - errors += add_directory(alloc, "/etc/mysql/", dirs); - #if defined(DEFAULT_SYSCONFDIR) if (DEFAULT_SYSCONFDIR[0]) errors += add_directory(alloc, DEFAULT_SYSCONFDIR, dirs); +#else + errors += add_directory(alloc, "/etc/", dirs); + errors += add_directory(alloc, "/etc/mysql/", dirs); #endif /* DEFAULT_SYSCONFDIR */ #endif diff --git a/mysys/my_delete.c b/mysys/my_delete.c index 8487f1d2e5a..0faf6079d98 100644 --- a/mysys/my_delete.c +++ b/mysys/my_delete.c @@ -31,7 +31,7 @@ int my_delete(const char *name, myf MyFlags) { int err; DBUG_ENTER("my_delete"); - DBUG_PRINT("my",("name %s MyFlags %d", name, MyFlags)); + DBUG_PRINT("my",("name %s MyFlags %lu", name, MyFlags)); #ifdef _WIN32 err = my_win_unlink(name); @@ -119,7 +119,7 @@ static int my_win_unlink(const char *name) if (handle != INVALID_HANDLE_VALUE) { /* - We opened file without sharing flags (exclusive), noone else has this file + We opened file without sharing flags (exclusive), no one else has this file opened, thus it is save to close handle to remove it. No renaming is necessary. */ diff --git a/mysys/my_dlerror.c b/mysys/my_dlerror.c new file mode 100644 index 00000000000..db21b7fc274 --- /dev/null +++ b/mysys/my_dlerror.c @@ -0,0 +1,31 @@ +/* + Copyright (c) 2017, MariaDB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +#include <my_global.h> +#include <string.h> + +const char *my_dlerror(const char *dlpath) +{ + const char *errmsg=dlerror(); + size_t dlpathlen= strlen(dlpath); + if (!strncmp(dlpath, errmsg, dlpathlen)) + { /* if errmsg starts from dlpath, trim this prefix */ + errmsg+=dlpathlen; + if (*errmsg == ':') errmsg++; + if (*errmsg == ' ') errmsg++; + } + return errmsg; +} diff --git a/mysys/my_error.c b/mysys/my_error.c index 23bc06c3d9d..44d112bc049 100644 --- a/mysys/my_error.c +++ b/mysys/my_error.c @@ -48,44 +48,73 @@ */ static struct my_err_head { - struct my_err_head *meh_next; /* chain link */ - const char** (*get_errmsgs) (); /* returns error message format */ - int meh_first; /* error number matching array slot 0 */ - int meh_last; /* error number matching last slot */ -} my_errmsgs_globerrs = {NULL, get_global_errmsgs, EE_ERROR_FIRST, EE_ERROR_LAST}; + struct my_err_head *meh_next; /* chain link */ + const char** (*get_errmsgs)(); /* returns error message format */ + uint meh_first; /* error number matching array slot 0 */ + uint meh_last; /* error number matching last slot */ +} my_errmsgs_globerrs= +{NULL, get_global_errmsgs, EE_ERROR_FIRST, EE_ERROR_LAST}; static struct my_err_head *my_errmsgs_list= &my_errmsgs_globerrs; -/* - Error message to user +/** + @brief Get an error format string from one of the my_error_register()ed sets + + @note + NULL values are possible even within a registered range. - SYNOPSIS - my_error() - nr Errno - MyFlags Flags - ... variable list + @param nr Errno + @retval NULL if no message is registered for this error number + @retval str C-string */ -void my_error(int nr, myf MyFlags, ...) +const char *my_get_err_msg(uint nr) { const char *format; struct my_err_head *meh_p; - va_list args; - char ebuff[ERRMSGSIZE]; - DBUG_ENTER("my_error"); - DBUG_PRINT("my", ("nr: %d MyFlags: %d errno: %d", nr, MyFlags, errno)); - /* Search for the error messages array, which could contain the message. */ + /* Search for the range this error is in. */ for (meh_p= my_errmsgs_list; meh_p; meh_p= meh_p->meh_next) if (nr <= meh_p->meh_last) break; - /* get the error message string. Default, if NULL or empty string (""). */ - if (! (format= (meh_p && (nr >= meh_p->meh_first)) ? - meh_p->get_errmsgs()[nr - meh_p->meh_first] : NULL) || ! *format) - (void) my_snprintf (ebuff, sizeof(ebuff), "Unknown error %d", nr); + /* + If we found the range this error number is in, get the format string. + If the string is empty, or a NULL pointer, or if we're out of return, + we return NULL. + */ + if (!(format= (meh_p && (nr >= meh_p->meh_first)) ? + meh_p->get_errmsgs()[nr - meh_p->meh_first] : NULL) || + !*format) + return NULL; + + return format; +} + + +/** + Fill in and print a previously registered error message. + + @note + Goes through the (sole) function registered in error_handler_hook + + @param nr error number + @param MyFlags Flags + @param ... variable list matching that error format string +*/ + +void my_error(uint nr, myf MyFlags, ...) +{ + const char *format; + va_list args; + char ebuff[ERRMSGSIZE]; + DBUG_ENTER("my_error"); + DBUG_PRINT("my", ("nr: %d MyFlags: %lu errno: %d", nr, MyFlags, errno)); + + if (!(format = my_get_err_msg(nr))) + (void) my_snprintf(ebuff, sizeof(ebuff), "Unknown error %d", nr); else { va_start(args,MyFlags); @@ -98,15 +127,16 @@ void my_error(int nr, myf MyFlags, ...) } -/* - Error as printf - - SYNOPSIS - my_printf_error() - error Errno - format Format string - MyFlags Flags - ... variable list +/** + Print an error message. + + @note + Goes through the (sole) function registered in error_handler_hook + + @param error error number + @param format format string + @param MyFlags Flags + @param ... variable list matching that error format string */ void my_printf_error(uint error, const char *format, myf MyFlags, ...) @@ -114,7 +144,7 @@ void my_printf_error(uint error, const char *format, myf MyFlags, ...) va_list args; char ebuff[ERRMSGSIZE]; DBUG_ENTER("my_printf_error"); - DBUG_PRINT("my", ("nr: %d MyFlags: %d errno: %d format: %s", + DBUG_PRINT("my", ("nr: %d MyFlags: %lu errno: %d format: %s", error, MyFlags, errno, format)); va_start(args,MyFlags); @@ -125,22 +155,23 @@ void my_printf_error(uint error, const char *format, myf MyFlags, ...) DBUG_VOID_RETURN; } -/* - Error with va_list - - SYNOPSIS - my_printv_error() - error Errno - format Format string - MyFlags Flags - ... variable list +/** + Print an error message. + + @note + Goes through the (sole) function registered in error_handler_hook + + @param error error number + @param format format string + @param MyFlags Flags + @param ap variable list matching that error format string */ void my_printv_error(uint error, const char *format, myf MyFlags, va_list ap) { char ebuff[ERRMSGSIZE]; DBUG_ENTER("my_printv_error"); - DBUG_PRINT("my", ("nr: %d MyFlags: %d errno: %d format: %s", + DBUG_PRINT("my", ("nr: %d MyFlags: %lu errno: %d format: %s", error, MyFlags, errno, format)); (void) my_vsnprintf(ebuff, sizeof(ebuff), format, ap); @@ -149,14 +180,15 @@ void my_printv_error(uint error, const char *format, myf MyFlags, va_list ap) } -/* - Give message using error_handler_hook +/** + Print an error message. + + @note + Goes through the (sole) function registered in error_handler_hook - SYNOPSIS - my_message() - error Errno - str Error message - MyFlags Flags + @param error error number + @param str error message + @param MyFlags Flags */ void my_message(uint error, const char *str, register myf MyFlags) @@ -165,16 +197,11 @@ void my_message(uint error, const char *str, register myf MyFlags) } -/* +/** Register error messages for use with my_error(). - SYNOPSIS - my_error_register() - errmsgs array of pointers to error messages - first error number of first message in the array - last error number of last message in the array + @description - DESCRIPTION The pointer array is expected to contain addresses to NUL-terminated C character strings. The array contains (last - first + 1) pointers. NULL pointers and empty strings ("") are allowed. These will be mapped to @@ -182,12 +209,15 @@ void my_message(uint error, const char *str, register myf MyFlags) This function registers the error numbers 'first' to 'last'. No overlapping with previously registered error numbers is allowed. - RETURN - 0 OK - != 0 Error + @param errmsgs array of pointers to error messages + @param first error number of first message in the array + @param last error number of last message in the array + + @retval 0 OK + @retval != 0 Error */ -int my_error_register(const char** (*get_errmsgs) (), int first, int last) +int my_error_register(const char** (*get_errmsgs) (void), uint first, uint last) { struct my_err_head *meh_p; struct my_err_head **search_meh_pp; @@ -223,28 +253,27 @@ int my_error_register(const char** (*get_errmsgs) (), int first, int last) } -/* +/** Unregister formerly registered error messages. - SYNOPSIS - my_error_unregister() - first error number of first message - last error number of last message + @description - DESCRIPTION This function unregisters the error numbers 'first' to 'last'. These must have been previously registered by my_error_register(). 'first' and 'last' must exactly match the registration. If a matching registration is present, the header is removed from the list and the pointer to the error messages pointers array is returned. + (The messages themselves are not released here as they may be static.) Otherwise, NULL is returned. - RETURN - non-NULL OK, returns address of error messages pointers array. - NULL Error, no such number range registered. + @param first error number of first message + @param last error number of last message + + @retval NULL Error, no such number range registered. + @retval non-NULL OK, returns address of error messages pointers array. */ -const char **my_error_unregister(int first, int last) +const char **my_error_unregister(uint first, uint last) { struct my_err_head *meh_p; struct my_err_head **search_meh_pp; @@ -274,6 +303,17 @@ const char **my_error_unregister(int first, int last) } +/** + Unregister all formerly registered error messages. + + @description + + This function unregisters all error numbers that previously have + been previously registered by my_error_register(). + All headers are removed from the list; the messages themselves are + not released here as they may be static. +*/ + void my_error_unregister_all(void) { struct my_err_head *cursor, *saved_next; diff --git a/mysys/my_file.c b/mysys/my_file.c index b3aef8494cb..23226595b2e 100644 --- a/mysys/my_file.c +++ b/mysys/my_file.c @@ -75,7 +75,7 @@ static uint set_max_open_files(uint max_file_limit) static uint set_max_open_files(uint max_file_limit) { /* We don't know the limit. Return best guess */ - return min(max_file_limit, OS_FILE_LIMIT); + return MY_MIN(max_file_limit, OS_FILE_LIMIT); } #endif @@ -98,7 +98,7 @@ uint my_set_max_open_files(uint files) DBUG_PRINT("enter",("files: %u my_file_limit: %u", files, my_file_limit)); files+= MY_FILE_MIN; - files= set_max_open_files(min(files, OS_FILE_LIMIT)); + files= set_max_open_files(MY_MIN(files, OS_FILE_LIMIT)); if (files <= MY_NFILE) DBUG_RETURN(files); @@ -108,9 +108,9 @@ uint my_set_max_open_files(uint files) /* Copy any initialized files */ memcpy((char*) tmp, (char*) my_file_info, - sizeof(*tmp) * min(my_file_limit, files)); + sizeof(*tmp) * MY_MIN(my_file_limit, files)); bzero((char*) (tmp + my_file_limit), - max((int) (files- my_file_limit), 0)*sizeof(*tmp)); + MY_MAX((int) (files- my_file_limit), 0)*sizeof(*tmp)); my_free_open_file_info(); /* Free if already allocated */ my_file_info= tmp; my_file_limit= files; diff --git a/mysys/my_fopen.c b/mysys/my_fopen.c index 9131a2549e0..c720e2c9168 100644 --- a/mysys/my_fopen.c +++ b/mysys/my_fopen.c @@ -42,9 +42,9 @@ static void make_ftype(char * to,int flag); FILE *my_fopen(const char *filename, int flags, myf MyFlags) { FILE *fd; - char type[5]; + char type[10]; DBUG_ENTER("my_fopen"); - DBUG_PRINT("my",("Name: '%s' flags: %d MyFlags: %d", + DBUG_PRINT("my",("Name: '%s' flags: %d MyFlags: %lu", filename, flags, MyFlags)); make_ftype(type,flags); @@ -81,8 +81,7 @@ FILE *my_fopen(const char *filename, int flags, myf MyFlags) my_errno=errno; DBUG_PRINT("error",("Got error %d on open",my_errno)); if (MyFlags & (MY_FFNF | MY_FAE | MY_WME)) - my_error((flags & O_RDONLY) || (flags == O_RDONLY ) ? EE_FILENOTFOUND : - EE_CANTCREATEFILE, + my_error((flags & O_RDONLY) ? EE_FILENOTFOUND : EE_CANTCREATEFILE, MYF(ME_BELL+ME_WAITTANG), filename, my_errno); DBUG_RETURN((FILE*) 0); } /* my_fopen */ @@ -223,7 +222,7 @@ int my_fclose(FILE *fd, myf MyFlags) { int err,file; DBUG_ENTER("my_fclose"); - DBUG_PRINT("my",("stream: 0x%lx MyFlags: %d", (long) fd, MyFlags)); + DBUG_PRINT("my",("stream: 0x%lx MyFlags: %lu", (long) fd, MyFlags)); mysql_mutex_lock(&THR_LOCK_open); file= my_fileno(fd); @@ -259,7 +258,7 @@ FILE *my_fdopen(File Filedes, const char *name, int Flags, myf MyFlags) FILE *fd; char type[5]; DBUG_ENTER("my_fdopen"); - DBUG_PRINT("my",("fd: %d Flags: %d MyFlags: %d", + DBUG_PRINT("my",("fd: %d Flags: %d MyFlags: %lu", Filedes, Flags, MyFlags)); make_ftype(type,Flags); @@ -345,9 +344,11 @@ static void make_ftype(register char * to, register int flag) else *to++= 'r'; -#if FILE_BINARY /* If we have binary-files */ if (flag & FILE_BINARY) *to++='b'; -#endif + + if (O_CLOEXEC) + *to++= 'e'; + *to='\0'; } /* make_ftype */ diff --git a/mysys/my_fstream.c b/mysys/my_fstream.c index e758609741b..de752fa149f 100644 --- a/mysys/my_fstream.c +++ b/mysys/my_fstream.c @@ -46,7 +46,7 @@ size_t my_fread(FILE *stream, uchar *Buffer, size_t Count, myf MyFlags) { size_t readbytes; DBUG_ENTER("my_fread"); - DBUG_PRINT("my",("stream: 0x%lx Buffer: 0x%lx Count: %u MyFlags: %d", + DBUG_PRINT("my",("stream: 0x%lx Buffer: 0x%lx Count: %u MyFlags: %lu", (long) stream, (long) Buffer, (uint) Count, MyFlags)); if ((readbytes= fread(Buffer, sizeof(char), Count, stream)) != Count) @@ -94,7 +94,7 @@ size_t my_fwrite(FILE *stream, const uchar *Buffer, size_t Count, myf MyFlags) uint errors; #endif DBUG_ENTER("my_fwrite"); - DBUG_PRINT("my",("stream: 0x%lx Buffer: 0x%lx Count: %u MyFlags: %d", + DBUG_PRINT("my",("stream: 0x%lx Buffer: 0x%lx Count: %u MyFlags: %lu", (long) stream, (long) Buffer, (uint) Count, MyFlags)); #if !defined(NO_BACKGROUND) && defined(USE_MY_STREAM) @@ -163,7 +163,7 @@ my_off_t my_fseek(FILE *stream, my_off_t pos, int whence, myf MyFlags __attribute__((unused))) { DBUG_ENTER("my_fseek"); - DBUG_PRINT("my",("stream: 0x%lx pos: %lu whence: %d MyFlags: %d", + DBUG_PRINT("my",("stream: 0x%lx pos: %lu whence: %d MyFlags: %lu", (long) stream, (long) pos, whence, MyFlags)); DBUG_RETURN(fseek(stream, (off_t) pos, whence) ? MY_FILEPOS_ERROR : (my_off_t) ftell(stream)); @@ -176,7 +176,7 @@ my_off_t my_ftell(FILE *stream, myf MyFlags __attribute__((unused))) { off_t pos; DBUG_ENTER("my_ftell"); - DBUG_PRINT("my",("stream: 0x%lx MyFlags: %d", (long) stream, MyFlags)); + DBUG_PRINT("my",("stream: 0x%lx MyFlags: %lu", (long) stream, MyFlags)); pos=ftell(stream); DBUG_PRINT("exit",("ftell: %lu",(ulong) pos)); DBUG_RETURN((my_off_t) pos); diff --git a/mysys/my_gethwaddr.c b/mysys/my_gethwaddr.c index c7aa7f17f1d..0fad7f90552 100644 --- a/mysys/my_gethwaddr.c +++ b/mysys/my_gethwaddr.c @@ -87,13 +87,17 @@ my_bool my_gethwaddr(uchar *to) int fd, res= 1; struct ifreq ifr[32]; struct ifconf ifc; + DBUG_ENTER("my_gethwaddr"); ifc.ifc_req= ifr; ifc.ifc_len= sizeof(ifr); fd = socket(AF_INET, SOCK_DGRAM, 0); if (fd < 0) + { + DBUG_PRINT("error", ("socket() call failed with %d", errno)); goto err; + } if (ioctl(fd, SIOCGIFCONF, (char*)&ifc) >= 0) { @@ -106,8 +110,8 @@ my_bool my_gethwaddr(uchar *to) ETHER_ADDR_LEN); #else /* - A bug in OpenSolaris used to prevent non-root from getting a mac address: - {no url. Oracle killed the old OpenSolaris bug database} + A bug in OpenSolaris used to prevent non-root from getting a mac + address: {no url. Oracle killed the old OpenSolaris bug database} Thus, we'll use an alternative method and extract the address from the arp table. @@ -124,7 +128,7 @@ my_bool my_gethwaddr(uchar *to) close(fd); err: - return res; + DBUG_RETURN(res); } #elif defined(_WIN32) diff --git a/mysys/my_getopt.c b/mysys/my_getopt.c index 425a2b13d66..57b28d1fd8a 100644 --- a/mysys/my_getopt.c +++ b/mysys/my_getopt.c @@ -16,6 +16,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA */ #include <my_global.h> +#include <my_default.h> #include <m_string.h> #include <stdlib.h> #include <my_sys.h> @@ -46,16 +47,17 @@ static char *check_struct_option(char *cur_arg, char *key_name); order of their arguments must correspond to each other. */ static const char *special_opt_prefix[]= -{"skip", "disable", "enable", "maximum", "loose", 0}; +{"skip", "disable", "enable", "maximum", "loose", "autoset", 0}; static const uint special_opt_prefix_lengths[]= -{ 4, 7, 6, 7, 5, 0}; +{ 4, 7, 6, 7, 5, 7, 0}; enum enum_special_opt -{ OPT_SKIP, OPT_DISABLE, OPT_ENABLE, OPT_MAXIMUM, OPT_LOOSE}; +{ OPT_SKIP, OPT_DISABLE, OPT_ENABLE, OPT_MAXIMUM, OPT_LOOSE, OPT_AUTOSET}; char *disabled_my_option= (char*) "0"; char *enabled_my_option= (char*) "1"; +char *autoset_my_option= (char*) "auto"; -/* +/* This is a flag that can be set in client programs. 0 means that my_getopt will not print error messages, but the client should do it by itself @@ -63,13 +65,21 @@ char *enabled_my_option= (char*) "1"; my_bool my_getopt_print_errors= 1; -/* +/* This is a flag that can be set in client programs. 1 means that my_getopt will skip over options it does not know how to handle. */ my_bool my_getopt_skip_unknown= 0; + +/* + This is a flag that can be set in client programs. 1 means that + my_getopt will reconize command line options by their unambiguous + prefixes. 0 means an option must be always specified in full. +*/ +my_bool my_getopt_prefix_matching= 1; + static void default_reporter(enum loglevel level, const char *format, ...) { @@ -189,7 +199,7 @@ int handle_options(int *argc, char ***argv, { uint UNINIT_VAR(opt_found), argvpos= 0, length; my_bool end_of_options= 0, must_be_var, set_maximum_value, - option_is_loose; + option_is_loose, option_is_autoset; char **pos, **pos_end, *optend, *opt_str, key_name[FN_REFLEN]; const char *UNINIT_VAR(prev_found); const struct my_option *optp; @@ -240,6 +250,7 @@ int handle_options(int *argc, char ***argv, must_be_var= 0; set_maximum_value= 0; option_is_loose= 0; + option_is_autoset= 0; cur_arg++; /* skip '-' */ if (*cur_arg == '-') /* check for long option, */ @@ -288,6 +299,8 @@ int handle_options(int *argc, char ***argv, length-= special_opt_prefix_lengths[i] + 1; if (i == OPT_LOOSE) option_is_loose= 1; + else if (i == OPT_AUTOSET) + option_is_autoset= 1; if ((opt_found= findopt(opt_str, length, &optp, &prev_found))) { if (opt_found > 1) @@ -449,6 +462,36 @@ int handle_options(int *argc, char ***argv, } argument= optend; } + else if (option_is_autoset) + { + if (optend) + { + my_getopt_error_reporter(ERROR_LEVEL, + "%s: automatically set " + "option '--%s' cannot take an argument", + my_progname, optp->name); + + DBUG_RETURN(EXIT_NO_ARGUMENT_ALLOWED); + } + /* + We support automatic setup only via get_one_option and only for + marked options. + */ + if (!get_one_option || + !(optp->var_type & GET_AUTO)) + { + my_getopt_error_reporter(option_is_loose ? + WARNING_LEVEL : ERROR_LEVEL, + "%s: automatic setup request is " + "unsupported by option '--%s'", + my_progname, optp->name); + if (!option_is_loose) + DBUG_RETURN(EXIT_ARGUMENT_INVALID); + continue; + } + else + argument= autoset_my_option; + } else if (optp->arg_type == REQUIRED_ARG && !optend) { /* Check if there are more arguments after this one, @@ -576,7 +619,8 @@ int handle_options(int *argc, char ***argv, (*argc)--; /* option handled (short), decrease argument count */ continue; } - if (((error= setval(optp, value, argument, set_maximum_value))) && + if ((!option_is_autoset) && + ((error= setval(optp, value, argument, set_maximum_value))) && !option_is_loose) DBUG_RETURN(error); if (get_one_option && get_one_option(optp->id, optp, argument)) @@ -850,6 +894,9 @@ static int findopt(char *optpat, uint length, if (!opt->name[length]) /* Exact match */ DBUG_RETURN(1); + if (!my_getopt_prefix_matching) + continue; + if (!count) { /* We only need to know one prev */ @@ -866,6 +913,14 @@ static int findopt(char *optpat, uint length, } } } + + if (count == 1) + my_getopt_error_reporter(INFORMATION_LEVEL, + "Using unique option prefix '%.*s' is error-prone " + "and can break in the future. " + "Please use the full name '%s' instead.", + length, optpat, *ffname); + DBUG_RETURN(count); } @@ -894,15 +949,39 @@ my_bool getopt_compare_strings(register const char *s, register const char *t, /* function: eval_num_suffix + Transforms suffix like k/m/g to their real value. +*/ + +static inline long eval_num_suffix(char *suffix, int *error) +{ + long num= 1; + if (*suffix == 'k' || *suffix == 'K') + num*= 1024L; + else if (*suffix == 'm' || *suffix == 'M') + num*= 1024L * 1024L; + else if (*suffix == 'g' || *suffix == 'G') + num*= 1024L * 1024L * 1024L; + else if (*suffix) + { + *error= 1; + return 0; + } + return num; +} + +/* + function: eval_num_suffix_ll + Transforms a number with a suffix to real number. Suffix can be k|K for kilo, m|M for mega or g|G for giga. */ -static longlong eval_num_suffix(char *argument, int *error, char *option_name) +static longlong eval_num_suffix_ll(char *argument, + int *error, char *option_name) { char *endchar; longlong num; - DBUG_ENTER("eval_num_suffix"); + DBUG_ENTER("eval_num_suffix_ll"); *error= 0; @@ -915,23 +994,47 @@ static longlong eval_num_suffix(char *argument, int *error, char *option_name) *error= 1; DBUG_RETURN(0); } - if (*endchar == 'k' || *endchar == 'K') - num*= 1024L; - else if (*endchar == 'm' || *endchar == 'M') - num*= 1024L * 1024L; - else if (*endchar == 'g' || *endchar == 'G') - num*= 1024L * 1024L * 1024L; - else if (*endchar) - { + num*= eval_num_suffix(endchar, error); + if (*error) fprintf(stderr, "Unknown suffix '%c' used for variable '%s' (value '%s')\n", *endchar, option_name, argument); + DBUG_RETURN(num); +} + +/* + function: eval_num_suffix_ull + + Transforms a number with a suffix to positive Integer. Suffix can + be k|K for kilo, m|M for mega or g|G for giga. +*/ + +static ulonglong eval_num_suffix_ull(char *argument, + int *error, char *option_name) +{ + char *endchar; + ulonglong num; + DBUG_ENTER("eval_num_suffix_ull"); + + *error= 0; + errno= 0; + num= strtoull(argument, &endchar, 10); + if (errno == ERANGE) + { + my_getopt_error_reporter(ERROR_LEVEL, + "Incorrect integer value: '%s'", argument); *error= 1; DBUG_RETURN(0); } + num*= eval_num_suffix(endchar, error); + if (*error) + fprintf(stderr, + "Unknown suffix '%c' used for variable '%s' (value '%s')\n", + *endchar, option_name, argument); DBUG_RETURN(num); } + /* function: getopt_ll @@ -945,7 +1048,7 @@ static longlong eval_num_suffix(char *argument, int *error, char *option_name) static longlong getopt_ll(char *arg, const struct my_option *optp, int *err) { - longlong num=eval_num_suffix(arg, err, (char*) optp->name); + longlong num=eval_num_suffix_ll(arg, err, (char*) optp->name); return getopt_ll_limit_value(num, optp, NULL); } @@ -1022,7 +1125,7 @@ longlong getopt_ll_limit_value(longlong num, const struct my_option *optp, static ulonglong getopt_ull(char *arg, const struct my_option *optp, int *err) { - ulonglong num= eval_num_suffix(arg, err, (char*) optp->name); + ulonglong num= eval_num_suffix_ull(arg, err, (char*) optp->name); return getopt_ull_limit_value(num, optp, NULL); } @@ -1258,7 +1361,8 @@ void my_cleanup_options(const struct my_option *options) SYNOPSIS init_variables() - options Array of options + options Array of options + func_init_one_value Call this function to init the variable NOTES We will initialize the value that is pointed to by options->value. @@ -1267,7 +1371,7 @@ void my_cleanup_options(const struct my_option *options) */ static void init_variables(const struct my_option *options, - init_func_p init_one_value) + init_func_p func_init_one_value) { DBUG_ENTER("init_variables"); for (; options->name; options++) @@ -1280,11 +1384,11 @@ static void init_variables(const struct my_option *options, set the value to default value. */ if (options->u_max_value) - init_one_value(options, options->u_max_value, options->max_value); + func_init_one_value(options, options->u_max_value, options->max_value); value= (options->var_type & GET_ASK_ADDR ? (*getopt_get_addr)("", 0, options, 0) : options->value); if (value) - init_one_value(options, value, options->def_value); + func_init_one_value(options, value, options->def_value); } DBUG_VOID_RETURN; } @@ -1299,6 +1403,48 @@ static uint print_name(const struct my_option *optp) return s - optp->name; } +/** prints option comment with indentation and wrapping. + + The comment column starts at startpos, and has width of width + Current cursor position is curpos, returns new cursor position + + @note can print one character beyond width! +*/ +static uint print_comment(const char *comment, + int curpos, int startpos, int width) +{ + const char *end= strend(comment); + int endpos= startpos + width; + + for (; curpos < startpos; curpos++) + putchar(' '); + + if (*comment == '.' || *comment == ',') + { + putchar(*comment); + comment++; + curpos++; + } + + while (end - comment > endpos - curpos) + { + const char *line_end; + for (line_end= comment + endpos - curpos; + line_end > comment && *line_end != ' '; + line_end--); + for (; comment < line_end; comment++) + putchar(*comment); + while (*comment == ' ') + comment++; /* skip the space, as a newline will take it's place now */ + putchar('\n'); + for (curpos= 0; curpos < startpos; curpos++) + putchar(' '); + } + printf("%s", comment); + return curpos + (end - comment); +} + + /* function: my_print_options @@ -1308,12 +1454,14 @@ static uint print_name(const struct my_option *optp) void my_print_help(const struct my_option *options) { uint col, name_space= 22, comment_space= 57; - const char *line_end; const struct my_option *optp; DBUG_ENTER("my_print_help"); for (optp= options; optp->name; optp++) { + const char *typelib_help= 0; + if (!optp->comment) + continue; if (optp->id && optp->id < 256) { printf(" -%c%s", optp->id, strlen(optp->name) ? ", " : " "); @@ -1350,29 +1498,51 @@ void my_print_help(const struct my_option *options) optp->arg_type == OPT_ARG ? "]" : ""); col+= (optp->arg_type == OPT_ARG) ? 5 : 3; } - if (col > name_space && optp->comment && *optp->comment) - { - putchar('\n'); - col= 0; - } } - for (; col < name_space; col++) - putchar(' '); if (optp->comment && *optp->comment) { - const char *comment= optp->comment, *end= strend(comment); + uint count; - while ((uint) (end - comment) > comment_space) + if (col > name_space) { - for (line_end= comment + comment_space; *line_end != ' '; line_end--); - for (; comment != line_end; comment++) - putchar(*comment); - comment++; /* skip the space, as a newline will take it's place now */ putchar('\n'); - for (col= 0; col < name_space; col++) - putchar(' '); + col= 0; + } + + col= print_comment(optp->comment, col, name_space, comment_space); + if (optp->var_type & GET_AUTO) + { + col= print_comment(" (Automatically configured unless set explicitly)", + col, name_space, comment_space); + } + + switch (optp->var_type & GET_TYPE_MASK) { + case GET_ENUM: + typelib_help= ". One of: "; + count= optp->typelib->count; + break; + case GET_SET: + typelib_help= ". Any combination of: "; + count= optp->typelib->count; + break; + case GET_FLAGSET: + typelib_help= ". Takes a comma-separated list of option=value pairs, " + "where value is on, off, or default, and options are: "; + count= optp->typelib->count - 1; + break; + } + if (typelib_help && + strstr(optp->comment, optp->typelib->type_names[0]) == NULL) + { + uint i; + col= print_comment(typelib_help, col, name_space, comment_space); + col= print_comment(optp->typelib->type_names[0], col, name_space, comment_space); + for (i= 1; i < count; i++) + { + col= print_comment(", ", col, name_space, comment_space); + col= print_comment(optp->typelib->type_names[i], col, name_space, comment_space); + } } - printf("%s", comment); } putchar('\n'); if ((optp->var_type & GET_TYPE_MASK) == GET_BOOL) diff --git a/mysys/my_getwd.c b/mysys/my_getwd.c index 444ed4273b5..bfa28f1d372 100644 --- a/mysys/my_getwd.c +++ b/mysys/my_getwd.c @@ -37,7 +37,7 @@ MyFlags Flags NOTES - Directory is allways ended with FN_LIBCHAR + Directory is always ended with FN_LIBCHAR RESULT 0 ok @@ -48,7 +48,7 @@ int my_getwd(char * buf, size_t size, myf MyFlags) { char * pos; DBUG_ENTER("my_getwd"); - DBUG_PRINT("my",("buf: 0x%lx size: %u MyFlags %d", + DBUG_PRINT("my",("buf: 0x%lx size: %u MyFlags %lu", (long) buf, (uint) size, MyFlags)); if (size < 1) @@ -95,7 +95,7 @@ int my_setwd(const char *dir, myf MyFlags) size_t length; char *start, *pos; DBUG_ENTER("my_setwd"); - DBUG_PRINT("my",("dir: '%s' MyFlags %d", dir, MyFlags)); + DBUG_PRINT("my",("dir: '%s' MyFlags %lu", dir, MyFlags)); start=(char *) dir; if (! dir[0] || (dir[0] == FN_LIBCHAR && dir[1] == 0)) @@ -157,12 +157,12 @@ int test_if_hard_path(register const char *dir_name) my_bool has_path(const char *name) { - return test(strchr(name, FN_LIBCHAR)) + return MY_TEST(strchr(name, FN_LIBCHAR)) #if FN_LIBCHAR != '/' - || test(strchr(name,'/')) + || MY_TEST(strchr(name, '/')) #endif #ifdef FN_DEVCHAR - || test(strchr(name, FN_DEVCHAR)) + || MY_TEST(strchr(name, FN_DEVCHAR)) #endif ; } diff --git a/mysys/my_handler_errors.h b/mysys/my_handler_errors.h deleted file mode 100644 index a115b8913f5..00000000000 --- a/mysys/my_handler_errors.h +++ /dev/null @@ -1,93 +0,0 @@ -#ifndef MYSYS_MY_HANDLER_ERRORS_INCLUDED -#define MYSYS_MY_HANDLER_ERRORS_INCLUDED - -/* Copyright (c) 2008, 2012, Oracle and/or its affiliates. - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; version 2 of the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software Foundation, - Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */ - -/* - Errors a handler can give you -*/ - -static const char *handler_error_messages[]= -{ - "Didn't find key on read or update", - "Duplicate key on write or update", - "Internal (unspecified) error in handler", - "Someone has changed the row since it was read (while the table was locked to prevent it)", - "Wrong index given to function", - "Undefined handler error 125", - "Index file is crashed", - "Record file is crashed", - "Out of memory in engine", - "Undefined handler error 129", - "Incorrect file format", - "Command not supported by database", - "Old database file", - "No record read before update", - "Record was already deleted (or record file crashed)", - "No more room in record file", - "No more room in index file", - "No more records (read after end of file)", - "Unsupported extension used for table", - "Too big row", - "Wrong create options", - "Duplicate unique key or constraint on write or update", - "Unknown character set used in table", - "Conflicting table definitions in sub-tables of MERGE table", - "Table is crashed and last repair failed", - "Table was marked as crashed and should be repaired", - "Lock timed out; Retry transaction", - "Lock table is full; Restart program with a larger lock table", - "Updates are not allowed under a read only transactions", - "Lock deadlock; Retry transaction", - "Foreign key constraint is incorrectly formed", - "Cannot add a child row", - "Cannot delete a parent row", - "No savepoint with that name", - "Non unique key block size", - "The table does not exist in engine", - "The table already existed in storage engine", - "Could not connect to storage engine", - "Unexpected null pointer found when using spatial index", - "The table changed in storage engine", - "There's no partition in table for the given value", - "Row-based binary logging of row failed", - "Index needed in foreign key constraint", - "Upholding foreign key constraints would lead to a duplicate key error in some other table", - "Table needs to be upgraded before it can be used", - "Table is read only", - "Failed to get next auto increment value", - "Failed to set row auto increment value", - "Unknown (generic) error from engine", - "Record was not update. Original values was same as new values", - "It is not possible to log this statement", - "The event was corrupt, leading to illegal data being read", - "The table is of a new format not supported by this version", - "The event could not be processed. No other handler error happened", - "Got a fatal error during initialization of handler", - "File too short; Expected more data in file", - "Read page with wrong checksum", - "Too many active concurrent transactions", - "Index column length exceeds limit", - "Index corrupted", - "Undo record too big", - "Table is being used in foreign key check", - "Row is not visible by the current transaction", - "Operation was interrupted by end user (probably kill command?)", - "Disk full", - "Incompatible key or row definition between the MariaDB .frm file and the information in the storage engine. You have to dump and restore the table to fix this" -}; - -#endif /* MYSYS_MY_HANDLER_ERRORS_INCLUDED */ diff --git a/mysys/my_init.c b/mysys/my_init.c index 5f6650f0ec9..84489a994e3 100644 --- a/mysys/my_init.c +++ b/mysys/my_init.c @@ -21,6 +21,7 @@ #include <m_string.h> #include <m_ctype.h> #include <signal.h> +#include <mysql/psi/mysql_stage.h> #ifdef __WIN__ #ifdef _MSC_VER #include <locale.h> @@ -98,7 +99,7 @@ my_bool my_init(void) if (my_progname) my_progname_short= my_progname + dirname_length(my_progname); - /* Initalize our mutex handling */ + /* Initialize our mutex handling */ my_mutex_init(); if (my_thread_global_init()) @@ -199,7 +200,6 @@ Voluntary context switches %ld, Involuntary context switches %ld\n", _CrtSetReportMode( _CRT_ASSERT, _CRTDBG_MODE_FILE ); _CrtSetReportFile( _CRT_ASSERT, _CRTDBG_FILE_STDERR ); _CrtCheckMemory(); - _CrtDumpMemoryLeaks(); #endif } @@ -414,7 +414,7 @@ static my_bool win32_init_tcp_ip() WORD wVersionRequested = MAKEWORD( 2, 2 ); WSADATA wsaData; /* Be a good citizen: maybe another lib has already initialised - sockets, so dont clobber them unless necessary */ + sockets, so don't clobber them unless necessary */ if (WSAStartup( wVersionRequested, &wsaData )) { /* Load failed, maybe because of previously loaded @@ -440,8 +440,10 @@ static my_bool win32_init_tcp_ip() } #endif /* __WIN__ */ -#ifdef HAVE_PSI_INTERFACE +PSI_stage_info stage_waiting_for_table_level_lock= +{0, "Waiting for table level lock", 0}; +#ifdef HAVE_PSI_INTERFACE #if !defined(HAVE_PREAD) && !defined(_WIN32) PSI_mutex_key key_my_file_info_mutex; #endif /* !defined(HAVE_PREAD) && !defined(_WIN32) */ @@ -451,9 +453,10 @@ PSI_mutex_key key_LOCK_localtime_r; #endif /* !defined(HAVE_LOCALTIME_R) || !defined(HAVE_GMTIME_R) */ PSI_mutex_key key_BITMAP_mutex, key_IO_CACHE_append_buffer_lock, - key_IO_CACHE_SHARE_mutex, key_KEY_CACHE_cache_lock, key_LOCK_alarm, + key_IO_CACHE_SHARE_mutex, key_KEY_CACHE_cache_lock, + key_LOCK_alarm, key_LOCK_timer, key_my_thread_var_mutex, key_THR_LOCK_charset, key_THR_LOCK_heap, - key_THR_LOCK_isam, key_THR_LOCK_lock, key_THR_LOCK_malloc, + key_THR_LOCK_lock, key_THR_LOCK_malloc, key_THR_LOCK_mutex, key_THR_LOCK_myisam, key_THR_LOCK_net, key_THR_LOCK_open, key_THR_LOCK_threads, key_TMPDIR_mutex, key_THR_LOCK_myisam_mmap, key_LOCK_uuid_generator; @@ -471,10 +474,10 @@ static PSI_mutex_info all_mysys_mutexes[]= { &key_IO_CACHE_SHARE_mutex, "IO_CACHE::SHARE_mutex", 0}, { &key_KEY_CACHE_cache_lock, "KEY_CACHE::cache_lock", 0}, { &key_LOCK_alarm, "LOCK_alarm", PSI_FLAG_GLOBAL}, + { &key_LOCK_timer, "LOCK_timer", PSI_FLAG_GLOBAL}, { &key_my_thread_var_mutex, "my_thread_var::mutex", 0}, { &key_THR_LOCK_charset, "THR_LOCK_charset", PSI_FLAG_GLOBAL}, { &key_THR_LOCK_heap, "THR_LOCK_heap", PSI_FLAG_GLOBAL}, - { &key_THR_LOCK_isam, "THR_LOCK_isam", PSI_FLAG_GLOBAL}, { &key_THR_LOCK_lock, "THR_LOCK_lock", PSI_FLAG_GLOBAL}, { &key_THR_LOCK_malloc, "THR_LOCK_malloc", PSI_FLAG_GLOBAL}, { &key_THR_LOCK_mutex, "THR_LOCK::mutex", 0}, @@ -487,13 +490,14 @@ static PSI_mutex_info all_mysys_mutexes[]= { &key_LOCK_uuid_generator, "LOCK_uuid_generator", PSI_FLAG_GLOBAL } }; -PSI_cond_key key_COND_alarm, key_IO_CACHE_SHARE_cond, +PSI_cond_key key_COND_alarm, key_COND_timer, key_IO_CACHE_SHARE_cond, key_IO_CACHE_SHARE_cond_writer, key_my_thread_var_suspend, key_THR_COND_threads, key_WT_RESOURCE_cond; static PSI_cond_info all_mysys_conds[]= { { &key_COND_alarm, "COND_alarm", PSI_FLAG_GLOBAL}, + { &key_COND_timer, "COND_timer", PSI_FLAG_GLOBAL}, { &key_IO_CACHE_SHARE_cond, "IO_CACHE_SHARE::cond", 0}, { &key_IO_CACHE_SHARE_cond_writer, "IO_CACHE_SHARE::cond_writer", 0}, { &key_my_thread_var_suspend, "my_thread_var::suspend", 0}, @@ -510,12 +514,17 @@ static PSI_rwlock_info all_mysys_rwlocks[]= #ifdef USE_ALARM_THREAD PSI_thread_key key_thread_alarm; +#endif +PSI_thread_key key_thread_timer; static PSI_thread_info all_mysys_threads[]= { - { &key_thread_alarm, "alarm", PSI_FLAG_GLOBAL} +#ifdef USE_ALARM_THREAD + { &key_thread_alarm, "alarm", PSI_FLAG_GLOBAL}, +#endif + { &key_thread_timer, "statement_timer", PSI_FLAG_GLOBAL} }; -#endif /* USE_ALARM_THREAD */ + #ifdef HUGETLB_USE_PROC_MEMINFO PSI_file_key key_file_proc_meminfo; @@ -531,30 +540,33 @@ static PSI_file_info all_mysys_files[]= { &key_file_cnf, "cnf", 0} }; +PSI_stage_info *all_mysys_stages[]= +{ + & stage_waiting_for_table_level_lock +}; + void my_init_mysys_psi_keys() { const char* category= "mysys"; int count; - if (PSI_server == NULL) - return; - count= sizeof(all_mysys_mutexes)/sizeof(all_mysys_mutexes[0]); - PSI_server->register_mutex(category, all_mysys_mutexes, count); + mysql_mutex_register(category, all_mysys_mutexes, count); count= sizeof(all_mysys_conds)/sizeof(all_mysys_conds[0]); - PSI_server->register_cond(category, all_mysys_conds, count); + mysql_cond_register(category, all_mysys_conds, count); count= sizeof(all_mysys_rwlocks)/sizeof(all_mysys_rwlocks[0]); - PSI_server->register_rwlock(category, all_mysys_rwlocks, count); + mysql_rwlock_register(category, all_mysys_rwlocks, count); -#ifdef USE_ALARM_THREAD count= sizeof(all_mysys_threads)/sizeof(all_mysys_threads[0]); - PSI_server->register_thread(category, all_mysys_threads, count); -#endif /* USE_ALARM_THREAD */ + mysql_thread_register(category, all_mysys_threads, count); count= sizeof(all_mysys_files)/sizeof(all_mysys_files[0]); - PSI_server->register_file(category, all_mysys_files, count); + mysql_file_register(category, all_mysys_files, count); + + count= array_elements(all_mysys_stages); + mysql_stage_register(category, all_mysys_stages, count); } #endif /* HAVE_PSI_INTERFACE */ diff --git a/mysys/my_lib.c b/mysys/my_lib.c index 08ff6a94823..8daed1101ec 100644 --- a/mysys/my_lib.c +++ b/mysys/my_lib.c @@ -14,7 +14,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -/* TODO: check for overun of memory for names. */ +/* TODO: check for overrun of memory for names. */ #include "mysys_priv.h" #include <m_string.h> @@ -22,10 +22,8 @@ #include "mysys_err.h" #if defined(HAVE_DIRENT_H) # include <dirent.h> -# define NAMLEN(dirent) strlen((dirent)->d_name) #else # define dirent direct -# define NAMLEN(dirent) (dirent)->d_namlen # if defined(HAVE_SYS_NDIR_H) # include <sys/ndir.h> # endif @@ -60,25 +58,29 @@ static int comp_names(struct fileinfo *a,struct fileinfo *b); +typedef struct { + MY_DIR dir; + DYNAMIC_ARRAY array; + MEM_ROOT root; +} MY_DIR_HANDLE; - /* We need this because program don't know with malloc we used */ +/* We need this because the caller doesn't know which malloc we've used */ -void my_dirend(MY_DIR *buffer) +void my_dirend(MY_DIR *dir) { + MY_DIR_HANDLE *dirh= (MY_DIR_HANDLE*) dir; DBUG_ENTER("my_dirend"); - if (buffer) + if (dirh) { - delete_dynamic((DYNAMIC_ARRAY*)((char*)buffer + - ALIGN_SIZE(sizeof(MY_DIR)))); - free_root((MEM_ROOT*)((char*)buffer + ALIGN_SIZE(sizeof(MY_DIR)) + - ALIGN_SIZE(sizeof(DYNAMIC_ARRAY))), MYF(0)); - my_free(buffer); + delete_dynamic(&dirh->array); + free_root(&dirh->root, MYF(0)); + my_free(dirh); } DBUG_VOID_RETURN; } /* my_dirend */ - /* Compare in sort of filenames */ + /* Compare in sort of filenames */ static int comp_names(struct fileinfo *a, struct fileinfo *b) { @@ -88,119 +90,106 @@ static int comp_names(struct fileinfo *a, struct fileinfo *b) #if !defined(_WIN32) +static char *directory_file_name (char * dst, const char *src) +{ + /* Process as Unix format: just remove test the final slash. */ + char *end; + DBUG_ASSERT(strlen(src) < (FN_REFLEN + 1)); + + if (src[0] == 0) + src= (char*) "."; /* Use empty as current */ + end= strnmov(dst, src, FN_REFLEN + 1); + if (end[-1] != FN_LIBCHAR) + { + *end++= FN_LIBCHAR; /* Add last '/' */ + *end='\0'; + } + return end; +} + MY_DIR *my_dir(const char *path, myf MyFlags) { - char *buffer; - MY_DIR *result= 0; + MY_DIR_HANDLE *dirh= 0; FILEINFO finfo; - DYNAMIC_ARRAY *dir_entries_storage; - MEM_ROOT *names_storage; DIR *dirp; struct dirent *dp; char tmp_path[FN_REFLEN + 2], *tmp_file; char dirent_tmp[sizeof(struct dirent)+_POSIX_PATH_MAX+1]; DBUG_ENTER("my_dir"); - DBUG_PRINT("my",("path: '%s' MyFlags: %d",path,MyFlags)); + DBUG_PRINT("my",("path: '%s' MyFlags: %lu",path,MyFlags)); - dirp = opendir(directory_file_name(tmp_path,(char *) path)); -#if defined(__amiga__) - if ((dirp->dd_fd) < 0) /* Directory doesn't exists */ - goto error; -#endif - if (dirp == NULL || - ! (buffer= my_malloc(ALIGN_SIZE(sizeof(MY_DIR)) + - ALIGN_SIZE(sizeof(DYNAMIC_ARRAY)) + - sizeof(MEM_ROOT), MyFlags))) + tmp_file= directory_file_name(tmp_path, path); + + if (!(dirp= opendir(tmp_path))) goto error; - dir_entries_storage= (DYNAMIC_ARRAY*)(buffer + ALIGN_SIZE(sizeof(MY_DIR))); - names_storage= (MEM_ROOT*)(buffer + ALIGN_SIZE(sizeof(MY_DIR)) + - ALIGN_SIZE(sizeof(DYNAMIC_ARRAY))); + if (!(dirh= my_malloc(sizeof(*dirh), MyFlags | MY_ZEROFILL))) + goto error; - if (my_init_dynamic_array(dir_entries_storage, sizeof(FILEINFO), - ENTRIES_START_SIZE, ENTRIES_INCREMENT)) - { - my_free(buffer); + if (my_init_dynamic_array(&dirh->array, sizeof(FILEINFO), + ENTRIES_START_SIZE, ENTRIES_INCREMENT, + MYF(MyFlags))) goto error; - } - init_alloc_root(names_storage, NAMES_START_SIZE, NAMES_START_SIZE); - /* MY_DIR structure is allocated and completly initialized at this point */ - result= (MY_DIR*)buffer; - - tmp_file=strend(tmp_path); + init_alloc_root(&dirh->root, NAMES_START_SIZE, NAMES_START_SIZE, + MYF(MyFlags)); dp= (struct dirent*) dirent_tmp; while (!(READDIR(dirp,(struct dirent*) dirent_tmp,dp))) { - if (!(finfo.name= strdup_root(names_storage, dp->d_name))) - goto error; + MY_STAT statbuf, *mystat= 0; + if (dp->d_name[0] == '.' && + (dp->d_name[1] == '\0' || + (dp->d_name[1] == '.' && dp->d_name[2] == '\0'))) + continue; /* . or .. */ + if (MyFlags & MY_WANT_STAT) { - if (!(finfo.mystat= (MY_STAT*)alloc_root(names_storage, - sizeof(MY_STAT)))) - goto error; - - bzero(finfo.mystat, sizeof(MY_STAT)); - (void) strmov(tmp_file,dp->d_name); - (void) my_stat(tmp_path, finfo.mystat, MyFlags); - if (!(finfo.mystat->st_mode & MY_S_IREAD)) + mystat= &statbuf; + bzero(mystat, sizeof(*mystat)); + (void) strmov(tmp_file, dp->d_name); + (void) my_stat(tmp_path, mystat, MyFlags); + if (!(mystat->st_mode & MY_S_IREAD)) continue; } - else - finfo.mystat= NULL; - if (push_dynamic(dir_entries_storage, (uchar*)&finfo)) + if (!(finfo.name= strdup_root(&dirh->root, dp->d_name))) + goto error; + + if (mystat && + !((mystat= memdup_root(&dirh->root, mystat, sizeof(*mystat))))) + goto error; + + finfo.mystat= mystat; + + if (push_dynamic(&dirh->array, (uchar*)&finfo)) goto error; } (void) closedir(dirp); - result->dir_entry= (FILEINFO *)dir_entries_storage->buffer; - result->number_off_files= dir_entries_storage->elements; - if (!(MyFlags & MY_DONT_SORT)) - my_qsort((void *) result->dir_entry, result->number_off_files, - sizeof(FILEINFO), (qsort_cmp) comp_names); - DBUG_RETURN(result); + if (MyFlags & MY_WANT_SORT) + sort_dynamic(&dirh->array, (qsort_cmp) comp_names); + + dirh->dir.dir_entry= dynamic_element(&dirh->array, 0, FILEINFO *); + dirh->dir.number_of_files= dirh->array.elements; + + DBUG_RETURN(&dirh->dir); error: my_errno=errno; if (dirp) (void) closedir(dirp); - my_dirend(result); + my_dirend(&dirh->dir); if (MyFlags & (MY_FAE | MY_WME)) - my_error(EE_DIR,MYF(ME_BELL+ME_WAITTANG),path,my_errno); - DBUG_RETURN((MY_DIR *) NULL); + my_error(EE_DIR, MYF(ME_BELL | ME_WAITTANG), path, my_errno); + DBUG_RETURN(NULL); } /* my_dir */ -/* - * Convert from directory name to filename. - * On UNIX, it's simple: just make sure there is a terminating / - - * Returns pointer to dst; - */ - -char * directory_file_name (char * dst, const char *src) -{ - /* Process as Unix format: just remove test the final slash. */ - char *end; - DBUG_ASSERT(strlen(src) < (FN_REFLEN + 1)); - - if (src[0] == 0) - src= (char*) "."; /* Use empty as current */ - end= strnmov(dst, src, FN_REFLEN + 1); - if (end[-1] != FN_LIBCHAR) - { - end[0]=FN_LIBCHAR; /* Add last '/' */ - end[1]='\0'; - } - return dst; -} - #else /* @@ -211,18 +200,11 @@ char * directory_file_name (char * dst, const char *src) MY_DIR *my_dir(const char *path, myf MyFlags) { - char *buffer; - MY_DIR *result= 0; + MY_DIR_HANDLE *dirh= 0; FILEINFO finfo; - DYNAMIC_ARRAY *dir_entries_storage; - MEM_ROOT *names_storage; -#ifdef __BORLANDC__ - struct ffblk find; -#else struct _finddata_t find; -#endif ushort mode; - char tmp_path[FN_REFLEN],*tmp_file,attrib; + char tmp_path[FN_REFLEN], *tmp_file,attrib; #ifdef _WIN64 __int64 handle; #else @@ -245,31 +227,18 @@ MY_DIR *my_dir(const char *path, myf MyFlags) tmp_file[2]='*'; tmp_file[3]='\0'; - if (!(buffer= my_malloc(ALIGN_SIZE(sizeof(MY_DIR)) + - ALIGN_SIZE(sizeof(DYNAMIC_ARRAY)) + - sizeof(MEM_ROOT), MyFlags))) + if (!(dirh= my_malloc(sizeof(*dirh), MyFlags | MY_ZEROFILL))) goto error; - - dir_entries_storage= (DYNAMIC_ARRAY*)(buffer + ALIGN_SIZE(sizeof(MY_DIR))); - names_storage= (MEM_ROOT*)(buffer + ALIGN_SIZE(sizeof(MY_DIR)) + - ALIGN_SIZE(sizeof(DYNAMIC_ARRAY))); - if (my_init_dynamic_array(dir_entries_storage, sizeof(FILEINFO), - ENTRIES_START_SIZE, ENTRIES_INCREMENT)) - { - my_free(buffer); + if (my_init_dynamic_array(&dirh->array, sizeof(FILEINFO), + ENTRIES_START_SIZE, ENTRIES_INCREMENT, + MYF(MyFlags))) goto error; - } - init_alloc_root(names_storage, NAMES_START_SIZE, NAMES_START_SIZE); - - /* MY_DIR structure is allocated and completly initialized at this point */ - result= (MY_DIR*)buffer; -#ifdef __BORLANDC__ - if ((handle= findfirst(tmp_path,&find,0)) == -1L) -#else + init_alloc_root(&dirh->root, NAMES_START_SIZE, NAMES_START_SIZE, + MYF(MyFlags)); + if ((handle=_findfirst(tmp_path,&find)) == -1L) -#endif { DBUG_PRINT("info", ("findfirst returned error, errno: %d", errno)); if (errno != EINVAL) @@ -282,12 +251,8 @@ MY_DIR *my_dir(const char *path, myf MyFlags) } else { - do { -#ifdef __BORLANDC__ - attrib= find.ff_attrib; -#else attrib= find.attrib; /* Do not show hidden and system files which Windows sometimes create. @@ -296,71 +261,55 @@ MY_DIR *my_dir(const char *path, myf MyFlags) */ if (attrib & (_A_HIDDEN | _A_SYSTEM)) continue; -#endif -#ifdef __BORLANDC__ - if (!(finfo.name= strdup_root(names_storage, find.ff_name))) - goto error; -#else - if (!(finfo.name= strdup_root(names_storage, find.name))) + + if (find.name[0] == '.' && + (find.name[1] == '\0' || + (find.name[1] == '.' && find.name[2] == '\0'))) + continue; /* . or .. */ + + if (!(finfo.name= strdup_root(&dirh->root, find.name))) goto error; -#endif if (MyFlags & MY_WANT_STAT) { - if (!(finfo.mystat= (MY_STAT*)alloc_root(names_storage, - sizeof(MY_STAT)))) + if (!(finfo.mystat= (MY_STAT*)alloc_root(&dirh->root, sizeof(MY_STAT)))) goto error; bzero(finfo.mystat, sizeof(MY_STAT)); -#ifdef __BORLANDC__ - finfo.mystat->st_size=find.ff_fsize; -#else finfo.mystat->st_size=find.size; -#endif mode= MY_S_IREAD; if (!(attrib & _A_RDONLY)) mode|= MY_S_IWRITE; if (attrib & _A_SUBDIR) mode|= MY_S_IFDIR; finfo.mystat->st_mode= mode; -#ifdef __BORLANDC__ - finfo.mystat->st_mtime= ((uint32) find.ff_ftime); -#else finfo.mystat->st_mtime= ((uint32) find.time_write); -#endif } else finfo.mystat= NULL; - if (push_dynamic(dir_entries_storage, (uchar*)&finfo)) + if (push_dynamic(&dirh->array, (uchar*)&finfo)) goto error; } -#ifdef __BORLANDC__ - while (findnext(&find) == 0); -#else while (_findnext(handle,&find) == 0); - _findclose(handle); -#endif } - result->dir_entry= (FILEINFO *)dir_entries_storage->buffer; - result->number_off_files= dir_entries_storage->elements; + if (MyFlags & MY_WANT_SORT) + sort_dynamic(&dirh->array, (qsort_cmp) comp_names); + + dirh->dir.dir_entry= dynamic_element(&dirh->array, 0, FILEINFO *); + dirh->dir.number_of_files= dirh->array.elements; - if (!(MyFlags & MY_DONT_SORT)) - my_qsort((void *) result->dir_entry, result->number_off_files, - sizeof(FILEINFO), (qsort_cmp) comp_names); - DBUG_PRINT("exit", ("found %d files", result->number_off_files)); - DBUG_RETURN(result); + DBUG_PRINT("exit", ("found %d files", dirh->dir.number_of_files)); + DBUG_RETURN(&dirh->dir); error: my_errno=errno; -#ifndef __BORLANDC__ if (handle != -1) _findclose(handle); -#endif - my_dirend(result); - if (MyFlags & MY_FAE+MY_WME) - my_error(EE_DIR,MYF(ME_BELL+ME_WAITTANG),path,errno); - DBUG_RETURN((MY_DIR *) NULL); + my_dirend(&dirh->dir); + if (MyFlags & (MY_FAE | MY_WME)) + my_error(EE_DIR,MYF(ME_BELL | ME_WAITTANG), path, errno); + DBUG_RETURN(NULL); } /* my_dir */ #endif /* _WIN32 */ @@ -375,7 +324,7 @@ int my_fstat(File Filedes, MY_STAT *stat_area, myf MyFlags __attribute__((unused))) { DBUG_ENTER("my_fstat"); - DBUG_PRINT("my",("fd: %d MyFlags: %d", Filedes, MyFlags)); + DBUG_PRINT("my",("fd: %d MyFlags: %lu", Filedes, MyFlags)); #ifdef _WIN32 DBUG_RETURN(my_win_fstat(Filedes, stat_area)); #else @@ -388,7 +337,7 @@ MY_STAT *my_stat(const char *path, MY_STAT *stat_area, myf my_flags) { int m_used; DBUG_ENTER("my_stat"); - DBUG_PRINT("my", ("path: '%s' stat_area: 0x%lx MyFlags: %d", path, + DBUG_PRINT("my", ("path: '%s' stat_area: 0x%lx MyFlags: %lu", path, (long) stat_area, my_flags)); if ((m_used= (stat_area == NULL))) diff --git a/mysys/my_lock.c b/mysys/my_lock.c index 54ec3838b58..082d8e9f5a0 100644 --- a/mysys/my_lock.c +++ b/mysys/my_lock.c @@ -131,7 +131,7 @@ error: RETURN VALUE 0 Success - -1 An error has occured and 'my_errno' is set + -1 An error has occurred and 'my_errno' is set to indicate the actual error code. */ @@ -144,7 +144,7 @@ int my_lock(File fd, int locktype, my_off_t start, my_off_t length, #endif DBUG_ENTER("my_lock"); - DBUG_PRINT("my",("fd: %d Op: %d start: %ld Length: %ld MyFlags: %d", + DBUG_PRINT("my",("fd: %d Op: %d start: %ld Length: %ld MyFlags: %lu", fd,locktype,(long) start,(long) length,MyFlags)); if (my_disable_locking && ! (MyFlags & MY_FORCE_LOCK)) DBUG_RETURN(0); @@ -203,7 +203,7 @@ int my_lock(File fd, int locktype, my_off_t start, my_off_t length, == MY_FILEPOS_ERROR) { /* - If an error has occured in my_seek then we will already + If an error has occurred in my_seek then we will already have an error code in my_errno; Just return error code. */ DBUG_RETURN(-1); diff --git a/mysys/my_malloc.c b/mysys/my_malloc.c index 5264210a6b4..99a15e6f432 100644 --- a/mysys/my_malloc.c +++ b/mysys/my_malloc.c @@ -1,4 +1,5 @@ -/* Copyright (c) 2000, 2013, Oracle and/or its affiliates +/* + Copyright (c) 2000, 2013, Oracle and/or its affiliates Copyright (c) 2009, 2014, SkySQL Ab This program is free software; you can redistribute it and/or modify @@ -18,6 +19,61 @@ #include "mysys_err.h" #include <m_string.h> +/* If we have our own safemalloc (for debugging) */ +#if defined(SAFEMALLOC) +#define MALLOC_SIZE_AND_FLAG(p,b) sf_malloc_usable_size(p,b) +#define MALLOC_PREFIX_SIZE 0 +#define MALLOC_STORE_SIZE(a,b,c,d) +#define MALLOC_FIX_POINTER_FOR_FREE(a) a +#else +/* + * We use double as prefix size as this guarantees the correct + * alignment on all platforms and will optimize things for + * memcpy(), memcmp() etc. + */ +#define MALLOC_PREFIX_SIZE (sizeof(double)) +#define MALLOC_SIZE(p) (*(size_t*) ((char*)(p) - MALLOC_PREFIX_SIZE)) +#define MALLOC_STORE_SIZE(p, type_of_p, size, flag) \ +{\ + *(size_t*) p= (size) | (flag); \ + (p)= (type_of_p) (((char*) (p)) + MALLOC_PREFIX_SIZE); \ +} +static inline size_t malloc_size_and_flag(void *p, my_bool *is_thread_specific) +{ + size_t size= MALLOC_SIZE(p); + *is_thread_specific= (size & 1); + return size & ~ (ulonglong) 1; +} +#define MALLOC_SIZE_AND_FLAG(p,b) malloc_size_and_flag(p, b); +#define MALLOC_FIX_POINTER_FOR_FREE(p) (((char*) (p)) - MALLOC_PREFIX_SIZE) +#endif /* SAFEMALLOC */ + + +/** + Inform application that memory usage has changed + + @param size Size of memory segment allocated or freed + @param flag 1 if thread specific (allocated by MY_THREAD_SPECIFIC), + 0 if system specific. + + The type os size is long long, to be able to handle negative numbers to + decrement the memory usage + + @return 0 - ok + 1 - failure, abort the allocation +*/ +static void dummy(long long size __attribute__((unused)), + my_bool is_thread_specific __attribute__((unused))) +{} + +static MALLOC_SIZE_CB update_malloc_size= dummy; + +void set_malloc_size_cb(MALLOC_SIZE_CB func) +{ + update_malloc_size= func ? func : dummy; +} + + /** Allocate a sized block of memory. @@ -30,7 +86,9 @@ void *my_malloc(size_t size, myf my_flags) { void* point; DBUG_ENTER("my_malloc"); - DBUG_PRINT("my",("size: %lu my_flags: %d", (ulong) size, my_flags)); + DBUG_PRINT("my",("size: %lu my_flags: %lu", (ulong) size, my_flags)); + compile_time_assert(sizeof(size_t) <= sizeof(double)); + if (!(my_flags & (MY_WME | MY_FAE))) my_flags|= my_global_flags; @@ -38,17 +96,9 @@ void *my_malloc(size_t size, myf my_flags) if (!size) size=1; - point= sf_malloc(size); - DBUG_EXECUTE_IF("simulate_out_of_memory", - { - my_free(point); - point= NULL; - }); - DBUG_EXECUTE_IF("simulate_persistent_out_of_memory", - { - free(point); - point= NULL; - }); + /* We have to align size to be able to store markers in it */ + size= ALIGN_SIZE(size); + point= sf_malloc(size + MALLOC_PREFIX_SIZE, my_flags); if (point == NULL) { @@ -58,13 +108,25 @@ void *my_malloc(size_t size, myf my_flags) if (my_flags & (MY_FAE+MY_WME)) my_error(EE_OUTOFMEMORY, MYF(ME_BELL + ME_WAITTANG + ME_NOREFRESH + ME_FATALERROR),size); - DBUG_EXECUTE_IF("simulate_out_of_memory", - DBUG_SET("-d,simulate_out_of_memory");); if (my_flags & MY_FAE) - exit(1); + abort(); + } + else + { + MALLOC_STORE_SIZE(point, void*, size, + MY_TEST(my_flags & MY_THREAD_SPECIFIC)); + update_malloc_size(size + MALLOC_PREFIX_SIZE, + MY_TEST(my_flags & MY_THREAD_SPECIFIC)); + TRASH_ALLOC(point, size); + DBUG_EXECUTE_IF("simulate_out_of_memory", + { + /* my_free() handles memory accounting */ + my_free(point); + point= NULL; + }); + if (my_flags & MY_ZEROFILL) + bzero(point, size); } - else if (my_flags & MY_ZEROFILL) - bzero(point, size); DBUG_PRINT("exit",("ptr: %p", point)); DBUG_RETURN(point); } @@ -83,23 +145,54 @@ void *my_malloc(size_t size, myf my_flags) void *my_realloc(void *oldpoint, size_t size, myf my_flags) { void *point; + size_t old_size; + my_bool old_flags; DBUG_ENTER("my_realloc"); - DBUG_PRINT("my",("ptr: %p size: %lu my_flags: %d", oldpoint, + DBUG_PRINT("my",("ptr: %p size: %lu my_flags: %lu", oldpoint, (ulong) size, my_flags)); DBUG_ASSERT(size > 0); if (!oldpoint && (my_flags & MY_ALLOW_ZERO_PTR)) DBUG_RETURN(my_malloc(size, my_flags)); - if ((point= sf_realloc(oldpoint, size)) == NULL) + + size= ALIGN_SIZE(size); + old_size= MALLOC_SIZE_AND_FLAG(oldpoint, &old_flags); + /* + Test that the new and old area are the same, if not MY_THREAD_MOVE is + given + */ + DBUG_ASSERT((MY_TEST(my_flags & MY_THREAD_SPECIFIC) == old_flags) || + (my_flags & MY_THREAD_MOVE)); + if ((point= sf_realloc(MALLOC_FIX_POINTER_FOR_FREE(oldpoint), + size + MALLOC_PREFIX_SIZE, my_flags)) == NULL) { if (my_flags & MY_FREE_ON_ERROR) + { + /* my_free will take care of size accounting */ my_free(oldpoint); + oldpoint= 0; + } if (my_flags & MY_HOLD_ON_ERROR) DBUG_RETURN(oldpoint); my_errno=errno; if (my_flags & (MY_FAE+MY_WME)) my_error(EE_OUTOFMEMORY, MYF(ME_BELL + ME_WAITTANG + ME_FATALERROR), size); } + else + { + MALLOC_STORE_SIZE(point, void*, size, + MY_TEST(my_flags & MY_THREAD_SPECIFIC)); + if (MY_TEST(my_flags & MY_THREAD_SPECIFIC) != old_flags) + { + /* memory moved between system and thread specific */ + update_malloc_size(-(longlong) old_size - MALLOC_PREFIX_SIZE, old_flags); + update_malloc_size((longlong) size + MALLOC_PREFIX_SIZE, + MY_TEST(my_flags & MY_THREAD_SPECIFIC)); + } + else + update_malloc_size((longlong)size - (longlong)old_size, old_flags); + } + DBUG_PRINT("exit",("ptr: %p", point)); DBUG_RETURN(point); } @@ -116,7 +209,14 @@ void my_free(void *ptr) { DBUG_ENTER("my_free"); DBUG_PRINT("my",("ptr: %p", ptr)); - sf_free(ptr); + if (ptr) + { + size_t old_size; + my_bool old_flags; + old_size= MALLOC_SIZE_AND_FLAG(ptr, &old_flags); + update_malloc_size(- (longlong) old_size - MALLOC_PREFIX_SIZE, old_flags); + sf_free(MALLOC_FIX_POINTER_FOR_FREE(ptr)); + } DBUG_VOID_RETURN; } diff --git a/mysys/my_open.c b/mysys/my_open.c index cafb94ef558..3999810eb2e 100644 --- a/mysys/my_open.c +++ b/mysys/my_open.c @@ -44,7 +44,7 @@ File my_open(const char *FileName, int Flags, myf MyFlags) { File fd; DBUG_ENTER("my_open"); - DBUG_PRINT("my",("Name: '%s' Flags: %d MyFlags: %d", + DBUG_PRINT("my",("Name: '%s' Flags: %d MyFlags: %lu", FileName, Flags, MyFlags)); if (!(MyFlags & (MY_WME | MY_FAE | MY_FFNF))) MyFlags|= my_global_flags; @@ -52,9 +52,9 @@ File my_open(const char *FileName, int Flags, myf MyFlags) fd= my_win_open(FileName, Flags); #else if (MyFlags & MY_NOSYMLINKS) - fd = open_nosymlinks(FileName, Flags, my_umask); + fd = open_nosymlinks(FileName, Flags | O_CLOEXEC, my_umask); else - fd = open(FileName, Flags, my_umask); + fd = open(FileName, Flags | O_CLOEXEC, my_umask); #endif fd= my_register_filename(fd, FileName, FILE_BY_OPEN, @@ -77,7 +77,7 @@ int my_close(File fd, myf MyFlags) { int err; DBUG_ENTER("my_close"); - DBUG_PRINT("my",("fd: %d MyFlags: %d",fd, MyFlags)); + DBUG_PRINT("my",("fd: %d MyFlags: %lu",fd, MyFlags)); if (!(MyFlags & (MY_WME | MY_FAE))) MyFlags|= my_global_flags; diff --git a/mysys/my_pread.c b/mysys/my_pread.c index 52d5c382960..2b3bfdc9e3d 100644 --- a/mysys/my_pread.c +++ b/mysys/my_pread.c @@ -28,7 +28,7 @@ SYNOPSIOS my_pread() - Filedes File decsriptor + Filedes File descriptor Buffer Buffer to read data into Count Number of bytes to read offset Position to read from @@ -51,7 +51,7 @@ size_t my_pread(File Filedes, uchar *Buffer, size_t Count, my_off_t offset, DBUG_ENTER("my_pread"); - DBUG_PRINT("my",("fd: %d Seek: %llu Buffer: %p Count: %lu MyFlags: %d", + DBUG_PRINT("my",("fd: %d Seek: %llu Buffer: %p Count: %lu MyFlags: %lu", Filedes, (ulonglong)offset, Buffer, (ulong)Count, MyFlags)); if (!(MyFlags & (MY_WME | MY_FAE | MY_FNABP))) @@ -120,7 +120,7 @@ size_t my_pread(File Filedes, uchar *Buffer, size_t Count, my_off_t offset, SYNOPSIOS my_pwrite() - Filedes File decsriptor + Filedes File descriptor Buffer Buffer to write data from Count Number of bytes to write offset Position to write to @@ -142,7 +142,7 @@ size_t my_pwrite(int Filedes, const uchar *Buffer, size_t Count, size_t writtenbytes, written; uint errors; DBUG_ENTER("my_pwrite"); - DBUG_PRINT("my",("fd: %d Seek: %llu Buffer: %p Count: %lu MyFlags: %d", + DBUG_PRINT("my",("fd: %d Seek: %llu Buffer: %p Count: %lu MyFlags: %lu", Filedes, (ulonglong)offset, Buffer, (ulong)Count, MyFlags)); errors= 0; written= 0; diff --git a/mysys/my_rdtsc.c b/mysys/my_rdtsc.c index ad11e8c6a6c..4228973caa8 100644 --- a/mysys/my_rdtsc.c +++ b/mysys/my_rdtsc.c @@ -249,6 +249,13 @@ ulonglong my_timer_cycles(void) clock_gettime(CLOCK_SGI_CYCLE, &tp); return (ulonglong) tp.tv_sec * 1000000000 + (ulonglong) tp.tv_nsec; } +#elif defined(__GNUC__) && defined(__s390__) + /* covers both s390 and s390x */ + { + ulonglong result; + __asm__ __volatile__ ("stck %0" : "=Q" (result) : : "cc"); + return result; + } #elif defined(HAVE_SYS_TIMES_H) && defined(HAVE_GETHRTIME) /* gethrtime may appear as either cycle or nanosecond counter */ return (ulonglong) gethrtime(); @@ -558,6 +565,8 @@ void my_timer_init(MY_TIMER_INFO *mti) mti->cycles.routine= MY_TIMER_ROUTINE_ASM_GCC_SPARC32; #elif defined(__sgi) && defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_SGI_CYCLE) mti->cycles.routine= MY_TIMER_ROUTINE_SGI_CYCLE; +#elif defined(__GNUC__) && defined(__s390__) + mti->cycles.routine= MY_TIMER_ROUTINE_ASM_S390; #elif defined(HAVE_SYS_TIMES_H) && defined(HAVE_GETHRTIME) mti->cycles.routine= MY_TIMER_ROUTINE_GETHRTIME; #else @@ -753,7 +762,6 @@ void my_timer_init(MY_TIMER_INFO *mti) mti->cycles.frequency= mti->microseconds.frequency; else { - ulonglong time1, time2; time1= my_timer_init_frequency(mti); /* Repeat once in case there was an interruption. */ time2= my_timer_init_frequency(mti); @@ -775,8 +783,7 @@ void my_timer_init(MY_TIMER_INFO *mti) && mti->microseconds.routine && mti->cycles.routine) { - int i; - ulonglong time1, time2, time3, time4; + ulonglong time3, time4; time1= my_timer_cycles(); time2= my_timer_milliseconds(); time3= time2; /* Avoids a Microsoft/IBM compiler warning */ @@ -801,8 +808,7 @@ void my_timer_init(MY_TIMER_INFO *mti) && mti->microseconds.routine && mti->cycles.routine) { - int i; - ulonglong time1, time2, time3, time4; + ulonglong time3, time4; time1= my_timer_cycles(); time2= my_timer_ticks(); time3= time2; /* Avoids a Microsoft/IBM compiler warning */ diff --git a/mysys/my_read.c b/mysys/my_read.c index 47f2d725f65..58ab9070c8c 100644 --- a/mysys/my_read.c +++ b/mysys/my_read.c @@ -38,7 +38,7 @@ size_t my_read(File Filedes, uchar *Buffer, size_t Count, myf MyFlags) size_t readbytes, save_count= 0; DBUG_ENTER("my_read"); DBUG_PRINT("my",("fd: %d Buffer: %p Count: %lu MyFlags: %lu", - Filedes, Buffer, (ulong) Count, (ulong) MyFlags)); + Filedes, Buffer, (ulong) Count, MyFlags)); if (!(MyFlags & (MY_WME | MY_FAE | MY_FNABP))) MyFlags|= my_global_flags; diff --git a/mysys/my_redel.c b/mysys/my_redel.c index e5e4f48d9d5..9091c74e6b5 100644 --- a/mysys/my_redel.c +++ b/mysys/my_redel.c @@ -32,10 +32,10 @@ struct utimbuf { /* Rename with copy stat form old file - Copy stats from old file to new file, deletes orginal and + Copy stats from old file to new file, deletes original and changes new file name to old file name - if MY_REDEL_MAKE_COPY is given, then the orginal file + if MY_REDEL_MAKE_COPY is given, then the original file is renamed to org_name-'current_time'.BAK */ @@ -46,7 +46,7 @@ int my_redel(const char *org_name, const char *tmp_name, { int error=1; DBUG_ENTER("my_redel"); - DBUG_PRINT("my",("org_name: '%s' tmp_name: '%s' MyFlags: %d", + DBUG_PRINT("my",("org_name: '%s' tmp_name: '%s' MyFlags: %lu", org_name,tmp_name,MyFlags)); if (!my_disable_copystat_in_redel && diff --git a/mysys/my_rename.c b/mysys/my_rename.c index b89bc4c8fbd..17f693629a8 100644 --- a/mysys/my_rename.c +++ b/mysys/my_rename.c @@ -25,22 +25,24 @@ int my_rename(const char *from, const char *to, myf MyFlags) { int error = 0; DBUG_ENTER("my_rename"); - DBUG_PRINT("my",("from %s to %s MyFlags %d", from, to, MyFlags)); + DBUG_PRINT("my",("from %s to %s MyFlags %lu", from, to, MyFlags)); -#if defined(HAVE_RENAME) #if defined(__WIN__) - /* - On windows we can't rename over an existing file: - Remove any conflicting files: - */ - (void) my_delete(to, MYF(0)); -#endif + if (!MoveFileEx(from, to, MOVEFILE_COPY_ALLOWED | + MOVEFILE_REPLACE_EXISTING)) + { + my_osmaperr(GetLastError()); +#elif defined(HAVE_RENAME) if (rename(from,to)) + { #else if (link(from, to) || unlink(from)) -#endif { - my_errno=errno; +#endif + if (errno == ENOENT && !access(from, F_OK)) + my_errno= ENOTDIR; + else + my_errno= errno; error = -1; if (MyFlags & (MY_FAE+MY_WME)) my_error(EE_LINK, MYF(ME_BELL+ME_WAITTANG),from,to,my_errno); diff --git a/mysys/my_rnd.c b/mysys/my_rnd.c index d460ff533ca..c38682a2012 100644 --- a/mysys/my_rnd.c +++ b/mysys/my_rnd.c @@ -14,6 +14,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */ #include "mysys_priv.h" +#include <my_rnd.h> #include <m_string.h> /* @@ -45,11 +46,52 @@ void my_rnd_init(struct my_rnd_struct *rand_st, ulong seed1, ulong seed2) RETURN VALUE generated pseudo random number + + NOTE: + This is codes so that it can be called by two threads at the same time + with minimum impact. + (As the number is supposed to be random, it doesn't matter much if + rand->seed1 or rand->seed2 are updated with slightly wrong numbers or + if two threads gets the same number. */ double my_rnd(struct my_rnd_struct *rand_st) { - rand_st->seed1=(rand_st->seed1*3+rand_st->seed2) % rand_st->max_value; - rand_st->seed2=(rand_st->seed1+rand_st->seed2+33) % rand_st->max_value; - return (((double) rand_st->seed1)/rand_st->max_value_dbl); + unsigned long seed1; + seed1= (rand_st->seed1*3+rand_st->seed2) % rand_st->max_value; + rand_st->seed2=(seed1+rand_st->seed2+33) % rand_st->max_value; + rand_st->seed1= seed1; + return (((double) seed1)/rand_st->max_value_dbl); +} + + +/** + Generate a random number using the OpenSSL/yaSSL supplied + random number generator if available. + + @param rand_st [INOUT] Structure used for number generation + only if none of the SSL libraries are + available. + + @retval Generated random number. +*/ + +double my_rnd_ssl(struct my_rnd_struct *rand_st) +{ + +#if defined(HAVE_YASSL) || defined(HAVE_OPENSSL) + int rc; + unsigned int res; + +#if defined(HAVE_YASSL) + rc= yaSSL::RAND_bytes((unsigned char *) &res, sizeof (unsigned int)); +#else + rc= RAND_bytes((unsigned char *) &res, sizeof (unsigned int)); +#endif /* HAVE_YASSL */ + + if (rc) + return (double)res / (double)UINT_MAX; +#endif /* defined(HAVE_YASSL) || defined(HAVE_OPENSSL) */ + + return my_rnd(rand_st); } diff --git a/mysys/my_seek.c b/mysys/my_seek.c index 1033d7ac806..e15179a408f 100644 --- a/mysys/my_seek.c +++ b/mysys/my_seek.c @@ -48,7 +48,7 @@ my_off_t my_seek(File fd, my_off_t pos, int whence, myf MyFlags) { os_off_t newpos= -1; DBUG_ENTER("my_seek"); - DBUG_PRINT("my",("fd: %d Pos: %llu Whence: %d MyFlags: %d", + DBUG_PRINT("my",("fd: %d Pos: %llu Whence: %d MyFlags: %lu", fd, (ulonglong) pos, whence, MyFlags)); DBUG_ASSERT(pos != MY_FILEPOS_ERROR); /* safety check */ @@ -84,7 +84,7 @@ my_off_t my_tell(File fd, myf MyFlags) { os_off_t pos; DBUG_ENTER("my_tell"); - DBUG_PRINT("my",("fd: %d MyFlags: %d",fd, MyFlags)); + DBUG_PRINT("my",("fd: %d MyFlags: %lu",fd, MyFlags)); DBUG_ASSERT(fd >= 0); #if defined (HAVE_TELL) && !defined (_WIN32) pos= tell(fd); diff --git a/mysys/my_static.c b/mysys/my_static.c index 48b1e5b8dd9..9236c1395fb 100644 --- a/mysys/my_static.c +++ b/mysys/my_static.c @@ -1,6 +1,5 @@ -/* - Copyright (c) 2000-2008 MySQL AB, 2009 Sun Microsystems, Inc. - Use is subject to license terms. +/* Copyright (c) 2000, 2011, Oracle and/or its affiliates. + Copyright (c) 2009, 2012, Monty Program Ab. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -73,24 +72,24 @@ ulong my_time_to_wait_for_lock=2; /* In seconds */ #ifdef SHARED_LIBRARY const char *globerrs[GLOBERRS]; /* my_error_messages is here */ #endif -void (*my_abort_hook)(int) = (void(*)(int)) exit; void (*error_handler_hook)(uint error, const char *str, myf MyFlags)= my_message_stderr; void (*fatal_error_handler_hook)(uint error, const char *str, myf MyFlags)= my_message_stderr; -static const char *proc_info_dummy(void *a __attribute__((unused)), - const char *b __attribute__((unused)), - const char *c __attribute__((unused)), - const char *d __attribute__((unused)), - const unsigned int e __attribute__((unused))) +static void proc_info_dummy(void *a __attribute__((unused)), + const PSI_stage_info *b __attribute__((unused)), + PSI_stage_info *c __attribute__((unused)), + const char *d __attribute__((unused)), + const char *e __attribute__((unused)), + const unsigned int f __attribute__((unused))) { - return 0; + return; } /* this is to be able to call set_thd_proc_info from the C code */ -const char *(*proc_info_hook)(void *, const char *, const char *, const char *, - const unsigned int)= proc_info_dummy; +void (*proc_info_hook)(void *, const PSI_stage_info *, PSI_stage_info *, + const char *, const char *, const unsigned int)= proc_info_dummy; void (*debug_sync_C_callback_ptr)(MYSQL_THD, const char *, size_t)= 0; /* How to disable options */ @@ -100,31 +99,3 @@ my_bool my_disable_async_io=0; my_bool my_disable_flush_key_blocks=0; my_bool my_disable_symlinks=0; my_bool my_disable_copystat_in_redel=0; - -/* - Note that PSI_hook and PSI_server are unconditionally - (no ifdef HAVE_PSI_INTERFACE) defined. - This is to ensure binary compatibility between the server and plugins, - in the case when: - - the server is not compiled with HAVE_PSI_INTERFACE - - a plugin is compiled with HAVE_PSI_INTERFACE - See the doxygen documentation for the performance schema. -*/ - -/** - Hook for the instrumentation interface. - Code implementing the instrumentation interface should register here. -*/ -struct PSI_bootstrap *PSI_hook= NULL; - -/** - Instance of the instrumentation interface for the MySQL server. - @todo This is currently a global variable, which is handy when - compiling instrumented code that is bundled with the server. - When dynamic plugin are truly supported, this variable will need - to be replaced by a macro, so that each XYZ plugin can have it's own - xyz_psi_server variable, obtained from PSI_bootstrap::get_interface() - with the version used at compile time for plugin XYZ. -*/ -PSI *PSI_server= NULL; - diff --git a/mysys/my_symlink.c b/mysys/my_symlink.c index 12959f1f24a..8580fac3595 100644 --- a/mysys/my_symlink.c +++ b/mysys/my_symlink.c @@ -129,6 +129,11 @@ int my_is_symlink(const char *filename __attribute__((unused))) to is guaranteed to never set to a string longer than FN_REFLEN (including the end \0) + + On error returns -1, unless error is file not found, in which case it + is 1. + + Sets my_errno to specific error number. */ int my_realpath(char *to, const char *filename, myf MyFlags) @@ -154,7 +159,10 @@ int my_realpath(char *to, const char *filename, myf MyFlags) if (MyFlags & MY_WME) my_error(EE_REALPATH, MYF(0), filename, my_errno); my_load_path(to, filename, NullS); - result= -1; + if (my_errno == ENOENT) + result= 1; + else + result= -1; } DBUG_RETURN(result); #elif defined(_WIN32) @@ -236,7 +244,7 @@ const char *my_open_parent_dir_nosymlinks(const char *pathname, int *pdfd) return pathname + (s - buf); } - fd = openat(dfd, s, O_NOFOLLOW | O_PATH); + fd = openat(dfd, s, O_NOFOLLOW | O_PATH | O_CLOEXEC); if (fd < 0) goto err; diff --git a/mysys/my_symlink2.c b/mysys/my_symlink2.c index c851468ce1b..f1ace5dcd77 100644 --- a/mysys/my_symlink2.c +++ b/mysys/my_symlink2.c @@ -17,7 +17,7 @@ /* Advanced symlink handling. This is used in MyISAM to let users symlinks tables to different disk. - The main idea with these functions is to automaticly create, delete and + The main idea with these functions is to automatically create, delete and rename files and symlinks like they would be one unit. */ diff --git a/mysys/my_sync.c b/mysys/my_sync.c index 77e7befebed..d1e239692f1 100644 --- a/mysys/my_sync.c +++ b/mysys/my_sync.c @@ -65,16 +65,13 @@ int my_sync(File fd, myf my_flags) { int res; DBUG_ENTER("my_sync"); - - DBUG_PRINT("my",("fd: %d my_flags: %d", fd, my_flags)); + DBUG_PRINT("my",("fd: %d my_flags: %lu", fd, my_flags)); if (my_disable_sync) DBUG_RETURN(0); statistic_increment(my_sync_count,&THR_LOCK_open); - DBUG_PRINT("my",("Fd: %d my_flags: %d", fd, my_flags)); - if (before_sync_wait) (*before_sync_wait)(); @@ -158,7 +155,7 @@ int my_sync_dir(const char *dir_name __attribute__((unused)), int res= 0; const char *correct_dir_name; DBUG_ENTER("my_sync_dir"); - DBUG_PRINT("my",("Dir: '%s' my_flags: %d", dir_name, my_flags)); + DBUG_PRINT("my",("Dir: '%s' my_flags: %lu", dir_name, my_flags)); /* Sometimes the path does not contain an explicit directory */ correct_dir_name= (dir_name[0] == 0) ? cur_dir_name : dir_name; /* diff --git a/mysys/my_thr_init.c b/mysys/my_thr_init.c index 55ee1db657e..f827fb136d3 100644 --- a/mysys/my_thr_init.c +++ b/mysys/my_thr_init.c @@ -25,7 +25,7 @@ pthread_key(struct st_my_thread_var*, THR_KEY_mysys); mysql_mutex_t THR_LOCK_malloc, THR_LOCK_open, - THR_LOCK_lock, THR_LOCK_isam, THR_LOCK_myisam, THR_LOCK_heap, + THR_LOCK_lock, THR_LOCK_myisam, THR_LOCK_heap, THR_LOCK_net, THR_LOCK_charset, THR_LOCK_threads, THR_LOCK_myisam_mmap; @@ -38,22 +38,6 @@ mysql_mutex_t LOCK_localtime_r; #ifdef _MSC_VER static void install_sigabrt_handler(); #endif -#ifdef TARGET_OS_LINUX - -/* - Dummy thread spawned in my_thread_global_init() below to avoid - race conditions in NPTL pthread_exit code. -*/ - -static pthread_handler_t -nptl_pthread_exit_hack_handler(void *arg __attribute((unused))) -{ - /* Do nothing! */ - pthread_exit(0); - return 0; -} - -#endif /* TARGET_OS_LINUX */ static uint get_thread_lib(void); @@ -75,7 +59,6 @@ static void my_thread_init_common_mutex(void) { mysql_mutex_init(key_THR_LOCK_open, &THR_LOCK_open, MY_MUTEX_INIT_FAST); mysql_mutex_init(key_THR_LOCK_lock, &THR_LOCK_lock, MY_MUTEX_INIT_FAST); - mysql_mutex_init(key_THR_LOCK_isam, &THR_LOCK_isam, MY_MUTEX_INIT_SLOW); mysql_mutex_init(key_THR_LOCK_myisam, &THR_LOCK_myisam, MY_MUTEX_INIT_SLOW); mysql_mutex_init(key_THR_LOCK_myisam_mmap, &THR_LOCK_myisam_mmap, MY_MUTEX_INIT_FAST); mysql_mutex_init(key_THR_LOCK_heap, &THR_LOCK_heap, MY_MUTEX_INIT_FAST); @@ -90,7 +73,6 @@ void my_thread_destroy_common_mutex(void) { mysql_mutex_destroy(&THR_LOCK_open); mysql_mutex_destroy(&THR_LOCK_lock); - mysql_mutex_destroy(&THR_LOCK_isam); mysql_mutex_destroy(&THR_LOCK_myisam); mysql_mutex_destroy(&THR_LOCK_myisam_mmap); mysql_mutex_destroy(&THR_LOCK_heap); @@ -210,33 +192,6 @@ my_bool my_thread_global_init(void) thd_lib_detected= get_thread_lib(); -#ifdef TARGET_OS_LINUX - /* - BUG#24507: Race conditions inside current NPTL pthread_exit() - implementation. - - To avoid a possible segmentation fault during concurrent - executions of pthread_exit(), a dummy thread is spawned which - initializes internal variables of pthread lib. See bug description - for a full explanation. - - TODO: Remove this code when fixed versions of glibc6 are in common - use. - */ - if (thd_lib_detected == THD_LIB_NPTL) - { - pthread_t dummy_thread; - pthread_attr_t dummy_thread_attr; - - pthread_attr_init(&dummy_thread_attr); - pthread_attr_setdetachstate(&dummy_thread_attr, PTHREAD_CREATE_JOINABLE); - - if (pthread_create(&dummy_thread,&dummy_thread_attr, - nptl_pthread_exit_hack_handler, NULL) == 0) - (void)pthread_join(dummy_thread, NULL); - } -#endif /* TARGET_OS_LINUX */ - my_thread_init_common_mutex(); return 0; @@ -318,7 +273,7 @@ my_bool my_thread_init(void) my_bool error=0; if (!my_thread_global_init_done) - return 1; /* cannot proceed with unintialized library */ + return 1; /* cannot proceed with uninitialized library */ #ifdef EXTRA_DEBUG_THREADS fprintf(stderr,"my_thread_init(): pthread_self: %p\n", pthread_self()); @@ -392,12 +347,16 @@ void my_thread_end(void) This must be done before trashing st_my_thread_var, because the LF_HASH depends on it. */ - if (PSI_server) - PSI_server->delete_current_thread(); + PSI_THREAD_CALL(delete_current_thread)(); #endif + /* + We need to disable DBUG early for this thread to ensure that the + the mutex calls doesn't enable it again + To this we have to both do DBUG_POP() and also reset THR_KEY_mysys + as the key is used by DBUG. + */ DBUG_POP(); - pthread_setspecific(THR_KEY_mysys,0); if (tmp && tmp->init) @@ -423,6 +382,9 @@ void my_thread_end(void) if (--THR_thread_count == 0) mysql_cond_signal(&THR_COND_threads); mysql_mutex_unlock(&THR_LOCK_threads); + + /* Trash variable so that we can detect false accesses to my_thread_var */ + tmp->init= 2; free(tmp); } } @@ -432,6 +394,10 @@ struct st_my_thread_var *_my_thread_var(void) return my_pthread_getspecific(struct st_my_thread_var*,THR_KEY_mysys); } +int set_mysys_var(struct st_my_thread_var *mysys_var) +{ + return my_pthread_setspecific_ptr(THR_KEY_mysys, mysys_var); +} /**************************************************************************** Get name of current thread. @@ -439,7 +405,12 @@ struct st_my_thread_var *_my_thread_var(void) my_thread_id my_thread_dbug_id() { - return my_thread_var->id; + /* + We need to do this test as some system thread may not yet have called + my_thread_init(). + */ + struct st_my_thread_var *tmp= my_thread_var; + return tmp ? tmp->id : 0; } #ifdef DBUG_OFF diff --git a/mysys/my_uuid.c b/mysys/my_uuid.c index d71612dec44..da947b09c48 100644 --- a/mysys/my_uuid.c +++ b/mysys/my_uuid.c @@ -40,6 +40,7 @@ */ #include "mysys_priv.h" +#include <my_rnd.h> #include <m_string.h> #include <myisampack.h> /* mi_int2store, mi_int4store */ @@ -123,7 +124,7 @@ void my_uuid_init(ulong seed1, ulong seed2) Create a global unique identifier (uuid) @func my_uuid() - @param to Store uuid here. Must be of size MY_uuid_SIZE (16) + @param to Store uuid here. Must be of size MY_UUID_SIZE (16) */ void my_uuid(uchar *to) @@ -151,7 +152,7 @@ void my_uuid(uchar *to) /* -1 so we won't make tv= uuid_time for nanoseq >= (tv - uuid_time) */ - delta= min(nanoseq, (ulong)(tv - uuid_time -1)); + delta= MY_MIN(nanoseq, (ulong)(tv - uuid_time -1)); tv-= delta; nanoseq-= delta; } diff --git a/mysys/my_windac.c b/mysys/my_windac.c index 51fe22e8f59..9b489759625 100644 --- a/mysys/my_windac.c +++ b/mysys/my_windac.c @@ -33,7 +33,7 @@ static my_bool is_nt() } /* - Auxilary structure to store pointers to the data which we need to keep + Auxiliary structure to store pointers to the data which we need to keep around while SECURITY_ATTRIBUTES is in use. */ diff --git a/mysys/my_winerr.c b/mysys/my_winerr.c index 15f52dd7f37..a3f6229b74e 100644 --- a/mysys/my_winerr.c +++ b/mysys/my_winerr.c @@ -75,7 +75,9 @@ static struct errentry errtable[]= { { ERROR_ALREADY_EXISTS, EEXIST }, /* 183 */ { ERROR_FILENAME_EXCED_RANGE, ENOENT }, /* 206 */ { ERROR_NESTING_NOT_ALLOWED, EAGAIN }, /* 215 */ - { ERROR_NOT_ENOUGH_QUOTA, ENOMEM } /* 1816 */ + { ERROR_FILE_SYSTEM_LIMITATION, EFBIG }, /* 665 */ + { ERROR_NO_SYSTEM_RESOURCES, ENOMEM }, /* 1450 */ + { ERROR_NOT_ENOUGH_QUOTA, ENOMEM } /* 1816 */ }; /* size of the table */ @@ -117,7 +119,7 @@ static int get_errno_from_oserr(unsigned long oserrno) return EINVAL; } -/* Set errno corresponsing to GetLastError() value */ +/* Set errno corresponding to GetLastError() value */ void my_osmaperr ( unsigned long oserrno) { errno= get_errno_from_oserr(oserrno); diff --git a/mysys/my_write.c b/mysys/my_write.c index 001f7296042..43735c18f0a 100644 --- a/mysys/my_write.c +++ b/mysys/my_write.c @@ -26,7 +26,7 @@ size_t my_write(File Filedes, const uchar *Buffer, size_t Count, myf MyFlags) size_t writtenbytes, written; uint errors; DBUG_ENTER("my_write"); - DBUG_PRINT("my",("fd: %d Buffer: %p Count: %lu MyFlags: %d", + DBUG_PRINT("my",("fd: %d Buffer: %p Count: %lu MyFlags: %lu", Filedes, Buffer, (ulong) Count, MyFlags)); errors= 0; written= 0; if (!(MyFlags & (MY_WME | MY_FAE | MY_FNABP))) diff --git a/mysys/mysys_priv.h b/mysys/mysys_priv.h index 1004bf8889f..892e09148ef 100644 --- a/mysys/mysys_priv.h +++ b/mysys/mysys_priv.h @@ -13,8 +13,14 @@ along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ +#ifndef MYSYS_PRIV_INCLUDED +#define MYSYS_PRIV_INCLUDED + #include <my_global.h> #include <my_sys.h> +#include <my_crypt.h> + +C_MODE_START #ifdef USE_SYSTEM_WRAPPERS #include "system_wrappers.h" @@ -39,25 +45,27 @@ extern PSI_mutex_key key_LOCK_localtime_r; extern PSI_mutex_key key_BITMAP_mutex, key_IO_CACHE_append_buffer_lock, key_IO_CACHE_SHARE_mutex, key_KEY_CACHE_cache_lock, key_LOCK_alarm, key_my_thread_var_mutex, key_THR_LOCK_charset, key_THR_LOCK_heap, - key_THR_LOCK_isam, key_THR_LOCK_lock, key_THR_LOCK_malloc, + key_THR_LOCK_lock, key_THR_LOCK_malloc, key_THR_LOCK_mutex, key_THR_LOCK_myisam, key_THR_LOCK_net, key_THR_LOCK_open, key_THR_LOCK_threads, key_LOCK_uuid_generator, - key_TMPDIR_mutex, key_THR_LOCK_myisam_mmap; + key_TMPDIR_mutex, key_THR_LOCK_myisam_mmap, key_LOCK_timer; -extern PSI_cond_key key_COND_alarm, key_IO_CACHE_SHARE_cond, +extern PSI_cond_key key_COND_alarm, key_COND_timer, key_IO_CACHE_SHARE_cond, key_IO_CACHE_SHARE_cond_writer, key_my_thread_var_suspend, key_THR_COND_threads; #ifdef USE_ALARM_THREAD extern PSI_thread_key key_thread_alarm; #endif /* USE_ALARM_THREAD */ - +extern PSI_thread_key key_thread_timer; extern PSI_rwlock_key key_SAFEHASH_mutex; #endif /* HAVE_PSI_INTERFACE */ +extern PSI_stage_info stage_waiting_for_table_level_lock; + extern mysql_mutex_t THR_LOCK_malloc, THR_LOCK_open, THR_LOCK_keycache; -extern mysql_mutex_t THR_LOCK_lock, THR_LOCK_isam, THR_LOCK_net; +extern mysql_mutex_t THR_LOCK_lock, THR_LOCK_net; extern mysql_mutex_t THR_LOCK_charset; #include <mysql/psi/mysql_file.h> @@ -69,13 +77,24 @@ extern PSI_file_key key_file_proc_meminfo; extern PSI_file_key key_file_charset, key_file_cnf; #endif /* HAVE_PSI_INTERFACE */ +typedef struct { + ulonglong counter; + uint block_length, last_block_length; + uchar key[MY_AES_BLOCK_SIZE]; + ulonglong inbuf_counter; +} IO_CACHE_CRYPT; + +extern int (*_my_b_encr_read)(IO_CACHE *info,uchar *Buffer,size_t Count); +extern int (*_my_b_encr_write)(IO_CACHE *info,const uchar *Buffer,size_t Count); + #ifdef SAFEMALLOC -void *sf_malloc(size_t size); -void *sf_realloc(void *ptr, size_t size); +void *sf_malloc(size_t size, myf my_flags); +void *sf_realloc(void *ptr, size_t size, myf my_flags); void sf_free(void *ptr); +size_t sf_malloc_usable_size(void *ptr, my_bool *is_thread_specific); #else -#define sf_malloc(X) malloc(X) -#define sf_realloc(X,Y) realloc(X,Y) +#define sf_malloc(X,Y) malloc(X) +#define sf_realloc(X,Y,Z) realloc(X,Y) #define sf_free(X) free(X) #endif @@ -154,3 +173,7 @@ extern File my_win_dup(File fd); extern File my_win_sopen(const char *path, int oflag, int shflag, int perm); extern File my_open_osfhandle(HANDLE handle, int oflag); #endif + +C_MODE_END + +#endif diff --git a/mysys/psi_noop.c b/mysys/psi_noop.c new file mode 100644 index 00000000000..6eecf56f797 --- /dev/null +++ b/mysys/psi_noop.c @@ -0,0 +1,771 @@ +/* Copyright (c) 2011, 2015, Oracle and/or its affiliates. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software Foundation, + 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA */ + +/* + Always provide the noop performance interface, for plugins. +*/ + +#define USE_PSI_V1 +#define HAVE_PSI_INTERFACE + +#include "my_global.h" +#include "my_pthread.h" +#include "my_sys.h" +#include "mysql/psi/psi.h" + +C_MODE_START + +#define NNN __attribute__((unused)) + +static void register_mutex_noop(const char *category NNN, + PSI_mutex_info *info NNN, + int count NNN) +{ + return; +} + +static void register_rwlock_noop(const char *category NNN, + PSI_rwlock_info *info NNN, + int count NNN) +{ + return; +} + +static void register_cond_noop(const char *category NNN, + PSI_cond_info *info NNN, + int count NNN) +{ + return; +} + +static void register_thread_noop(const char *category NNN, + PSI_thread_info *info NNN, + int count NNN) +{ + return; +} + +static void register_file_noop(const char *category NNN, + PSI_file_info *info NNN, + int count NNN) +{ + return; +} + +static void register_stage_noop(const char *category NNN, + PSI_stage_info **info_array NNN, + int count NNN) +{ + return; +} + +static void register_statement_noop(const char *category NNN, + PSI_statement_info *info NNN, + int count NNN) +{ + return; +} + +static void register_socket_noop(const char *category NNN, + PSI_socket_info *info NNN, + int count NNN) +{ + return; +} + +static PSI_mutex* +init_mutex_noop(PSI_mutex_key key NNN, const void *identity NNN) +{ + return NULL; +} + +static void destroy_mutex_noop(PSI_mutex* mutex NNN) +{ + return; +} + +static PSI_rwlock* +init_rwlock_noop(PSI_rwlock_key key NNN, const void *identity NNN) +{ + return NULL; +} + +static void destroy_rwlock_noop(PSI_rwlock* rwlock NNN) +{ + return; +} + +static PSI_cond* +init_cond_noop(PSI_cond_key key NNN, const void *identity NNN) +{ + return NULL; +} + +static void destroy_cond_noop(PSI_cond* cond NNN) +{ + return; +} + +static PSI_socket* +init_socket_noop(PSI_socket_key key NNN, const my_socket *fd NNN, + const struct sockaddr *addr NNN, socklen_t addr_len NNN) +{ + return NULL; +} + +static void destroy_socket_noop(PSI_socket* socket NNN) +{ + return; +} + +static PSI_table_share* +get_table_share_noop(my_bool temporary NNN, struct TABLE_SHARE *share NNN) +{ + return NULL; +} + +static void release_table_share_noop(PSI_table_share* share NNN) +{ + return; +} + +static void +drop_table_share_noop(my_bool temporary NNN, const char *schema_name NNN, + int schema_name_length NNN, const char *table_name NNN, + int table_name_length NNN) +{ + return; +} + +static PSI_table* +open_table_noop(PSI_table_share *share NNN, const void *identity NNN) +{ + return NULL; +} + +static void unbind_table_noop(PSI_table *table NNN) +{ + return; +} + +static PSI_table* +rebind_table_noop(PSI_table_share *share NNN, + const void *identity NNN, + PSI_table *table NNN) +{ + return NULL; +} + +static void close_table_noop(PSI_table *table NNN) +{ + return; +} + +static void create_file_noop(PSI_file_key key NNN, + const char *name NNN, File file NNN) +{ + return; +} + +static int spawn_thread_noop(PSI_thread_key key NNN, + pthread_t *thread NNN, + const pthread_attr_t *attr NNN, + void *(*start_routine)(void*) NNN, void *arg NNN) +{ + return pthread_create(thread, attr, start_routine, arg); +} + +static PSI_thread* +new_thread_noop(PSI_thread_key key NNN, + const void *identity NNN, ulonglong thread_id NNN) +{ + return NULL; +} + +static void set_thread_id_noop(PSI_thread *thread NNN, ulonglong id NNN) +{ + return; +} + +static PSI_thread* +get_thread_noop(void NNN) +{ + return NULL; +} + +static void set_thread_user_noop(const char *user NNN, int user_len NNN) +{ + return; +} + +static void set_thread_user_host_noop(const char *user NNN, int user_len NNN, + const char *host NNN, int host_len NNN) +{ + return; +} + +static void set_thread_db_noop(const char* db NNN, int db_len NNN) +{ + return; +} + +static void set_thread_command_noop(int command NNN) +{ + return; +} + +static void set_thread_start_time_noop(time_t start_time NNN) +{ + return; +} + +static void set_thread_state_noop(const char* state NNN) +{ + return; +} + +static void set_thread_info_noop(const char* info NNN, uint info_len NNN) +{ + return; +} + +static void set_thread_noop(PSI_thread* thread NNN) +{ + return; +} + +static void delete_current_thread_noop(void) +{ + return; +} + +static void delete_thread_noop(PSI_thread *thread NNN) +{ + return; +} + +static PSI_file_locker* +get_thread_file_name_locker_noop(PSI_file_locker_state *state NNN, + PSI_file_key key NNN, + enum PSI_file_operation op NNN, + const char *name NNN, const void *identity NNN) +{ + return NULL; +} + +static PSI_file_locker* +get_thread_file_stream_locker_noop(PSI_file_locker_state *state NNN, + PSI_file *file NNN, + enum PSI_file_operation op NNN) +{ + return NULL; +} + + +static PSI_file_locker* +get_thread_file_descriptor_locker_noop(PSI_file_locker_state *state NNN, + File file NNN, + enum PSI_file_operation op NNN) +{ + return NULL; +} + +static void unlock_mutex_noop(PSI_mutex *mutex NNN) +{ + return; +} + +static void unlock_rwlock_noop(PSI_rwlock *rwlock NNN) +{ + return; +} + +static void signal_cond_noop(PSI_cond* cond NNN) +{ + return; +} + +static void broadcast_cond_noop(PSI_cond* cond NNN) +{ + return; +} + +static PSI_idle_locker* +start_idle_wait_noop(PSI_idle_locker_state* state NNN, + const char *src_file NNN, uint src_line NNN) +{ + return NULL; +} + +static void end_idle_wait_noop(PSI_idle_locker* locker NNN) +{ + return; +} + +static PSI_mutex_locker* +start_mutex_wait_noop(PSI_mutex_locker_state *state NNN, + PSI_mutex *mutex NNN, + PSI_mutex_operation op NNN, + const char *src_file NNN, uint src_line NNN) +{ + return NULL; +} + +static void end_mutex_wait_noop(PSI_mutex_locker* locker NNN, int rc NNN) +{ + return; +} + + +static PSI_rwlock_locker* +start_rwlock_rdwait_noop(struct PSI_rwlock_locker_state_v1 *state NNN, + struct PSI_rwlock *rwlock NNN, + enum PSI_rwlock_operation op NNN, + const char *src_file NNN, uint src_line NNN) +{ + return NULL; +} + +static void end_rwlock_rdwait_noop(PSI_rwlock_locker* locker NNN, int rc NNN) +{ + return; +} + +static struct PSI_rwlock_locker* +start_rwlock_wrwait_noop(struct PSI_rwlock_locker_state_v1 *state NNN, + struct PSI_rwlock *rwlock NNN, + enum PSI_rwlock_operation op NNN, + const char *src_file NNN, uint src_line NNN) +{ + return NULL; +} + +static void end_rwlock_wrwait_noop(PSI_rwlock_locker* locker NNN, int rc NNN) +{ + return; +} + +static struct PSI_cond_locker* +start_cond_wait_noop(struct PSI_cond_locker_state_v1 *state NNN, + struct PSI_cond *cond NNN, + struct PSI_mutex *mutex NNN, + enum PSI_cond_operation op NNN, + const char *src_file NNN, uint src_line NNN) +{ + return NULL; +} + +static void end_cond_wait_noop(PSI_cond_locker* locker NNN, int rc NNN) +{ + return; +} + +static struct PSI_table_locker* +start_table_io_wait_noop(struct PSI_table_locker_state_v1 *state NNN, + struct PSI_table *table NNN, + enum PSI_table_io_operation op NNN, + uint index NNN, + const char *src_file NNN, uint src_line NNN) +{ + return NULL; +} + +static void end_table_io_wait_noop(PSI_table_locker* locker NNN) +{ + return; +} + +static struct PSI_table_locker* +start_table_lock_wait_noop(struct PSI_table_locker_state_v1 *state NNN, + struct PSI_table *table NNN, + enum PSI_table_lock_operation op NNN, + ulong flags NNN, + const char *src_file NNN, uint src_line NNN) +{ + return NULL; +} + +static void end_table_lock_wait_noop(PSI_table_locker* locker NNN) +{ + return; +} + +static void start_file_open_wait_noop(PSI_file_locker *locker NNN, + const char *src_file NNN, + uint src_line NNN) +{ + return; +} + +static PSI_file* end_file_open_wait_noop(PSI_file_locker *locker NNN, + void *result NNN) +{ + return NULL; +} + +static void end_file_open_wait_and_bind_to_descriptor_noop + (PSI_file_locker *locker NNN, File file NNN) +{ + return; +} + +static void start_file_wait_noop(PSI_file_locker *locker NNN, + size_t count NNN, + const char *src_file NNN, + uint src_line NNN) +{ + return; +} + +static void end_file_wait_noop(PSI_file_locker *locker NNN, + size_t count NNN) +{ + return; +} + +static void start_file_close_wait_noop(PSI_file_locker *locker NNN, + const char *src_file NNN, + uint src_line NNN) +{ + return; +} + +static void end_file_close_wait_noop(PSI_file_locker *locker NNN, + int result NNN) +{ + return; +} + +static void start_stage_noop(PSI_stage_key key NNN, + const char *src_file NNN, int src_line NNN) +{ + return; +} + +static void end_stage_noop(void) +{ + return; +} + +static PSI_statement_locker* +get_thread_statement_locker_noop(PSI_statement_locker_state *state NNN, + PSI_statement_key key NNN, + const void *charset NNN) +{ + return NULL; +} + +static PSI_statement_locker* +refine_statement_noop(PSI_statement_locker *locker NNN, + PSI_statement_key key NNN) +{ + return NULL; +} + +static void start_statement_noop(PSI_statement_locker *locker NNN, + const char *db NNN, uint db_len NNN, + const char *src_file NNN, uint src_line NNN) +{ + return; +} + +static void set_statement_text_noop(PSI_statement_locker *locker NNN, + const char *text NNN, uint text_len NNN) +{ + return; +} + +static void set_statement_lock_time_noop(PSI_statement_locker *locker NNN, + ulonglong count NNN) +{ + return; +} + +static void set_statement_rows_sent_noop(PSI_statement_locker *locker NNN, + ulonglong count NNN) +{ + return; +} + +static void set_statement_rows_examined_noop(PSI_statement_locker *locker NNN, + ulonglong count NNN) +{ + return; +} + +static void inc_statement_created_tmp_disk_tables_noop(PSI_statement_locker *locker NNN, + ulong count NNN) +{ + return; +} + +static void inc_statement_created_tmp_tables_noop(PSI_statement_locker *locker NNN, + ulong count NNN) +{ + return; +} + +static void inc_statement_select_full_join_noop(PSI_statement_locker *locker NNN, + ulong count NNN) +{ + return; +} + +static void inc_statement_select_full_range_join_noop(PSI_statement_locker *locker NNN, + ulong count NNN) +{ + return; +} + +static void inc_statement_select_range_noop(PSI_statement_locker *locker NNN, + ulong count NNN) +{ + return; +} + +static void inc_statement_select_range_check_noop(PSI_statement_locker *locker NNN, + ulong count NNN) +{ + return; +} + +static void inc_statement_select_scan_noop(PSI_statement_locker *locker NNN, + ulong count NNN) +{ + return; +} + +static void inc_statement_sort_merge_passes_noop(PSI_statement_locker *locker NNN, + ulong count NNN) +{ + return; +} + +static void inc_statement_sort_range_noop(PSI_statement_locker *locker NNN, + ulong count NNN) +{ + return; +} + +static void inc_statement_sort_rows_noop(PSI_statement_locker *locker NNN, + ulong count NNN) +{ + return; +} + +static void inc_statement_sort_scan_noop(PSI_statement_locker *locker NNN, + ulong count NNN) +{ + return; +} + +static void set_statement_no_index_used_noop(PSI_statement_locker *locker NNN) +{ + return; +} + +static void set_statement_no_good_index_used_noop(PSI_statement_locker *locker NNN) +{ + return; +} + +static void end_statement_noop(PSI_statement_locker *locker NNN, + void *stmt_da NNN) +{ + return; +} + +static PSI_socket_locker* +start_socket_wait_noop(PSI_socket_locker_state *state NNN, + PSI_socket *socket NNN, + PSI_socket_operation op NNN, + size_t count NNN, + const char *src_file NNN, + uint src_line NNN) +{ + return NULL; +} + +static void end_socket_wait_noop(PSI_socket_locker *locker NNN, + size_t count NNN) +{ + return; +} + +static void set_socket_state_noop(PSI_socket *socket NNN, + enum PSI_socket_state state NNN) +{ + return; +} + +static void set_socket_info_noop(PSI_socket *socket NNN, + const my_socket *fd NNN, + const struct sockaddr *addr NNN, + socklen_t addr_len NNN) +{ + return; +} + +static void set_socket_thread_owner_noop(PSI_socket *socket NNN) +{ + return; +} + +static struct PSI_digest_locker* +digest_start_noop(PSI_statement_locker *locker NNN) +{ + return NULL; +} + +static void +digest_end_noop(PSI_digest_locker *locker NNN, + const struct sql_digest_storage *digest NNN) +{ + return; +} + +static int +set_thread_connect_attrs_noop(const char *buffer NNN, + uint length NNN, + const void *from_cs NNN) +{ + return 0; +} + +static PSI PSI_noop= +{ + register_mutex_noop, + register_rwlock_noop, + register_cond_noop, + register_thread_noop, + register_file_noop, + register_stage_noop, + register_statement_noop, + register_socket_noop, + init_mutex_noop, + destroy_mutex_noop, + init_rwlock_noop, + destroy_rwlock_noop, + init_cond_noop, + destroy_cond_noop, + init_socket_noop, + destroy_socket_noop, + get_table_share_noop, + release_table_share_noop, + drop_table_share_noop, + open_table_noop, + unbind_table_noop, + rebind_table_noop, + close_table_noop, + create_file_noop, + spawn_thread_noop, + new_thread_noop, + set_thread_id_noop, + get_thread_noop, + set_thread_user_noop, + set_thread_user_host_noop, + set_thread_db_noop, + set_thread_command_noop, + set_thread_start_time_noop, + set_thread_state_noop, + set_thread_info_noop, + set_thread_noop, + delete_current_thread_noop, + delete_thread_noop, + get_thread_file_name_locker_noop, + get_thread_file_stream_locker_noop, + get_thread_file_descriptor_locker_noop, + unlock_mutex_noop, + unlock_rwlock_noop, + signal_cond_noop, + broadcast_cond_noop, + start_idle_wait_noop, + end_idle_wait_noop, + start_mutex_wait_noop, + end_mutex_wait_noop, + start_rwlock_rdwait_noop, + end_rwlock_rdwait_noop, + start_rwlock_wrwait_noop, + end_rwlock_wrwait_noop, + start_cond_wait_noop, + end_cond_wait_noop, + start_table_io_wait_noop, + end_table_io_wait_noop, + start_table_lock_wait_noop, + end_table_lock_wait_noop, + start_file_open_wait_noop, + end_file_open_wait_noop, + end_file_open_wait_and_bind_to_descriptor_noop, + start_file_wait_noop, + end_file_wait_noop, + start_file_close_wait_noop, + end_file_close_wait_noop, + start_stage_noop, + end_stage_noop, + get_thread_statement_locker_noop, + refine_statement_noop, + start_statement_noop, + set_statement_text_noop, + set_statement_lock_time_noop, + set_statement_rows_sent_noop, + set_statement_rows_examined_noop, + inc_statement_created_tmp_disk_tables_noop, + inc_statement_created_tmp_tables_noop, + inc_statement_select_full_join_noop, + inc_statement_select_full_range_join_noop, + inc_statement_select_range_noop, + inc_statement_select_range_check_noop, + inc_statement_select_scan_noop, + inc_statement_sort_merge_passes_noop, + inc_statement_sort_range_noop, + inc_statement_sort_rows_noop, + inc_statement_sort_scan_noop, + set_statement_no_index_used_noop, + set_statement_no_good_index_used_noop, + end_statement_noop, + start_socket_wait_noop, + end_socket_wait_noop, + set_socket_state_noop, + set_socket_info_noop, + set_socket_thread_owner_noop, + digest_start_noop, + digest_end_noop, + set_thread_connect_attrs_noop +}; + +/** + Hook for the instrumentation interface. + Code implementing the instrumentation interface should register here. +*/ +struct PSI_bootstrap *PSI_hook= NULL; + +/** + Instance of the instrumentation interface for the MySQL server. + @todo This is currently a global variable, which is handy when + compiling instrumented code that is bundled with the server. + When dynamic plugin are truly supported, this variable will need + to be replaced by a macro, so that each XYZ plugin can have it's own + xyz_psi_server variable, obtained from PSI_bootstrap::get_interface() + with the version used at compile time for plugin XYZ. +*/ + +PSI *PSI_server= & PSI_noop; + +void set_psi_server(PSI *psi) +{ + PSI_server= psi; +} + +C_MODE_END diff --git a/mysys/ptr_cmp.c b/mysys/ptr_cmp.c index 6e373e98972..add92edd713 100644 --- a/mysys/ptr_cmp.c +++ b/mysys/ptr_cmp.c @@ -52,6 +52,11 @@ static int ptr_compare_0(size_t *compare_length, uchar **a, uchar **b); static int ptr_compare_1(size_t *compare_length, uchar **a, uchar **b); static int ptr_compare_2(size_t *compare_length, uchar **a, uchar **b); static int ptr_compare_3(size_t *compare_length, uchar **a, uchar **b); +static int degenerate_compare_func(size_t *compare_length, uchar **a, uchar **b) +{ + DBUG_ASSERT(*compare_length == 0); + return 0; +} #endif /* __sun */ /* Get a pointer to a optimal byte-compare function for a given size */ @@ -64,6 +69,8 @@ qsort2_cmp get_ptr_compare (size_t size __attribute__((unused))) #else qsort2_cmp get_ptr_compare (size_t size) { + if (size == 0) + return (qsort2_cmp) degenerate_compare_func; if (size < 4) return (qsort2_cmp) ptr_compare; switch (size & 3) { @@ -88,8 +95,8 @@ qsort2_cmp get_ptr_compare (size_t size) static int ptr_compare(size_t *compare_length, uchar **a, uchar **b) { - reg3 int length= *compare_length; - reg1 uchar *first,*last; + size_t length= *compare_length; + uchar *first,*last; DBUG_ASSERT(length > 0); first= *a; last= *b; @@ -104,8 +111,8 @@ static int ptr_compare(size_t *compare_length, uchar **a, uchar **b) static int ptr_compare_0(size_t *compare_length,uchar **a, uchar **b) { - reg3 int length= *compare_length; - reg1 uchar *first,*last; + size_t length= *compare_length; + uchar *first,*last; first= *a; last= *b; loop: @@ -125,8 +132,8 @@ static int ptr_compare_0(size_t *compare_length,uchar **a, uchar **b) static int ptr_compare_1(size_t *compare_length,uchar **a, uchar **b) { - reg3 int length= *compare_length-1; - reg1 uchar *first,*last; + size_t length= *compare_length-1; + uchar *first,*last; first= *a+1; last= *b+1; cmp(-1); @@ -146,8 +153,8 @@ static int ptr_compare_1(size_t *compare_length,uchar **a, uchar **b) static int ptr_compare_2(size_t *compare_length,uchar **a, uchar **b) { - reg3 int length= *compare_length-2; - reg1 uchar *first,*last; + size_t length= *compare_length-2; + uchar *first,*last; first= *a +2 ; last= *b +2; cmp(-2); @@ -168,8 +175,8 @@ static int ptr_compare_2(size_t *compare_length,uchar **a, uchar **b) static int ptr_compare_3(size_t *compare_length,uchar **a, uchar **b) { - reg3 int length= *compare_length-3; - reg1 uchar *first,*last; + size_t length= *compare_length-3; + uchar *first,*last; first= *a +3 ; last= *b +3; cmp(-3); diff --git a/mysys/queues.c b/mysys/queues.c index 418163d7c58..5d09ce2063f 100644 --- a/mysys/queues.c +++ b/mysys/queues.c @@ -27,7 +27,7 @@ This code originates from the Unireg project. Code for generell handling of priority Queues. - Implemention of queues from "Algoritms in C" by Robert Sedgewick. + Implementation of queues from "Algorithms in C" by Robert Sedgewick. The queue can optionally store the position in queue in the element that is in the queue. This allows one to remove any element from the queue diff --git a/mysys/rijndael.c b/mysys/rijndael.c deleted file mode 100644 index e893a886726..00000000000 --- a/mysys/rijndael.c +++ /dev/null @@ -1,1379 +0,0 @@ -/* Copyright (c) 2002, 2010, Oracle and/or its affiliates. All rights reserved. - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; version 2 of the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ - - -/* - Based on version 3.0 (December 2000) - - Optimised ANSI C code for the Rijndael cipher (now AES) - - author Vincent Rijmen <vincent.rijmen@esat.kuleuven.ac.be> - author Antoon Bosselaers <antoon.bosselaers@esat.kuleuven.ac.be> - author Paulo Barreto <paulo.barreto@terra.com.br> -*/ - -#include <my_global.h> -#include "rijndael.h" - -/* - Define the following to use fastest and much larger code (~10K extra code) - #define FULL_UNROLL -*/ - -static const uint32 Te0[256]= -{ - 0xc66363a5U, 0xf87c7c84U, 0xee777799U, 0xf67b7b8dU, - 0xfff2f20dU, 0xd66b6bbdU, 0xde6f6fb1U, 0x91c5c554U, - 0x60303050U, 0x02010103U, 0xce6767a9U, 0x562b2b7dU, - 0xe7fefe19U, 0xb5d7d762U, 0x4dababe6U, 0xec76769aU, - 0x8fcaca45U, 0x1f82829dU, 0x89c9c940U, 0xfa7d7d87U, - 0xeffafa15U, 0xb25959ebU, 0x8e4747c9U, 0xfbf0f00bU, - 0x41adadecU, 0xb3d4d467U, 0x5fa2a2fdU, 0x45afafeaU, - 0x239c9cbfU, 0x53a4a4f7U, 0xe4727296U, 0x9bc0c05bU, - 0x75b7b7c2U, 0xe1fdfd1cU, 0x3d9393aeU, 0x4c26266aU, - 0x6c36365aU, 0x7e3f3f41U, 0xf5f7f702U, 0x83cccc4fU, - 0x6834345cU, 0x51a5a5f4U, 0xd1e5e534U, 0xf9f1f108U, - 0xe2717193U, 0xabd8d873U, 0x62313153U, 0x2a15153fU, - 0x0804040cU, 0x95c7c752U, 0x46232365U, 0x9dc3c35eU, - 0x30181828U, 0x379696a1U, 0x0a05050fU, 0x2f9a9ab5U, - 0x0e070709U, 0x24121236U, 0x1b80809bU, 0xdfe2e23dU, - 0xcdebeb26U, 0x4e272769U, 0x7fb2b2cdU, 0xea75759fU, - 0x1209091bU, 0x1d83839eU, 0x582c2c74U, 0x341a1a2eU, - 0x361b1b2dU, 0xdc6e6eb2U, 0xb45a5aeeU, 0x5ba0a0fbU, - 0xa45252f6U, 0x763b3b4dU, 0xb7d6d661U, 0x7db3b3ceU, - 0x5229297bU, 0xdde3e33eU, 0x5e2f2f71U, 0x13848497U, - 0xa65353f5U, 0xb9d1d168U, 0x00000000U, 0xc1eded2cU, - 0x40202060U, 0xe3fcfc1fU, 0x79b1b1c8U, 0xb65b5bedU, - 0xd46a6abeU, 0x8dcbcb46U, 0x67bebed9U, 0x7239394bU, - 0x944a4adeU, 0x984c4cd4U, 0xb05858e8U, 0x85cfcf4aU, - 0xbbd0d06bU, 0xc5efef2aU, 0x4faaaae5U, 0xedfbfb16U, - 0x864343c5U, 0x9a4d4dd7U, 0x66333355U, 0x11858594U, - 0x8a4545cfU, 0xe9f9f910U, 0x04020206U, 0xfe7f7f81U, - 0xa05050f0U, 0x783c3c44U, 0x259f9fbaU, 0x4ba8a8e3U, - 0xa25151f3U, 0x5da3a3feU, 0x804040c0U, 0x058f8f8aU, - 0x3f9292adU, 0x219d9dbcU, 0x70383848U, 0xf1f5f504U, - 0x63bcbcdfU, 0x77b6b6c1U, 0xafdada75U, 0x42212163U, - 0x20101030U, 0xe5ffff1aU, 0xfdf3f30eU, 0xbfd2d26dU, - 0x81cdcd4cU, 0x180c0c14U, 0x26131335U, 0xc3ecec2fU, - 0xbe5f5fe1U, 0x359797a2U, 0x884444ccU, 0x2e171739U, - 0x93c4c457U, 0x55a7a7f2U, 0xfc7e7e82U, 0x7a3d3d47U, - 0xc86464acU, 0xba5d5de7U, 0x3219192bU, 0xe6737395U, - 0xc06060a0U, 0x19818198U, 0x9e4f4fd1U, 0xa3dcdc7fU, - 0x44222266U, 0x542a2a7eU, 0x3b9090abU, 0x0b888883U, - 0x8c4646caU, 0xc7eeee29U, 0x6bb8b8d3U, 0x2814143cU, - 0xa7dede79U, 0xbc5e5ee2U, 0x160b0b1dU, 0xaddbdb76U, - 0xdbe0e03bU, 0x64323256U, 0x743a3a4eU, 0x140a0a1eU, - 0x924949dbU, 0x0c06060aU, 0x4824246cU, 0xb85c5ce4U, - 0x9fc2c25dU, 0xbdd3d36eU, 0x43acacefU, 0xc46262a6U, - 0x399191a8U, 0x319595a4U, 0xd3e4e437U, 0xf279798bU, - 0xd5e7e732U, 0x8bc8c843U, 0x6e373759U, 0xda6d6db7U, - 0x018d8d8cU, 0xb1d5d564U, 0x9c4e4ed2U, 0x49a9a9e0U, - 0xd86c6cb4U, 0xac5656faU, 0xf3f4f407U, 0xcfeaea25U, - 0xca6565afU, 0xf47a7a8eU, 0x47aeaee9U, 0x10080818U, - 0x6fbabad5U, 0xf0787888U, 0x4a25256fU, 0x5c2e2e72U, - 0x381c1c24U, 0x57a6a6f1U, 0x73b4b4c7U, 0x97c6c651U, - 0xcbe8e823U, 0xa1dddd7cU, 0xe874749cU, 0x3e1f1f21U, - 0x964b4bddU, 0x61bdbddcU, 0x0d8b8b86U, 0x0f8a8a85U, - 0xe0707090U, 0x7c3e3e42U, 0x71b5b5c4U, 0xcc6666aaU, - 0x904848d8U, 0x06030305U, 0xf7f6f601U, 0x1c0e0e12U, - 0xc26161a3U, 0x6a35355fU, 0xae5757f9U, 0x69b9b9d0U, - 0x17868691U, 0x99c1c158U, 0x3a1d1d27U, 0x279e9eb9U, - 0xd9e1e138U, 0xebf8f813U, 0x2b9898b3U, 0x22111133U, - 0xd26969bbU, 0xa9d9d970U, 0x078e8e89U, 0x339494a7U, - 0x2d9b9bb6U, 0x3c1e1e22U, 0x15878792U, 0xc9e9e920U, - 0x87cece49U, 0xaa5555ffU, 0x50282878U, 0xa5dfdf7aU, - 0x038c8c8fU, 0x59a1a1f8U, 0x09898980U, 0x1a0d0d17U, - 0x65bfbfdaU, 0xd7e6e631U, 0x844242c6U, 0xd06868b8U, - 0x824141c3U, 0x299999b0U, 0x5a2d2d77U, 0x1e0f0f11U, - 0x7bb0b0cbU, 0xa85454fcU, 0x6dbbbbd6U, 0x2c16163aU, -}; - -static const uint32 Te1[256]= -{ - 0xa5c66363U, 0x84f87c7cU, 0x99ee7777U, 0x8df67b7bU, - 0x0dfff2f2U, 0xbdd66b6bU, 0xb1de6f6fU, 0x5491c5c5U, - 0x50603030U, 0x03020101U, 0xa9ce6767U, 0x7d562b2bU, - 0x19e7fefeU, 0x62b5d7d7U, 0xe64dababU, 0x9aec7676U, - 0x458fcacaU, 0x9d1f8282U, 0x4089c9c9U, 0x87fa7d7dU, - 0x15effafaU, 0xebb25959U, 0xc98e4747U, 0x0bfbf0f0U, - 0xec41adadU, 0x67b3d4d4U, 0xfd5fa2a2U, 0xea45afafU, - 0xbf239c9cU, 0xf753a4a4U, 0x96e47272U, 0x5b9bc0c0U, - 0xc275b7b7U, 0x1ce1fdfdU, 0xae3d9393U, 0x6a4c2626U, - 0x5a6c3636U, 0x417e3f3fU, 0x02f5f7f7U, 0x4f83ccccU, - 0x5c683434U, 0xf451a5a5U, 0x34d1e5e5U, 0x08f9f1f1U, - 0x93e27171U, 0x73abd8d8U, 0x53623131U, 0x3f2a1515U, - 0x0c080404U, 0x5295c7c7U, 0x65462323U, 0x5e9dc3c3U, - 0x28301818U, 0xa1379696U, 0x0f0a0505U, 0xb52f9a9aU, - 0x090e0707U, 0x36241212U, 0x9b1b8080U, 0x3ddfe2e2U, - 0x26cdebebU, 0x694e2727U, 0xcd7fb2b2U, 0x9fea7575U, - 0x1b120909U, 0x9e1d8383U, 0x74582c2cU, 0x2e341a1aU, - 0x2d361b1bU, 0xb2dc6e6eU, 0xeeb45a5aU, 0xfb5ba0a0U, - 0xf6a45252U, 0x4d763b3bU, 0x61b7d6d6U, 0xce7db3b3U, - 0x7b522929U, 0x3edde3e3U, 0x715e2f2fU, 0x97138484U, - 0xf5a65353U, 0x68b9d1d1U, 0x00000000U, 0x2cc1ededU, - 0x60402020U, 0x1fe3fcfcU, 0xc879b1b1U, 0xedb65b5bU, - 0xbed46a6aU, 0x468dcbcbU, 0xd967bebeU, 0x4b723939U, - 0xde944a4aU, 0xd4984c4cU, 0xe8b05858U, 0x4a85cfcfU, - 0x6bbbd0d0U, 0x2ac5efefU, 0xe54faaaaU, 0x16edfbfbU, - 0xc5864343U, 0xd79a4d4dU, 0x55663333U, 0x94118585U, - 0xcf8a4545U, 0x10e9f9f9U, 0x06040202U, 0x81fe7f7fU, - 0xf0a05050U, 0x44783c3cU, 0xba259f9fU, 0xe34ba8a8U, - 0xf3a25151U, 0xfe5da3a3U, 0xc0804040U, 0x8a058f8fU, - 0xad3f9292U, 0xbc219d9dU, 0x48703838U, 0x04f1f5f5U, - 0xdf63bcbcU, 0xc177b6b6U, 0x75afdadaU, 0x63422121U, - 0x30201010U, 0x1ae5ffffU, 0x0efdf3f3U, 0x6dbfd2d2U, - 0x4c81cdcdU, 0x14180c0cU, 0x35261313U, 0x2fc3ececU, - 0xe1be5f5fU, 0xa2359797U, 0xcc884444U, 0x392e1717U, - 0x5793c4c4U, 0xf255a7a7U, 0x82fc7e7eU, 0x477a3d3dU, - 0xacc86464U, 0xe7ba5d5dU, 0x2b321919U, 0x95e67373U, - 0xa0c06060U, 0x98198181U, 0xd19e4f4fU, 0x7fa3dcdcU, - 0x66442222U, 0x7e542a2aU, 0xab3b9090U, 0x830b8888U, - 0xca8c4646U, 0x29c7eeeeU, 0xd36bb8b8U, 0x3c281414U, - 0x79a7dedeU, 0xe2bc5e5eU, 0x1d160b0bU, 0x76addbdbU, - 0x3bdbe0e0U, 0x56643232U, 0x4e743a3aU, 0x1e140a0aU, - 0xdb924949U, 0x0a0c0606U, 0x6c482424U, 0xe4b85c5cU, - 0x5d9fc2c2U, 0x6ebdd3d3U, 0xef43acacU, 0xa6c46262U, - 0xa8399191U, 0xa4319595U, 0x37d3e4e4U, 0x8bf27979U, - 0x32d5e7e7U, 0x438bc8c8U, 0x596e3737U, 0xb7da6d6dU, - 0x8c018d8dU, 0x64b1d5d5U, 0xd29c4e4eU, 0xe049a9a9U, - 0xb4d86c6cU, 0xfaac5656U, 0x07f3f4f4U, 0x25cfeaeaU, - 0xafca6565U, 0x8ef47a7aU, 0xe947aeaeU, 0x18100808U, - 0xd56fbabaU, 0x88f07878U, 0x6f4a2525U, 0x725c2e2eU, - 0x24381c1cU, 0xf157a6a6U, 0xc773b4b4U, 0x5197c6c6U, - 0x23cbe8e8U, 0x7ca1ddddU, 0x9ce87474U, 0x213e1f1fU, - 0xdd964b4bU, 0xdc61bdbdU, 0x860d8b8bU, 0x850f8a8aU, - 0x90e07070U, 0x427c3e3eU, 0xc471b5b5U, 0xaacc6666U, - 0xd8904848U, 0x05060303U, 0x01f7f6f6U, 0x121c0e0eU, - 0xa3c26161U, 0x5f6a3535U, 0xf9ae5757U, 0xd069b9b9U, - 0x91178686U, 0x5899c1c1U, 0x273a1d1dU, 0xb9279e9eU, - 0x38d9e1e1U, 0x13ebf8f8U, 0xb32b9898U, 0x33221111U, - 0xbbd26969U, 0x70a9d9d9U, 0x89078e8eU, 0xa7339494U, - 0xb62d9b9bU, 0x223c1e1eU, 0x92158787U, 0x20c9e9e9U, - 0x4987ceceU, 0xffaa5555U, 0x78502828U, 0x7aa5dfdfU, - 0x8f038c8cU, 0xf859a1a1U, 0x80098989U, 0x171a0d0dU, - 0xda65bfbfU, 0x31d7e6e6U, 0xc6844242U, 0xb8d06868U, - 0xc3824141U, 0xb0299999U, 0x775a2d2dU, 0x111e0f0fU, - 0xcb7bb0b0U, 0xfca85454U, 0xd66dbbbbU, 0x3a2c1616U, -}; - -static const uint32 Te2[256]= -{ - 0x63a5c663U, 0x7c84f87cU, 0x7799ee77U, 0x7b8df67bU, - 0xf20dfff2U, 0x6bbdd66bU, 0x6fb1de6fU, 0xc55491c5U, - 0x30506030U, 0x01030201U, 0x67a9ce67U, 0x2b7d562bU, - 0xfe19e7feU, 0xd762b5d7U, 0xabe64dabU, 0x769aec76U, - 0xca458fcaU, 0x829d1f82U, 0xc94089c9U, 0x7d87fa7dU, - 0xfa15effaU, 0x59ebb259U, 0x47c98e47U, 0xf00bfbf0U, - 0xadec41adU, 0xd467b3d4U, 0xa2fd5fa2U, 0xafea45afU, - 0x9cbf239cU, 0xa4f753a4U, 0x7296e472U, 0xc05b9bc0U, - 0xb7c275b7U, 0xfd1ce1fdU, 0x93ae3d93U, 0x266a4c26U, - 0x365a6c36U, 0x3f417e3fU, 0xf702f5f7U, 0xcc4f83ccU, - 0x345c6834U, 0xa5f451a5U, 0xe534d1e5U, 0xf108f9f1U, - 0x7193e271U, 0xd873abd8U, 0x31536231U, 0x153f2a15U, - 0x040c0804U, 0xc75295c7U, 0x23654623U, 0xc35e9dc3U, - 0x18283018U, 0x96a13796U, 0x050f0a05U, 0x9ab52f9aU, - 0x07090e07U, 0x12362412U, 0x809b1b80U, 0xe23ddfe2U, - 0xeb26cdebU, 0x27694e27U, 0xb2cd7fb2U, 0x759fea75U, - 0x091b1209U, 0x839e1d83U, 0x2c74582cU, 0x1a2e341aU, - 0x1b2d361bU, 0x6eb2dc6eU, 0x5aeeb45aU, 0xa0fb5ba0U, - 0x52f6a452U, 0x3b4d763bU, 0xd661b7d6U, 0xb3ce7db3U, - 0x297b5229U, 0xe33edde3U, 0x2f715e2fU, 0x84971384U, - 0x53f5a653U, 0xd168b9d1U, 0x00000000U, 0xed2cc1edU, - 0x20604020U, 0xfc1fe3fcU, 0xb1c879b1U, 0x5bedb65bU, - 0x6abed46aU, 0xcb468dcbU, 0xbed967beU, 0x394b7239U, - 0x4ade944aU, 0x4cd4984cU, 0x58e8b058U, 0xcf4a85cfU, - 0xd06bbbd0U, 0xef2ac5efU, 0xaae54faaU, 0xfb16edfbU, - 0x43c58643U, 0x4dd79a4dU, 0x33556633U, 0x85941185U, - 0x45cf8a45U, 0xf910e9f9U, 0x02060402U, 0x7f81fe7fU, - 0x50f0a050U, 0x3c44783cU, 0x9fba259fU, 0xa8e34ba8U, - 0x51f3a251U, 0xa3fe5da3U, 0x40c08040U, 0x8f8a058fU, - 0x92ad3f92U, 0x9dbc219dU, 0x38487038U, 0xf504f1f5U, - 0xbcdf63bcU, 0xb6c177b6U, 0xda75afdaU, 0x21634221U, - 0x10302010U, 0xff1ae5ffU, 0xf30efdf3U, 0xd26dbfd2U, - 0xcd4c81cdU, 0x0c14180cU, 0x13352613U, 0xec2fc3ecU, - 0x5fe1be5fU, 0x97a23597U, 0x44cc8844U, 0x17392e17U, - 0xc45793c4U, 0xa7f255a7U, 0x7e82fc7eU, 0x3d477a3dU, - 0x64acc864U, 0x5de7ba5dU, 0x192b3219U, 0x7395e673U, - 0x60a0c060U, 0x81981981U, 0x4fd19e4fU, 0xdc7fa3dcU, - 0x22664422U, 0x2a7e542aU, 0x90ab3b90U, 0x88830b88U, - 0x46ca8c46U, 0xee29c7eeU, 0xb8d36bb8U, 0x143c2814U, - 0xde79a7deU, 0x5ee2bc5eU, 0x0b1d160bU, 0xdb76addbU, - 0xe03bdbe0U, 0x32566432U, 0x3a4e743aU, 0x0a1e140aU, - 0x49db9249U, 0x060a0c06U, 0x246c4824U, 0x5ce4b85cU, - 0xc25d9fc2U, 0xd36ebdd3U, 0xacef43acU, 0x62a6c462U, - 0x91a83991U, 0x95a43195U, 0xe437d3e4U, 0x798bf279U, - 0xe732d5e7U, 0xc8438bc8U, 0x37596e37U, 0x6db7da6dU, - 0x8d8c018dU, 0xd564b1d5U, 0x4ed29c4eU, 0xa9e049a9U, - 0x6cb4d86cU, 0x56faac56U, 0xf407f3f4U, 0xea25cfeaU, - 0x65afca65U, 0x7a8ef47aU, 0xaee947aeU, 0x08181008U, - 0xbad56fbaU, 0x7888f078U, 0x256f4a25U, 0x2e725c2eU, - 0x1c24381cU, 0xa6f157a6U, 0xb4c773b4U, 0xc65197c6U, - 0xe823cbe8U, 0xdd7ca1ddU, 0x749ce874U, 0x1f213e1fU, - 0x4bdd964bU, 0xbddc61bdU, 0x8b860d8bU, 0x8a850f8aU, - 0x7090e070U, 0x3e427c3eU, 0xb5c471b5U, 0x66aacc66U, - 0x48d89048U, 0x03050603U, 0xf601f7f6U, 0x0e121c0eU, - 0x61a3c261U, 0x355f6a35U, 0x57f9ae57U, 0xb9d069b9U, - 0x86911786U, 0xc15899c1U, 0x1d273a1dU, 0x9eb9279eU, - 0xe138d9e1U, 0xf813ebf8U, 0x98b32b98U, 0x11332211U, - 0x69bbd269U, 0xd970a9d9U, 0x8e89078eU, 0x94a73394U, - 0x9bb62d9bU, 0x1e223c1eU, 0x87921587U, 0xe920c9e9U, - 0xce4987ceU, 0x55ffaa55U, 0x28785028U, 0xdf7aa5dfU, - 0x8c8f038cU, 0xa1f859a1U, 0x89800989U, 0x0d171a0dU, - 0xbfda65bfU, 0xe631d7e6U, 0x42c68442U, 0x68b8d068U, - 0x41c38241U, 0x99b02999U, 0x2d775a2dU, 0x0f111e0fU, - 0xb0cb7bb0U, 0x54fca854U, 0xbbd66dbbU, 0x163a2c16U, -}; - -static const uint32 Te3[256]= -{ - 0x6363a5c6U, 0x7c7c84f8U, 0x777799eeU, 0x7b7b8df6U, - 0xf2f20dffU, 0x6b6bbdd6U, 0x6f6fb1deU, 0xc5c55491U, - 0x30305060U, 0x01010302U, 0x6767a9ceU, 0x2b2b7d56U, - 0xfefe19e7U, 0xd7d762b5U, 0xababe64dU, 0x76769aecU, - 0xcaca458fU, 0x82829d1fU, 0xc9c94089U, 0x7d7d87faU, - 0xfafa15efU, 0x5959ebb2U, 0x4747c98eU, 0xf0f00bfbU, - 0xadadec41U, 0xd4d467b3U, 0xa2a2fd5fU, 0xafafea45U, - 0x9c9cbf23U, 0xa4a4f753U, 0x727296e4U, 0xc0c05b9bU, - 0xb7b7c275U, 0xfdfd1ce1U, 0x9393ae3dU, 0x26266a4cU, - 0x36365a6cU, 0x3f3f417eU, 0xf7f702f5U, 0xcccc4f83U, - 0x34345c68U, 0xa5a5f451U, 0xe5e534d1U, 0xf1f108f9U, - 0x717193e2U, 0xd8d873abU, 0x31315362U, 0x15153f2aU, - 0x04040c08U, 0xc7c75295U, 0x23236546U, 0xc3c35e9dU, - 0x18182830U, 0x9696a137U, 0x05050f0aU, 0x9a9ab52fU, - 0x0707090eU, 0x12123624U, 0x80809b1bU, 0xe2e23ddfU, - 0xebeb26cdU, 0x2727694eU, 0xb2b2cd7fU, 0x75759feaU, - 0x09091b12U, 0x83839e1dU, 0x2c2c7458U, 0x1a1a2e34U, - 0x1b1b2d36U, 0x6e6eb2dcU, 0x5a5aeeb4U, 0xa0a0fb5bU, - 0x5252f6a4U, 0x3b3b4d76U, 0xd6d661b7U, 0xb3b3ce7dU, - 0x29297b52U, 0xe3e33eddU, 0x2f2f715eU, 0x84849713U, - 0x5353f5a6U, 0xd1d168b9U, 0x00000000U, 0xeded2cc1U, - 0x20206040U, 0xfcfc1fe3U, 0xb1b1c879U, 0x5b5bedb6U, - 0x6a6abed4U, 0xcbcb468dU, 0xbebed967U, 0x39394b72U, - 0x4a4ade94U, 0x4c4cd498U, 0x5858e8b0U, 0xcfcf4a85U, - 0xd0d06bbbU, 0xefef2ac5U, 0xaaaae54fU, 0xfbfb16edU, - 0x4343c586U, 0x4d4dd79aU, 0x33335566U, 0x85859411U, - 0x4545cf8aU, 0xf9f910e9U, 0x02020604U, 0x7f7f81feU, - 0x5050f0a0U, 0x3c3c4478U, 0x9f9fba25U, 0xa8a8e34bU, - 0x5151f3a2U, 0xa3a3fe5dU, 0x4040c080U, 0x8f8f8a05U, - 0x9292ad3fU, 0x9d9dbc21U, 0x38384870U, 0xf5f504f1U, - 0xbcbcdf63U, 0xb6b6c177U, 0xdada75afU, 0x21216342U, - 0x10103020U, 0xffff1ae5U, 0xf3f30efdU, 0xd2d26dbfU, - 0xcdcd4c81U, 0x0c0c1418U, 0x13133526U, 0xecec2fc3U, - 0x5f5fe1beU, 0x9797a235U, 0x4444cc88U, 0x1717392eU, - 0xc4c45793U, 0xa7a7f255U, 0x7e7e82fcU, 0x3d3d477aU, - 0x6464acc8U, 0x5d5de7baU, 0x19192b32U, 0x737395e6U, - 0x6060a0c0U, 0x81819819U, 0x4f4fd19eU, 0xdcdc7fa3U, - 0x22226644U, 0x2a2a7e54U, 0x9090ab3bU, 0x8888830bU, - 0x4646ca8cU, 0xeeee29c7U, 0xb8b8d36bU, 0x14143c28U, - 0xdede79a7U, 0x5e5ee2bcU, 0x0b0b1d16U, 0xdbdb76adU, - 0xe0e03bdbU, 0x32325664U, 0x3a3a4e74U, 0x0a0a1e14U, - 0x4949db92U, 0x06060a0cU, 0x24246c48U, 0x5c5ce4b8U, - 0xc2c25d9fU, 0xd3d36ebdU, 0xacacef43U, 0x6262a6c4U, - 0x9191a839U, 0x9595a431U, 0xe4e437d3U, 0x79798bf2U, - 0xe7e732d5U, 0xc8c8438bU, 0x3737596eU, 0x6d6db7daU, - 0x8d8d8c01U, 0xd5d564b1U, 0x4e4ed29cU, 0xa9a9e049U, - 0x6c6cb4d8U, 0x5656faacU, 0xf4f407f3U, 0xeaea25cfU, - 0x6565afcaU, 0x7a7a8ef4U, 0xaeaee947U, 0x08081810U, - 0xbabad56fU, 0x787888f0U, 0x25256f4aU, 0x2e2e725cU, - 0x1c1c2438U, 0xa6a6f157U, 0xb4b4c773U, 0xc6c65197U, - 0xe8e823cbU, 0xdddd7ca1U, 0x74749ce8U, 0x1f1f213eU, - 0x4b4bdd96U, 0xbdbddc61U, 0x8b8b860dU, 0x8a8a850fU, - 0x707090e0U, 0x3e3e427cU, 0xb5b5c471U, 0x6666aaccU, - 0x4848d890U, 0x03030506U, 0xf6f601f7U, 0x0e0e121cU, - 0x6161a3c2U, 0x35355f6aU, 0x5757f9aeU, 0xb9b9d069U, - 0x86869117U, 0xc1c15899U, 0x1d1d273aU, 0x9e9eb927U, - 0xe1e138d9U, 0xf8f813ebU, 0x9898b32bU, 0x11113322U, - 0x6969bbd2U, 0xd9d970a9U, 0x8e8e8907U, 0x9494a733U, - 0x9b9bb62dU, 0x1e1e223cU, 0x87879215U, 0xe9e920c9U, - 0xcece4987U, 0x5555ffaaU, 0x28287850U, 0xdfdf7aa5U, - 0x8c8c8f03U, 0xa1a1f859U, 0x89898009U, 0x0d0d171aU, - 0xbfbfda65U, 0xe6e631d7U, 0x4242c684U, 0x6868b8d0U, - 0x4141c382U, 0x9999b029U, 0x2d2d775aU, 0x0f0f111eU, - 0xb0b0cb7bU, 0x5454fca8U, 0xbbbbd66dU, 0x16163a2cU, -}; - -static const uint32 Te4[256]= -{ - 0x63636363U, 0x7c7c7c7cU, 0x77777777U, 0x7b7b7b7bU, - 0xf2f2f2f2U, 0x6b6b6b6bU, 0x6f6f6f6fU, 0xc5c5c5c5U, - 0x30303030U, 0x01010101U, 0x67676767U, 0x2b2b2b2bU, - 0xfefefefeU, 0xd7d7d7d7U, 0xababababU, 0x76767676U, - 0xcacacacaU, 0x82828282U, 0xc9c9c9c9U, 0x7d7d7d7dU, - 0xfafafafaU, 0x59595959U, 0x47474747U, 0xf0f0f0f0U, - 0xadadadadU, 0xd4d4d4d4U, 0xa2a2a2a2U, 0xafafafafU, - 0x9c9c9c9cU, 0xa4a4a4a4U, 0x72727272U, 0xc0c0c0c0U, - 0xb7b7b7b7U, 0xfdfdfdfdU, 0x93939393U, 0x26262626U, - 0x36363636U, 0x3f3f3f3fU, 0xf7f7f7f7U, 0xccccccccU, - 0x34343434U, 0xa5a5a5a5U, 0xe5e5e5e5U, 0xf1f1f1f1U, - 0x71717171U, 0xd8d8d8d8U, 0x31313131U, 0x15151515U, - 0x04040404U, 0xc7c7c7c7U, 0x23232323U, 0xc3c3c3c3U, - 0x18181818U, 0x96969696U, 0x05050505U, 0x9a9a9a9aU, - 0x07070707U, 0x12121212U, 0x80808080U, 0xe2e2e2e2U, - 0xebebebebU, 0x27272727U, 0xb2b2b2b2U, 0x75757575U, - 0x09090909U, 0x83838383U, 0x2c2c2c2cU, 0x1a1a1a1aU, - 0x1b1b1b1bU, 0x6e6e6e6eU, 0x5a5a5a5aU, 0xa0a0a0a0U, - 0x52525252U, 0x3b3b3b3bU, 0xd6d6d6d6U, 0xb3b3b3b3U, - 0x29292929U, 0xe3e3e3e3U, 0x2f2f2f2fU, 0x84848484U, - 0x53535353U, 0xd1d1d1d1U, 0x00000000U, 0xededededU, - 0x20202020U, 0xfcfcfcfcU, 0xb1b1b1b1U, 0x5b5b5b5bU, - 0x6a6a6a6aU, 0xcbcbcbcbU, 0xbebebebeU, 0x39393939U, - 0x4a4a4a4aU, 0x4c4c4c4cU, 0x58585858U, 0xcfcfcfcfU, - 0xd0d0d0d0U, 0xefefefefU, 0xaaaaaaaaU, 0xfbfbfbfbU, - 0x43434343U, 0x4d4d4d4dU, 0x33333333U, 0x85858585U, - 0x45454545U, 0xf9f9f9f9U, 0x02020202U, 0x7f7f7f7fU, - 0x50505050U, 0x3c3c3c3cU, 0x9f9f9f9fU, 0xa8a8a8a8U, - 0x51515151U, 0xa3a3a3a3U, 0x40404040U, 0x8f8f8f8fU, - 0x92929292U, 0x9d9d9d9dU, 0x38383838U, 0xf5f5f5f5U, - 0xbcbcbcbcU, 0xb6b6b6b6U, 0xdadadadaU, 0x21212121U, - 0x10101010U, 0xffffffffU, 0xf3f3f3f3U, 0xd2d2d2d2U, - 0xcdcdcdcdU, 0x0c0c0c0cU, 0x13131313U, 0xececececU, - 0x5f5f5f5fU, 0x97979797U, 0x44444444U, 0x17171717U, - 0xc4c4c4c4U, 0xa7a7a7a7U, 0x7e7e7e7eU, 0x3d3d3d3dU, - 0x64646464U, 0x5d5d5d5dU, 0x19191919U, 0x73737373U, - 0x60606060U, 0x81818181U, 0x4f4f4f4fU, 0xdcdcdcdcU, - 0x22222222U, 0x2a2a2a2aU, 0x90909090U, 0x88888888U, - 0x46464646U, 0xeeeeeeeeU, 0xb8b8b8b8U, 0x14141414U, - 0xdedededeU, 0x5e5e5e5eU, 0x0b0b0b0bU, 0xdbdbdbdbU, - 0xe0e0e0e0U, 0x32323232U, 0x3a3a3a3aU, 0x0a0a0a0aU, - 0x49494949U, 0x06060606U, 0x24242424U, 0x5c5c5c5cU, - 0xc2c2c2c2U, 0xd3d3d3d3U, 0xacacacacU, 0x62626262U, - 0x91919191U, 0x95959595U, 0xe4e4e4e4U, 0x79797979U, - 0xe7e7e7e7U, 0xc8c8c8c8U, 0x37373737U, 0x6d6d6d6dU, - 0x8d8d8d8dU, 0xd5d5d5d5U, 0x4e4e4e4eU, 0xa9a9a9a9U, - 0x6c6c6c6cU, 0x56565656U, 0xf4f4f4f4U, 0xeaeaeaeaU, - 0x65656565U, 0x7a7a7a7aU, 0xaeaeaeaeU, 0x08080808U, - 0xbabababaU, 0x78787878U, 0x25252525U, 0x2e2e2e2eU, - 0x1c1c1c1cU, 0xa6a6a6a6U, 0xb4b4b4b4U, 0xc6c6c6c6U, - 0xe8e8e8e8U, 0xddddddddU, 0x74747474U, 0x1f1f1f1fU, - 0x4b4b4b4bU, 0xbdbdbdbdU, 0x8b8b8b8bU, 0x8a8a8a8aU, - 0x70707070U, 0x3e3e3e3eU, 0xb5b5b5b5U, 0x66666666U, - 0x48484848U, 0x03030303U, 0xf6f6f6f6U, 0x0e0e0e0eU, - 0x61616161U, 0x35353535U, 0x57575757U, 0xb9b9b9b9U, - 0x86868686U, 0xc1c1c1c1U, 0x1d1d1d1dU, 0x9e9e9e9eU, - 0xe1e1e1e1U, 0xf8f8f8f8U, 0x98989898U, 0x11111111U, - 0x69696969U, 0xd9d9d9d9U, 0x8e8e8e8eU, 0x94949494U, - 0x9b9b9b9bU, 0x1e1e1e1eU, 0x87878787U, 0xe9e9e9e9U, - 0xcecececeU, 0x55555555U, 0x28282828U, 0xdfdfdfdfU, - 0x8c8c8c8cU, 0xa1a1a1a1U, 0x89898989U, 0x0d0d0d0dU, - 0xbfbfbfbfU, 0xe6e6e6e6U, 0x42424242U, 0x68686868U, - 0x41414141U, 0x99999999U, 0x2d2d2d2dU, 0x0f0f0f0fU, - 0xb0b0b0b0U, 0x54545454U, 0xbbbbbbbbU, 0x16161616U, -}; - -static const uint32 Td0[256]= -{ - 0x51f4a750U, 0x7e416553U, 0x1a17a4c3U, 0x3a275e96U, - 0x3bab6bcbU, 0x1f9d45f1U, 0xacfa58abU, 0x4be30393U, - 0x2030fa55U, 0xad766df6U, 0x88cc7691U, 0xf5024c25U, - 0x4fe5d7fcU, 0xc52acbd7U, 0x26354480U, 0xb562a38fU, - 0xdeb15a49U, 0x25ba1b67U, 0x45ea0e98U, 0x5dfec0e1U, - 0xc32f7502U, 0x814cf012U, 0x8d4697a3U, 0x6bd3f9c6U, - 0x038f5fe7U, 0x15929c95U, 0xbf6d7aebU, 0x955259daU, - 0xd4be832dU, 0x587421d3U, 0x49e06929U, 0x8ec9c844U, - 0x75c2896aU, 0xf48e7978U, 0x99583e6bU, 0x27b971ddU, - 0xbee14fb6U, 0xf088ad17U, 0xc920ac66U, 0x7dce3ab4U, - 0x63df4a18U, 0xe51a3182U, 0x97513360U, 0x62537f45U, - 0xb16477e0U, 0xbb6bae84U, 0xfe81a01cU, 0xf9082b94U, - 0x70486858U, 0x8f45fd19U, 0x94de6c87U, 0x527bf8b7U, - 0xab73d323U, 0x724b02e2U, 0xe31f8f57U, 0x6655ab2aU, - 0xb2eb2807U, 0x2fb5c203U, 0x86c57b9aU, 0xd33708a5U, - 0x302887f2U, 0x23bfa5b2U, 0x02036abaU, 0xed16825cU, - 0x8acf1c2bU, 0xa779b492U, 0xf307f2f0U, 0x4e69e2a1U, - 0x65daf4cdU, 0x0605bed5U, 0xd134621fU, 0xc4a6fe8aU, - 0x342e539dU, 0xa2f355a0U, 0x058ae132U, 0xa4f6eb75U, - 0x0b83ec39U, 0x4060efaaU, 0x5e719f06U, 0xbd6e1051U, - 0x3e218af9U, 0x96dd063dU, 0xdd3e05aeU, 0x4de6bd46U, - 0x91548db5U, 0x71c45d05U, 0x0406d46fU, 0x605015ffU, - 0x1998fb24U, 0xd6bde997U, 0x894043ccU, 0x67d99e77U, - 0xb0e842bdU, 0x07898b88U, 0xe7195b38U, 0x79c8eedbU, - 0xa17c0a47U, 0x7c420fe9U, 0xf8841ec9U, 0x00000000U, - 0x09808683U, 0x322bed48U, 0x1e1170acU, 0x6c5a724eU, - 0xfd0efffbU, 0x0f853856U, 0x3daed51eU, 0x362d3927U, - 0x0a0fd964U, 0x685ca621U, 0x9b5b54d1U, 0x24362e3aU, - 0x0c0a67b1U, 0x9357e70fU, 0xb4ee96d2U, 0x1b9b919eU, - 0x80c0c54fU, 0x61dc20a2U, 0x5a774b69U, 0x1c121a16U, - 0xe293ba0aU, 0xc0a02ae5U, 0x3c22e043U, 0x121b171dU, - 0x0e090d0bU, 0xf28bc7adU, 0x2db6a8b9U, 0x141ea9c8U, - 0x57f11985U, 0xaf75074cU, 0xee99ddbbU, 0xa37f60fdU, - 0xf701269fU, 0x5c72f5bcU, 0x44663bc5U, 0x5bfb7e34U, - 0x8b432976U, 0xcb23c6dcU, 0xb6edfc68U, 0xb8e4f163U, - 0xd731dccaU, 0x42638510U, 0x13972240U, 0x84c61120U, - 0x854a247dU, 0xd2bb3df8U, 0xaef93211U, 0xc729a16dU, - 0x1d9e2f4bU, 0xdcb230f3U, 0x0d8652ecU, 0x77c1e3d0U, - 0x2bb3166cU, 0xa970b999U, 0x119448faU, 0x47e96422U, - 0xa8fc8cc4U, 0xa0f03f1aU, 0x567d2cd8U, 0x223390efU, - 0x87494ec7U, 0xd938d1c1U, 0x8ccaa2feU, 0x98d40b36U, - 0xa6f581cfU, 0xa57ade28U, 0xdab78e26U, 0x3fadbfa4U, - 0x2c3a9de4U, 0x5078920dU, 0x6a5fcc9bU, 0x547e4662U, - 0xf68d13c2U, 0x90d8b8e8U, 0x2e39f75eU, 0x82c3aff5U, - 0x9f5d80beU, 0x69d0937cU, 0x6fd52da9U, 0xcf2512b3U, - 0xc8ac993bU, 0x10187da7U, 0xe89c636eU, 0xdb3bbb7bU, - 0xcd267809U, 0x6e5918f4U, 0xec9ab701U, 0x834f9aa8U, - 0xe6956e65U, 0xaaffe67eU, 0x21bccf08U, 0xef15e8e6U, - 0xbae79bd9U, 0x4a6f36ceU, 0xea9f09d4U, 0x29b07cd6U, - 0x31a4b2afU, 0x2a3f2331U, 0xc6a59430U, 0x35a266c0U, - 0x744ebc37U, 0xfc82caa6U, 0xe090d0b0U, 0x33a7d815U, - 0xf104984aU, 0x41ecdaf7U, 0x7fcd500eU, 0x1791f62fU, - 0x764dd68dU, 0x43efb04dU, 0xccaa4d54U, 0xe49604dfU, - 0x9ed1b5e3U, 0x4c6a881bU, 0xc12c1fb8U, 0x4665517fU, - 0x9d5eea04U, 0x018c355dU, 0xfa877473U, 0xfb0b412eU, - 0xb3671d5aU, 0x92dbd252U, 0xe9105633U, 0x6dd64713U, - 0x9ad7618cU, 0x37a10c7aU, 0x59f8148eU, 0xeb133c89U, - 0xcea927eeU, 0xb761c935U, 0xe11ce5edU, 0x7a47b13cU, - 0x9cd2df59U, 0x55f2733fU, 0x1814ce79U, 0x73c737bfU, - 0x53f7cdeaU, 0x5ffdaa5bU, 0xdf3d6f14U, 0x7844db86U, - 0xcaaff381U, 0xb968c43eU, 0x3824342cU, 0xc2a3405fU, - 0x161dc372U, 0xbce2250cU, 0x283c498bU, 0xff0d9541U, - 0x39a80171U, 0x080cb3deU, 0xd8b4e49cU, 0x6456c190U, - 0x7bcb8461U, 0xd532b670U, 0x486c5c74U, 0xd0b85742U, -}; - -static const uint32 Td1[256]= -{ - 0x5051f4a7U, 0x537e4165U, 0xc31a17a4U, 0x963a275eU, - 0xcb3bab6bU, 0xf11f9d45U, 0xabacfa58U, 0x934be303U, - 0x552030faU, 0xf6ad766dU, 0x9188cc76U, 0x25f5024cU, - 0xfc4fe5d7U, 0xd7c52acbU, 0x80263544U, 0x8fb562a3U, - 0x49deb15aU, 0x6725ba1bU, 0x9845ea0eU, 0xe15dfec0U, - 0x02c32f75U, 0x12814cf0U, 0xa38d4697U, 0xc66bd3f9U, - 0xe7038f5fU, 0x9515929cU, 0xebbf6d7aU, 0xda955259U, - 0x2dd4be83U, 0xd3587421U, 0x2949e069U, 0x448ec9c8U, - 0x6a75c289U, 0x78f48e79U, 0x6b99583eU, 0xdd27b971U, - 0xb6bee14fU, 0x17f088adU, 0x66c920acU, 0xb47dce3aU, - 0x1863df4aU, 0x82e51a31U, 0x60975133U, 0x4562537fU, - 0xe0b16477U, 0x84bb6baeU, 0x1cfe81a0U, 0x94f9082bU, - 0x58704868U, 0x198f45fdU, 0x8794de6cU, 0xb7527bf8U, - 0x23ab73d3U, 0xe2724b02U, 0x57e31f8fU, 0x2a6655abU, - 0x07b2eb28U, 0x032fb5c2U, 0x9a86c57bU, 0xa5d33708U, - 0xf2302887U, 0xb223bfa5U, 0xba02036aU, 0x5ced1682U, - 0x2b8acf1cU, 0x92a779b4U, 0xf0f307f2U, 0xa14e69e2U, - 0xcd65daf4U, 0xd50605beU, 0x1fd13462U, 0x8ac4a6feU, - 0x9d342e53U, 0xa0a2f355U, 0x32058ae1U, 0x75a4f6ebU, - 0x390b83ecU, 0xaa4060efU, 0x065e719fU, 0x51bd6e10U, - 0xf93e218aU, 0x3d96dd06U, 0xaedd3e05U, 0x464de6bdU, - 0xb591548dU, 0x0571c45dU, 0x6f0406d4U, 0xff605015U, - 0x241998fbU, 0x97d6bde9U, 0xcc894043U, 0x7767d99eU, - 0xbdb0e842U, 0x8807898bU, 0x38e7195bU, 0xdb79c8eeU, - 0x47a17c0aU, 0xe97c420fU, 0xc9f8841eU, 0x00000000U, - 0x83098086U, 0x48322bedU, 0xac1e1170U, 0x4e6c5a72U, - 0xfbfd0effU, 0x560f8538U, 0x1e3daed5U, 0x27362d39U, - 0x640a0fd9U, 0x21685ca6U, 0xd19b5b54U, 0x3a24362eU, - 0xb10c0a67U, 0x0f9357e7U, 0xd2b4ee96U, 0x9e1b9b91U, - 0x4f80c0c5U, 0xa261dc20U, 0x695a774bU, 0x161c121aU, - 0x0ae293baU, 0xe5c0a02aU, 0x433c22e0U, 0x1d121b17U, - 0x0b0e090dU, 0xadf28bc7U, 0xb92db6a8U, 0xc8141ea9U, - 0x8557f119U, 0x4caf7507U, 0xbbee99ddU, 0xfda37f60U, - 0x9ff70126U, 0xbc5c72f5U, 0xc544663bU, 0x345bfb7eU, - 0x768b4329U, 0xdccb23c6U, 0x68b6edfcU, 0x63b8e4f1U, - 0xcad731dcU, 0x10426385U, 0x40139722U, 0x2084c611U, - 0x7d854a24U, 0xf8d2bb3dU, 0x11aef932U, 0x6dc729a1U, - 0x4b1d9e2fU, 0xf3dcb230U, 0xec0d8652U, 0xd077c1e3U, - 0x6c2bb316U, 0x99a970b9U, 0xfa119448U, 0x2247e964U, - 0xc4a8fc8cU, 0x1aa0f03fU, 0xd8567d2cU, 0xef223390U, - 0xc787494eU, 0xc1d938d1U, 0xfe8ccaa2U, 0x3698d40bU, - 0xcfa6f581U, 0x28a57adeU, 0x26dab78eU, 0xa43fadbfU, - 0xe42c3a9dU, 0x0d507892U, 0x9b6a5fccU, 0x62547e46U, - 0xc2f68d13U, 0xe890d8b8U, 0x5e2e39f7U, 0xf582c3afU, - 0xbe9f5d80U, 0x7c69d093U, 0xa96fd52dU, 0xb3cf2512U, - 0x3bc8ac99U, 0xa710187dU, 0x6ee89c63U, 0x7bdb3bbbU, - 0x09cd2678U, 0xf46e5918U, 0x01ec9ab7U, 0xa8834f9aU, - 0x65e6956eU, 0x7eaaffe6U, 0x0821bccfU, 0xe6ef15e8U, - 0xd9bae79bU, 0xce4a6f36U, 0xd4ea9f09U, 0xd629b07cU, - 0xaf31a4b2U, 0x312a3f23U, 0x30c6a594U, 0xc035a266U, - 0x37744ebcU, 0xa6fc82caU, 0xb0e090d0U, 0x1533a7d8U, - 0x4af10498U, 0xf741ecdaU, 0x0e7fcd50U, 0x2f1791f6U, - 0x8d764dd6U, 0x4d43efb0U, 0x54ccaa4dU, 0xdfe49604U, - 0xe39ed1b5U, 0x1b4c6a88U, 0xb8c12c1fU, 0x7f466551U, - 0x049d5eeaU, 0x5d018c35U, 0x73fa8774U, 0x2efb0b41U, - 0x5ab3671dU, 0x5292dbd2U, 0x33e91056U, 0x136dd647U, - 0x8c9ad761U, 0x7a37a10cU, 0x8e59f814U, 0x89eb133cU, - 0xeecea927U, 0x35b761c9U, 0xede11ce5U, 0x3c7a47b1U, - 0x599cd2dfU, 0x3f55f273U, 0x791814ceU, 0xbf73c737U, - 0xea53f7cdU, 0x5b5ffdaaU, 0x14df3d6fU, 0x867844dbU, - 0x81caaff3U, 0x3eb968c4U, 0x2c382434U, 0x5fc2a340U, - 0x72161dc3U, 0x0cbce225U, 0x8b283c49U, 0x41ff0d95U, - 0x7139a801U, 0xde080cb3U, 0x9cd8b4e4U, 0x906456c1U, - 0x617bcb84U, 0x70d532b6U, 0x74486c5cU, 0x42d0b857U, -}; - -static const uint32 Td2[256]= -{ - 0xa75051f4U, 0x65537e41U, 0xa4c31a17U, 0x5e963a27U, - 0x6bcb3babU, 0x45f11f9dU, 0x58abacfaU, 0x03934be3U, - 0xfa552030U, 0x6df6ad76U, 0x769188ccU, 0x4c25f502U, - 0xd7fc4fe5U, 0xcbd7c52aU, 0x44802635U, 0xa38fb562U, - 0x5a49deb1U, 0x1b6725baU, 0x0e9845eaU, 0xc0e15dfeU, - 0x7502c32fU, 0xf012814cU, 0x97a38d46U, 0xf9c66bd3U, - 0x5fe7038fU, 0x9c951592U, 0x7aebbf6dU, 0x59da9552U, - 0x832dd4beU, 0x21d35874U, 0x692949e0U, 0xc8448ec9U, - 0x896a75c2U, 0x7978f48eU, 0x3e6b9958U, 0x71dd27b9U, - 0x4fb6bee1U, 0xad17f088U, 0xac66c920U, 0x3ab47dceU, - 0x4a1863dfU, 0x3182e51aU, 0x33609751U, 0x7f456253U, - 0x77e0b164U, 0xae84bb6bU, 0xa01cfe81U, 0x2b94f908U, - 0x68587048U, 0xfd198f45U, 0x6c8794deU, 0xf8b7527bU, - 0xd323ab73U, 0x02e2724bU, 0x8f57e31fU, 0xab2a6655U, - 0x2807b2ebU, 0xc2032fb5U, 0x7b9a86c5U, 0x08a5d337U, - 0x87f23028U, 0xa5b223bfU, 0x6aba0203U, 0x825ced16U, - 0x1c2b8acfU, 0xb492a779U, 0xf2f0f307U, 0xe2a14e69U, - 0xf4cd65daU, 0xbed50605U, 0x621fd134U, 0xfe8ac4a6U, - 0x539d342eU, 0x55a0a2f3U, 0xe132058aU, 0xeb75a4f6U, - 0xec390b83U, 0xefaa4060U, 0x9f065e71U, 0x1051bd6eU, - - 0x8af93e21U, 0x063d96ddU, 0x05aedd3eU, 0xbd464de6U, - 0x8db59154U, 0x5d0571c4U, 0xd46f0406U, 0x15ff6050U, - 0xfb241998U, 0xe997d6bdU, 0x43cc8940U, 0x9e7767d9U, - 0x42bdb0e8U, 0x8b880789U, 0x5b38e719U, 0xeedb79c8U, - 0x0a47a17cU, 0x0fe97c42U, 0x1ec9f884U, 0x00000000U, - 0x86830980U, 0xed48322bU, 0x70ac1e11U, 0x724e6c5aU, - 0xfffbfd0eU, 0x38560f85U, 0xd51e3daeU, 0x3927362dU, - 0xd9640a0fU, 0xa621685cU, 0x54d19b5bU, 0x2e3a2436U, - 0x67b10c0aU, 0xe70f9357U, 0x96d2b4eeU, 0x919e1b9bU, - 0xc54f80c0U, 0x20a261dcU, 0x4b695a77U, 0x1a161c12U, - 0xba0ae293U, 0x2ae5c0a0U, 0xe0433c22U, 0x171d121bU, - 0x0d0b0e09U, 0xc7adf28bU, 0xa8b92db6U, 0xa9c8141eU, - 0x198557f1U, 0x074caf75U, 0xddbbee99U, 0x60fda37fU, - 0x269ff701U, 0xf5bc5c72U, 0x3bc54466U, 0x7e345bfbU, - 0x29768b43U, 0xc6dccb23U, 0xfc68b6edU, 0xf163b8e4U, - 0xdccad731U, 0x85104263U, 0x22401397U, 0x112084c6U, - 0x247d854aU, 0x3df8d2bbU, 0x3211aef9U, 0xa16dc729U, - 0x2f4b1d9eU, 0x30f3dcb2U, 0x52ec0d86U, 0xe3d077c1U, - 0x166c2bb3U, 0xb999a970U, 0x48fa1194U, 0x642247e9U, - 0x8cc4a8fcU, 0x3f1aa0f0U, 0x2cd8567dU, 0x90ef2233U, - 0x4ec78749U, 0xd1c1d938U, 0xa2fe8ccaU, 0x0b3698d4U, - 0x81cfa6f5U, 0xde28a57aU, 0x8e26dab7U, 0xbfa43fadU, - 0x9de42c3aU, 0x920d5078U, 0xcc9b6a5fU, 0x4662547eU, - 0x13c2f68dU, 0xb8e890d8U, 0xf75e2e39U, 0xaff582c3U, - 0x80be9f5dU, 0x937c69d0U, 0x2da96fd5U, 0x12b3cf25U, - 0x993bc8acU, 0x7da71018U, 0x636ee89cU, 0xbb7bdb3bU, - 0x7809cd26U, 0x18f46e59U, 0xb701ec9aU, 0x9aa8834fU, - 0x6e65e695U, 0xe67eaaffU, 0xcf0821bcU, 0xe8e6ef15U, - 0x9bd9bae7U, 0x36ce4a6fU, 0x09d4ea9fU, 0x7cd629b0U, - 0xb2af31a4U, 0x23312a3fU, 0x9430c6a5U, 0x66c035a2U, - 0xbc37744eU, 0xcaa6fc82U, 0xd0b0e090U, 0xd81533a7U, - 0x984af104U, 0xdaf741ecU, 0x500e7fcdU, 0xf62f1791U, - 0xd68d764dU, 0xb04d43efU, 0x4d54ccaaU, 0x04dfe496U, - 0xb5e39ed1U, 0x881b4c6aU, 0x1fb8c12cU, 0x517f4665U, - 0xea049d5eU, 0x355d018cU, 0x7473fa87U, 0x412efb0bU, - 0x1d5ab367U, 0xd25292dbU, 0x5633e910U, 0x47136dd6U, - 0x618c9ad7U, 0x0c7a37a1U, 0x148e59f8U, 0x3c89eb13U, - 0x27eecea9U, 0xc935b761U, 0xe5ede11cU, 0xb13c7a47U, - 0xdf599cd2U, 0x733f55f2U, 0xce791814U, 0x37bf73c7U, - 0xcdea53f7U, 0xaa5b5ffdU, 0x6f14df3dU, 0xdb867844U, - 0xf381caafU, 0xc43eb968U, 0x342c3824U, 0x405fc2a3U, - 0xc372161dU, 0x250cbce2U, 0x498b283cU, 0x9541ff0dU, - 0x017139a8U, 0xb3de080cU, 0xe49cd8b4U, 0xc1906456U, - 0x84617bcbU, 0xb670d532U, 0x5c74486cU, 0x5742d0b8U, -}; - -static const uint32 Td3[256]= -{ - 0xf4a75051U, 0x4165537eU, 0x17a4c31aU, 0x275e963aU, - 0xab6bcb3bU, 0x9d45f11fU, 0xfa58abacU, 0xe303934bU, - 0x30fa5520U, 0x766df6adU, 0xcc769188U, 0x024c25f5U, - 0xe5d7fc4fU, 0x2acbd7c5U, 0x35448026U, 0x62a38fb5U, - 0xb15a49deU, 0xba1b6725U, 0xea0e9845U, 0xfec0e15dU, - 0x2f7502c3U, 0x4cf01281U, 0x4697a38dU, 0xd3f9c66bU, - 0x8f5fe703U, 0x929c9515U, 0x6d7aebbfU, 0x5259da95U, - 0xbe832dd4U, 0x7421d358U, 0xe0692949U, 0xc9c8448eU, - 0xc2896a75U, 0x8e7978f4U, 0x583e6b99U, 0xb971dd27U, - 0xe14fb6beU, 0x88ad17f0U, 0x20ac66c9U, 0xce3ab47dU, - 0xdf4a1863U, 0x1a3182e5U, 0x51336097U, 0x537f4562U, - 0x6477e0b1U, 0x6bae84bbU, 0x81a01cfeU, 0x082b94f9U, - 0x48685870U, 0x45fd198fU, 0xde6c8794U, 0x7bf8b752U, - 0x73d323abU, 0x4b02e272U, 0x1f8f57e3U, 0x55ab2a66U, - 0xeb2807b2U, 0xb5c2032fU, 0xc57b9a86U, 0x3708a5d3U, - 0x2887f230U, 0xbfa5b223U, 0x036aba02U, 0x16825cedU, - 0xcf1c2b8aU, 0x79b492a7U, 0x07f2f0f3U, 0x69e2a14eU, - 0xdaf4cd65U, 0x05bed506U, 0x34621fd1U, 0xa6fe8ac4U, - 0x2e539d34U, 0xf355a0a2U, 0x8ae13205U, 0xf6eb75a4U, - 0x83ec390bU, 0x60efaa40U, 0x719f065eU, 0x6e1051bdU, - 0x218af93eU, 0xdd063d96U, 0x3e05aeddU, 0xe6bd464dU, - 0x548db591U, 0xc45d0571U, 0x06d46f04U, 0x5015ff60U, - 0x98fb2419U, 0xbde997d6U, 0x4043cc89U, 0xd99e7767U, - 0xe842bdb0U, 0x898b8807U, 0x195b38e7U, 0xc8eedb79U, - 0x7c0a47a1U, 0x420fe97cU, 0x841ec9f8U, 0x00000000U, - 0x80868309U, 0x2bed4832U, 0x1170ac1eU, 0x5a724e6cU, - 0x0efffbfdU, 0x8538560fU, 0xaed51e3dU, 0x2d392736U, - 0x0fd9640aU, 0x5ca62168U, 0x5b54d19bU, 0x362e3a24U, - 0x0a67b10cU, 0x57e70f93U, 0xee96d2b4U, 0x9b919e1bU, - 0xc0c54f80U, 0xdc20a261U, 0x774b695aU, 0x121a161cU, - 0x93ba0ae2U, 0xa02ae5c0U, 0x22e0433cU, 0x1b171d12U, - 0x090d0b0eU, 0x8bc7adf2U, 0xb6a8b92dU, 0x1ea9c814U, - 0xf1198557U, 0x75074cafU, 0x99ddbbeeU, 0x7f60fda3U, - 0x01269ff7U, 0x72f5bc5cU, 0x663bc544U, 0xfb7e345bU, - 0x4329768bU, 0x23c6dccbU, 0xedfc68b6U, 0xe4f163b8U, - 0x31dccad7U, 0x63851042U, 0x97224013U, 0xc6112084U, - 0x4a247d85U, 0xbb3df8d2U, 0xf93211aeU, 0x29a16dc7U, - 0x9e2f4b1dU, 0xb230f3dcU, 0x8652ec0dU, 0xc1e3d077U, - 0xb3166c2bU, 0x70b999a9U, 0x9448fa11U, 0xe9642247U, - 0xfc8cc4a8U, 0xf03f1aa0U, 0x7d2cd856U, 0x3390ef22U, - 0x494ec787U, 0x38d1c1d9U, 0xcaa2fe8cU, 0xd40b3698U, - 0xf581cfa6U, 0x7ade28a5U, 0xb78e26daU, 0xadbfa43fU, - 0x3a9de42cU, 0x78920d50U, 0x5fcc9b6aU, 0x7e466254U, - 0x8d13c2f6U, 0xd8b8e890U, 0x39f75e2eU, 0xc3aff582U, - 0x5d80be9fU, 0xd0937c69U, 0xd52da96fU, 0x2512b3cfU, - 0xac993bc8U, 0x187da710U, 0x9c636ee8U, 0x3bbb7bdbU, - 0x267809cdU, 0x5918f46eU, 0x9ab701ecU, 0x4f9aa883U, - 0x956e65e6U, 0xffe67eaaU, 0xbccf0821U, 0x15e8e6efU, - 0xe79bd9baU, 0x6f36ce4aU, 0x9f09d4eaU, 0xb07cd629U, - 0xa4b2af31U, 0x3f23312aU, 0xa59430c6U, 0xa266c035U, - 0x4ebc3774U, 0x82caa6fcU, 0x90d0b0e0U, 0xa7d81533U, - 0x04984af1U, 0xecdaf741U, 0xcd500e7fU, 0x91f62f17U, - 0x4dd68d76U, 0xefb04d43U, 0xaa4d54ccU, 0x9604dfe4U, - 0xd1b5e39eU, 0x6a881b4cU, 0x2c1fb8c1U, 0x65517f46U, - 0x5eea049dU, 0x8c355d01U, 0x877473faU, 0x0b412efbU, - 0x671d5ab3U, 0xdbd25292U, 0x105633e9U, 0xd647136dU, - 0xd7618c9aU, 0xa10c7a37U, 0xf8148e59U, 0x133c89ebU, - 0xa927eeceU, 0x61c935b7U, 0x1ce5ede1U, 0x47b13c7aU, - 0xd2df599cU, 0xf2733f55U, 0x14ce7918U, 0xc737bf73U, - 0xf7cdea53U, 0xfdaa5b5fU, 0x3d6f14dfU, 0x44db8678U, - 0xaff381caU, 0x68c43eb9U, 0x24342c38U, 0xa3405fc2U, - 0x1dc37216U, 0xe2250cbcU, 0x3c498b28U, 0x0d9541ffU, - 0xa8017139U, 0x0cb3de08U, 0xb4e49cd8U, 0x56c19064U, - 0xcb84617bU, 0x32b670d5U, 0x6c5c7448U, 0xb85742d0U, -}; - -static const uint32 Td4[256]= -{ - 0x52525252U, 0x09090909U, 0x6a6a6a6aU, 0xd5d5d5d5U, - 0x30303030U, 0x36363636U, 0xa5a5a5a5U, 0x38383838U, - 0xbfbfbfbfU, 0x40404040U, 0xa3a3a3a3U, 0x9e9e9e9eU, - 0x81818181U, 0xf3f3f3f3U, 0xd7d7d7d7U, 0xfbfbfbfbU, - 0x7c7c7c7cU, 0xe3e3e3e3U, 0x39393939U, 0x82828282U, - 0x9b9b9b9bU, 0x2f2f2f2fU, 0xffffffffU, 0x87878787U, - 0x34343434U, 0x8e8e8e8eU, 0x43434343U, 0x44444444U, - 0xc4c4c4c4U, 0xdedededeU, 0xe9e9e9e9U, 0xcbcbcbcbU, - 0x54545454U, 0x7b7b7b7bU, 0x94949494U, 0x32323232U, - 0xa6a6a6a6U, 0xc2c2c2c2U, 0x23232323U, 0x3d3d3d3dU, - 0xeeeeeeeeU, 0x4c4c4c4cU, 0x95959595U, 0x0b0b0b0bU, - 0x42424242U, 0xfafafafaU, 0xc3c3c3c3U, 0x4e4e4e4eU, - 0x08080808U, 0x2e2e2e2eU, 0xa1a1a1a1U, 0x66666666U, - 0x28282828U, 0xd9d9d9d9U, 0x24242424U, 0xb2b2b2b2U, - 0x76767676U, 0x5b5b5b5bU, 0xa2a2a2a2U, 0x49494949U, - 0x6d6d6d6dU, 0x8b8b8b8bU, 0xd1d1d1d1U, 0x25252525U, - 0x72727272U, 0xf8f8f8f8U, 0xf6f6f6f6U, 0x64646464U, - 0x86868686U, 0x68686868U, 0x98989898U, 0x16161616U, - 0xd4d4d4d4U, 0xa4a4a4a4U, 0x5c5c5c5cU, 0xccccccccU, - 0x5d5d5d5dU, 0x65656565U, 0xb6b6b6b6U, 0x92929292U, - 0x6c6c6c6cU, 0x70707070U, 0x48484848U, 0x50505050U, - 0xfdfdfdfdU, 0xededededU, 0xb9b9b9b9U, 0xdadadadaU, - 0x5e5e5e5eU, 0x15151515U, 0x46464646U, 0x57575757U, - 0xa7a7a7a7U, 0x8d8d8d8dU, 0x9d9d9d9dU, 0x84848484U, - 0x90909090U, 0xd8d8d8d8U, 0xababababU, 0x00000000U, - 0x8c8c8c8cU, 0xbcbcbcbcU, 0xd3d3d3d3U, 0x0a0a0a0aU, - 0xf7f7f7f7U, 0xe4e4e4e4U, 0x58585858U, 0x05050505U, - 0xb8b8b8b8U, 0xb3b3b3b3U, 0x45454545U, 0x06060606U, - 0xd0d0d0d0U, 0x2c2c2c2cU, 0x1e1e1e1eU, 0x8f8f8f8fU, - 0xcacacacaU, 0x3f3f3f3fU, 0x0f0f0f0fU, 0x02020202U, - 0xc1c1c1c1U, 0xafafafafU, 0xbdbdbdbdU, 0x03030303U, - 0x01010101U, 0x13131313U, 0x8a8a8a8aU, 0x6b6b6b6bU, - 0x3a3a3a3aU, 0x91919191U, 0x11111111U, 0x41414141U, - 0x4f4f4f4fU, 0x67676767U, 0xdcdcdcdcU, 0xeaeaeaeaU, - 0x97979797U, 0xf2f2f2f2U, 0xcfcfcfcfU, 0xcecececeU, - 0xf0f0f0f0U, 0xb4b4b4b4U, 0xe6e6e6e6U, 0x73737373U, - 0x96969696U, 0xacacacacU, 0x74747474U, 0x22222222U, - 0xe7e7e7e7U, 0xadadadadU, 0x35353535U, 0x85858585U, - 0xe2e2e2e2U, 0xf9f9f9f9U, 0x37373737U, 0xe8e8e8e8U, - 0x1c1c1c1cU, 0x75757575U, 0xdfdfdfdfU, 0x6e6e6e6eU, - 0x47474747U, 0xf1f1f1f1U, 0x1a1a1a1aU, 0x71717171U, - 0x1d1d1d1dU, 0x29292929U, 0xc5c5c5c5U, 0x89898989U, - 0x6f6f6f6fU, 0xb7b7b7b7U, 0x62626262U, 0x0e0e0e0eU, - 0xaaaaaaaaU, 0x18181818U, 0xbebebebeU, 0x1b1b1b1bU, - 0xfcfcfcfcU, 0x56565656U, 0x3e3e3e3eU, 0x4b4b4b4bU, - 0xc6c6c6c6U, 0xd2d2d2d2U, 0x79797979U, 0x20202020U, - 0x9a9a9a9aU, 0xdbdbdbdbU, 0xc0c0c0c0U, 0xfefefefeU, - 0x78787878U, 0xcdcdcdcdU, 0x5a5a5a5aU, 0xf4f4f4f4U, - 0x1f1f1f1fU, 0xddddddddU, 0xa8a8a8a8U, 0x33333333U, - 0x88888888U, 0x07070707U, 0xc7c7c7c7U, 0x31313131U, - 0xb1b1b1b1U, 0x12121212U, 0x10101010U, 0x59595959U, - 0x27272727U, 0x80808080U, 0xececececU, 0x5f5f5f5fU, - 0x60606060U, 0x51515151U, 0x7f7f7f7fU, 0xa9a9a9a9U, - 0x19191919U, 0xb5b5b5b5U, 0x4a4a4a4aU, 0x0d0d0d0dU, - 0x2d2d2d2dU, 0xe5e5e5e5U, 0x7a7a7a7aU, 0x9f9f9f9fU, - 0x93939393U, 0xc9c9c9c9U, 0x9c9c9c9cU, 0xefefefefU, - 0xa0a0a0a0U, 0xe0e0e0e0U, 0x3b3b3b3bU, 0x4d4d4d4dU, - 0xaeaeaeaeU, 0x2a2a2a2aU, 0xf5f5f5f5U, 0xb0b0b0b0U, - 0xc8c8c8c8U, 0xebebebebU, 0xbbbbbbbbU, 0x3c3c3c3cU, - 0x83838383U, 0x53535353U, 0x99999999U, 0x61616161U, - 0x17171717U, 0x2b2b2b2bU, 0x04040404U, 0x7e7e7e7eU, - 0xbabababaU, 0x77777777U, 0xd6d6d6d6U, 0x26262626U, - 0xe1e1e1e1U, 0x69696969U, 0x14141414U, 0x63636363U, - 0x55555555U, 0x21212121U, 0x0c0c0c0cU, 0x7d7d7d7dU, -}; - - -/* for 128-bit blocks, Rijndael never uses more than 10 rcon values */ -static const uint32 rcon[]= -{ - 0x01000000, 0x02000000, 0x04000000, 0x08000000, - 0x10000000, 0x20000000, 0x40000000, 0x80000000, - 0x1B000000, 0x36000000, -}; - -#if defined(_MSC_VER) && defined(__i386__) - -#define RJ_SWAP(x) (_lrotl(x, 8) & 0x00ff00ff | _lrotr(x, 8) & 0xff00ff00) -#define GETuint32(p) RJ_SWAP(*((uint32 *)(p))) -#define PUTuint32(ct, st) { *((uint32 *)(ct)) = RJ_SWAP((st)); } - -#else - -#define GETuint32(pt) (((uint32)(pt)[0] << 24) ^ ((uint32)(pt)[1] << 16)\ - ^ ((uint32)(pt)[2] << 8) ^ ((uint32)(pt)[3])) -#define PUTuint32(ct, st) { (ct)[0] = (uint8)((st) >> 24); (ct)[1]\ -= (uint8)((st) >> 16); (ct)[2] = (uint8)((st) >> 8); (ct)[3] = (uint8)(st); } - -#endif /* defined(_MSC_VER) && defined(__i386__) */ - - -/* - Expand the cipher key into the encryption key schedule. - - RETURN - The number of rounds for the given cipher key size. -*/ - -int rijndaelKeySetupEnc(uint32 rk[/*4*(Nr + 1)*/], const uint8 cipherKey[], - int keyBits) -{ - int i = 0; - uint32 temp; - - rk[0] = GETuint32(cipherKey ); - rk[1] = GETuint32(cipherKey + 4); - rk[2] = GETuint32(cipherKey + 8); - rk[3] = GETuint32(cipherKey + 12); - if (keyBits == 128) - { - for (;;) - { - temp = rk[3]; - rk[4] = (rk[0] ^ - (Te4[(temp >> 16) & 0xff] & 0xff000000) ^ - (Te4[(temp >> 8) & 0xff] & 0x00ff0000) ^ - (Te4[(temp ) & 0xff] & 0x0000ff00) ^ - (Te4[(temp >> 24) ] & 0x000000ff) ^ - rcon[i]); - rk[5] = rk[1] ^ rk[4]; - rk[6] = rk[2] ^ rk[5]; - rk[7] = rk[3] ^ rk[6]; - if (++i == 10) - return 10; - rk += 4; - } - } - rk[4] = GETuint32(cipherKey + 16); - rk[5] = GETuint32(cipherKey + 20); - if (keyBits == 192) - { - for (;;) - { - temp = rk[ 5]; - rk[ 6] = (rk[ 0] ^ - (Te4[(temp >> 16) & 0xff] & 0xff000000) ^ - (Te4[(temp >> 8) & 0xff] & 0x00ff0000) ^ - (Te4[(temp ) & 0xff] & 0x0000ff00) ^ - (Te4[(temp >> 24) ] & 0x000000ff) ^ - rcon[i]); - rk[ 7] = rk[ 1] ^ rk[ 6]; - rk[ 8] = rk[ 2] ^ rk[ 7]; - rk[ 9] = rk[ 3] ^ rk[ 8]; - if (++i == 8) - { - return 12; - } - rk[10] = rk[ 4] ^ rk[ 9]; - rk[11] = rk[ 5] ^ rk[10]; - rk += 6; - } - } - rk[6] = GETuint32(cipherKey + 24); - rk[7] = GETuint32(cipherKey + 28); - if (keyBits == 256) - { - for (;;) - { - temp = rk[ 7]; - rk[ 8] = (rk[ 0] ^ - (Te4[(temp >> 16) & 0xff] & 0xff000000) ^ - (Te4[(temp >> 8) & 0xff] & 0x00ff0000) ^ - (Te4[(temp ) & 0xff] & 0x0000ff00) ^ - (Te4[(temp >> 24) ] & 0x000000ff) ^ - rcon[i]); - rk[ 9] = rk[ 1] ^ rk[ 8]; - rk[10] = rk[ 2] ^ rk[ 9]; - rk[11] = rk[ 3] ^ rk[10]; - if (++i == 7) - { - return 14; - } - temp = rk[11]; - rk[12] = (rk[ 4] ^ - (Te4[(temp >> 24) ] & 0xff000000) ^ - (Te4[(temp >> 16) & 0xff] & 0x00ff0000) ^ - (Te4[(temp >> 8) & 0xff] & 0x0000ff00) ^ - (Te4[(temp ) & 0xff] & 0x000000ff)); - rk[13] = rk[ 5] ^ rk[12]; - rk[14] = rk[ 6] ^ rk[13]; - rk[15] = rk[ 7] ^ rk[14]; - rk += 8; - } - } - return 0; -} - - -/* - Expand the cipher key into the decryption key schedule. - - RETURN - The number of rounds for the given cipher key size. -*/ - -int rijndaelKeySetupDec(uint32 rk[/*4*(Nr + 1)*/], const uint8 cipherKey[], - int keyBits) -{ - int nr, i, j; - uint32 temp; - - /* expand the cipher key: */ - nr = rijndaelKeySetupEnc(rk, cipherKey, keyBits); - /* invert the order of the round keys: */ - for (i = 0, j = 4*nr; i < j; i += 4, j -= 4) - { - temp = rk[i ]; rk[i ] = rk[j ]; rk[j ] = temp; - temp = rk[i + 1]; rk[i + 1] = rk[j + 1]; rk[j + 1] = temp; - temp = rk[i + 2]; rk[i + 2] = rk[j + 2]; rk[j + 2] = temp; - temp = rk[i + 3]; rk[i + 3] = rk[j + 3]; rk[j + 3] = temp; - } - /* - Apply the inverse MixColumn transform to all round keys but the first - and the last: - */ - for (i = 1; i < nr; i++) - { - rk += 4; - - rk[0]= ( - Td0[Te4[(rk[0] >> 24) ] & 0xff] ^ - Td1[Te4[(rk[0] >> 16) & 0xff] & 0xff] ^ - Td2[Te4[(rk[0] >> 8) & 0xff] & 0xff] ^ - Td3[Te4[(rk[0] ) & 0xff] & 0xff]); - - rk[1]= (Td0[Te4[(rk[1] >> 24) ] & 0xff] ^ - Td1[Te4[(rk[1] >> 16) & 0xff] & 0xff] ^ - Td2[Te4[(rk[1] >> 8) & 0xff] & 0xff] ^ - Td3[Te4[(rk[1] ) & 0xff] & 0xff]); - - rk[2]= (Td0[Te4[(rk[2] >> 24) ] & 0xff] ^ - Td1[Te4[(rk[2] >> 16) & 0xff] & 0xff] ^ - Td2[Te4[(rk[2] >> 8) & 0xff] & 0xff] ^ - Td3[Te4[(rk[2] ) & 0xff] & 0xff]); - - rk[3]= (Td0[Te4[(rk[3] >> 24) ] & 0xff] ^ - Td1[Te4[(rk[3] >> 16) & 0xff] & 0xff] ^ - Td2[Te4[(rk[3] >> 8) & 0xff] & 0xff] ^ - Td3[Te4[(rk[3] ) & 0xff] & 0xff]); - } - return nr; -} - - -void rijndaelEncrypt(const uint32 rk[/*4*(Nr + 1)*/], int Nr, - const uint8 pt[16], uint8 ct[16]) -{ - uint32 s0, s1, s2, s3, t0, t1, t2, t3; -#ifndef FULL_UNROLL - int r; -#endif /* FULL_UNROLL */ - - /* map byte array block to cipher state and add initial round key: */ - s0 = GETuint32(pt ) ^ rk[0]; - s1 = GETuint32(pt + 4) ^ rk[1]; - s2 = GETuint32(pt + 8) ^ rk[2]; - s3 = GETuint32(pt + 12) ^ rk[3]; - -#ifdef FULL_UNROLL - /* round 1: */ - t0= (Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >> 8) & 0xff] - ^ Te3[s3 & 0xff] ^ rk[ 4]); - t1= (Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >> 8) & 0xff] - ^ Te3[s0 & 0xff] ^ rk[ 5]); - t2= (Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >> 8) & 0xff] - ^ Te3[s1 & 0xff] ^ rk[ 6]); - t3= (Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >> 8) & 0xff] - ^ Te3[s2 & 0xff] ^ rk[ 7]); - - /* round 2: */ - s0= (Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ Te2[(t2 >> 8) & 0xff] - ^ Te3[t3 & 0xff] ^ rk[ 8]); - s1= (Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ Te2[(t3 >> 8) & 0xff] - ^ Te3[t0 & 0xff] ^ rk[ 9]); - s2= (Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ Te2[(t0 >> 8) & 0xff] - ^ Te3[t1 & 0xff] ^ rk[10]); - s3= (Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ Te2[(t1 >> 8) & 0xff] - ^ Te3[t2 & 0xff] ^ rk[11]); - - /* round 3: */ - t0= (Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >> 8) & 0xff] - ^ Te3[s3 & 0xff] ^ rk[12]); - t1= (Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >> 8) & 0xff] - ^ Te3[s0 & 0xff] ^ rk[13]); - t2= (Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >> 8) & 0xff] - ^ Te3[s1 & 0xff] ^ rk[14]); - t3= (Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >> 8) & 0xff] - ^ Te3[s2 & 0xff] ^ rk[15]); - - /* round 4: */ - s0= (Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ Te2[(t2 >> 8) & 0xff] - ^ Te3[t3 & 0xff] ^ rk[16]); - s1= (Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ Te2[(t3 >> 8) & 0xff] - ^ Te3[t0 & 0xff] ^ rk[17]); - s2= (Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ Te2[(t0 >> 8) & 0xff] - ^ Te3[t1 & 0xff] ^ rk[18]); - s3= (Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ Te2[(t1 >> 8) & 0xff] - ^ Te3[t2 & 0xff] ^ rk[19]); - - /* round 5: */ - t0= (Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >> 8) & 0xff] - ^ Te3[s3 & 0xff] ^ rk[20]); - t1= (Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >> 8) & 0xff] - ^ Te3[s0 & 0xff] ^ rk[21]); - t2= (Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >> 8) & 0xff] - ^ Te3[s1 & 0xff] ^ rk[22]); - t3= (Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >> 8) & 0xff] - ^ Te3[s2 & 0xff] ^ rk[23]); - - /* round 6: */ - s0= (Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ Te2[(t2 >> 8) & 0xff] - ^ Te3[t3 & 0xff] ^ rk[24]); - s1= (Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ Te2[(t3 >> 8) & 0xff] - ^ Te3[t0 & 0xff] ^ rk[25]); - s2= (Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ Te2[(t0 >> 8) & 0xff] - ^ Te3[t1 & 0xff] ^ rk[26]); - s3= (Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ Te2[(t1 >> 8) & 0xff] - ^ Te3[t2 & 0xff] ^ rk[27]); - - /* round 7: */ - t0= (Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >> 8) & 0xff] - ^ Te3[s3 & 0xff] ^ rk[28]); - t1= (Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >> 8) & 0xff] - ^ Te3[s0 & 0xff] ^ rk[29]); - t2= (Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >> 8) & 0xff] - ^ Te3[s1 & 0xff] ^ rk[30]); - t3= (Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >> 8) & 0xff] - ^ Te3[s2 & 0xff] ^ rk[31]); - - /* round 8: */ - s0= (Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ Te2[(t2 >> 8) & 0xff] - ^ Te3[t3 & 0xff] ^ rk[32]); - s1= (Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ Te2[(t3 >> 8) & 0xff] - ^ Te3[t0 & 0xff] ^ rk[33]); - s2= (Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ Te2[(t0 >> 8) & 0xff] - ^ Te3[t1 & 0xff] ^ rk[34]); - s3= (Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ Te2[(t1 >> 8) & 0xff] - ^ Te3[t2 & 0xff] ^ rk[35]); - - /* round 9: */ - t0= (Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >> 8) & 0xff] - ^ Te3[s3 & 0xff] ^ rk[36]); - t1= (Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >> 8) & 0xff] - ^ Te3[s0 & 0xff] ^ rk[37]); - t2= (Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >> 8) & 0xff] - ^ Te3[s1 & 0xff] ^ rk[38]); - t3= (Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >> 8) & 0xff] - ^ Te3[s2 & 0xff] ^ rk[39]); - - if (Nr > 10) - { - /* round 10: */ - s0= (Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ Te2[(t2 >> 8) & 0xff] - ^ Te3[t3 & 0xff] ^ rk[40]); - s1= (Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ Te2[(t3 >> 8) & 0xff] - ^ Te3[t0 & 0xff] ^ rk[41]); - s2= (Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ Te2[(t0 >> 8) & 0xff] - ^ Te3[t1 & 0xff] ^ rk[42]); - s3= (Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ Te2[(t1 >> 8) & 0xff] - ^ Te3[t2 & 0xff] ^ rk[43]); - - /* round 11: */ - t0= (Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >> 8) & 0xff] - ^ Te3[s3 & 0xff] ^ rk[44]); - t1= (Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >> 8) & 0xff] - ^ Te3[s0 & 0xff] ^ rk[45]); - t2= (Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >> 8) & 0xff] - ^ Te3[s1 & 0xff] ^ rk[46]); - t3= (Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >> 8) & 0xff] - ^ Te3[s2 & 0xff] ^ rk[47]); - - if (Nr > 12) - { - /* round 12: */ - s0= (Te0[t0 >> 24] ^ Te1[(t1 >> 16) & 0xff] ^ Te2[(t2 >> 8) & 0xff] - ^ Te3[t3 & 0xff] ^ rk[48]); - s1= (Te0[t1 >> 24] ^ Te1[(t2 >> 16) & 0xff] ^ Te2[(t3 >> 8) & 0xff] - ^ Te3[t0 & 0xff] ^ rk[49]); - s2= (Te0[t2 >> 24] ^ Te1[(t3 >> 16) & 0xff] ^ Te2[(t0 >> 8) & 0xff] - ^ Te3[t1 & 0xff] ^ rk[50]); - s3= (Te0[t3 >> 24] ^ Te1[(t0 >> 16) & 0xff] ^ Te2[(t1 >> 8) & 0xff] - ^ Te3[t2 & 0xff] ^ rk[51]); - - /* round 13: */ - t0= (Te0[s0 >> 24] ^ Te1[(s1 >> 16) & 0xff] ^ Te2[(s2 >> 8) & 0xff] - ^ Te3[s3 & 0xff] ^ rk[52]); - t1= (Te0[s1 >> 24] ^ Te1[(s2 >> 16) & 0xff] ^ Te2[(s3 >> 8) & 0xff] - ^ Te3[s0 & 0xff] ^ rk[53]); - t2= (Te0[s2 >> 24] ^ Te1[(s3 >> 16) & 0xff] ^ Te2[(s0 >> 8) & 0xff] - ^ Te3[s1 & 0xff] ^ rk[54]); - t3= (Te0[s3 >> 24] ^ Te1[(s0 >> 16) & 0xff] ^ Te2[(s1 >> 8) & 0xff] - ^ Te3[s2 & 0xff] ^ rk[55]); - } - } - rk += Nr << 2; -#else /* !FULL_UNROLL */ - - /* Nr - 1 full rounds: */ - - r = Nr >> 1; - for (;;) - { - t0= (Te0[(s0 >> 24) ] ^ - Te1[(s1 >> 16) & 0xff] ^ - Te2[(s2 >> 8) & 0xff] ^ - Te3[(s3 ) & 0xff] ^ - rk[4]); - - t1= (Te0[(s1 >> 24) ] ^ - Te1[(s2 >> 16) & 0xff] ^ - Te2[(s3 >> 8) & 0xff] ^ - Te3[(s0 ) & 0xff] ^ - rk[5]); - - t2= (Te0[(s2 >> 24) ] ^ - Te1[(s3 >> 16) & 0xff] ^ - Te2[(s0 >> 8) & 0xff] ^ - Te3[(s1 ) & 0xff] ^ - rk[6]); - - t3= (Te0[(s3 >> 24) ] ^ - Te1[(s0 >> 16) & 0xff] ^ - Te2[(s1 >> 8) & 0xff] ^ - Te3[(s2 ) & 0xff] ^ - rk[7]); - - rk+= 8; - if (--r == 0) - break; - - s0= (Te0[(t0 >> 24) ] ^ - Te1[(t1 >> 16) & 0xff] ^ - Te2[(t2 >> 8) & 0xff] ^ - Te3[(t3 ) & 0xff] ^ - rk[0]); - - s1= (Te0[(t1 >> 24) ] ^ - Te1[(t2 >> 16) & 0xff] ^ - Te2[(t3 >> 8) & 0xff] ^ - Te3[(t0 ) & 0xff] ^ - rk[1]); - - s2= (Te0[(t2 >> 24) ] ^ - Te1[(t3 >> 16) & 0xff] ^ - Te2[(t0 >> 8) & 0xff] ^ - Te3[(t1 ) & 0xff] ^ - rk[2]); - - s3= (Te0[(t3 >> 24) ] ^ - Te1[(t0 >> 16) & 0xff] ^ - Te2[(t1 >> 8) & 0xff] ^ - Te3[(t2 ) & 0xff] ^ - rk[3]); - } -#endif /* FULL_UNROLL */ - - /* Apply last round and map cipher state to byte array block: */ - s0= ((Te4[(t0 >> 24) ] & 0xff000000) ^ - (Te4[(t1 >> 16) & 0xff] & 0x00ff0000) ^ - (Te4[(t2 >> 8) & 0xff] & 0x0000ff00) ^ - (Te4[(t3 ) & 0xff] & 0x000000ff) ^ - rk[0]); - PUTuint32(ct , s0); - - s1= ((Te4[(t1 >> 24) ] & 0xff000000) ^ - (Te4[(t2 >> 16) & 0xff] & 0x00ff0000) ^ - (Te4[(t3 >> 8) & 0xff] & 0x0000ff00) ^ - (Te4[(t0 ) & 0xff] & 0x000000ff) ^ - rk[1]); - PUTuint32(ct + 4, s1); - - s2= ((Te4[(t2 >> 24) ] & 0xff000000) ^ - (Te4[(t3 >> 16) & 0xff] & 0x00ff0000) ^ - (Te4[(t0 >> 8) & 0xff] & 0x0000ff00) ^ - (Te4[(t1 ) & 0xff] & 0x000000ff) ^ - rk[2]); - PUTuint32(ct + 8, s2); - - s3= ((Te4[(t3 >> 24) ] & 0xff000000) ^ - (Te4[(t0 >> 16) & 0xff] & 0x00ff0000) ^ - (Te4[(t1 >> 8) & 0xff] & 0x0000ff00) ^ - (Te4[(t2 ) & 0xff] & 0x000000ff) ^ - rk[3]); - PUTuint32(ct + 12, s3); -} - - -void rijndaelDecrypt(const uint32 rk[/*4*(Nr + 1)*/], int Nr, - const uint8 ct[16], uint8 pt[16]) -{ - uint32 s0, s1, s2, s3, t0, t1, t2, t3; -#ifndef FULL_UNROLL - int r; -#endif /* FULL_UNROLL */ - - /* Map byte array block to cipher state and add initial round key: */ - - s0 = GETuint32(ct ) ^ rk[0]; - s1 = GETuint32(ct + 4) ^ rk[1]; - s2 = GETuint32(ct + 8) ^ rk[2]; - s3 = GETuint32(ct + 12) ^ rk[3]; - -#ifdef FULL_UNROLL - /* round 1: */ - t0= (Td0[s0 >> 24] ^ Td1[(s3 >> 16) & 0xff] ^ Td2[(s2 >> 8) & 0xff] - ^ Td3[s1 & 0xff] ^ rk[ 4]); - t1= (Td0[s1 >> 24] ^ Td1[(s0 >> 16) & 0xff] ^ Td2[(s3 >> 8) & 0xff] - ^ Td3[s2 & 0xff] ^ rk[ 5]); - t2= (Td0[s2 >> 24] ^ Td1[(s1 >> 16) & 0xff] ^ Td2[(s0 >> 8) & 0xff] - ^ Td3[s3 & 0xff] ^ rk[ 6]); - t3= (Td0[s3 >> 24] ^ Td1[(s2 >> 16) & 0xff] ^ Td2[(s1 >> 8) & 0xff] - ^ Td3[s0 & 0xff] ^ rk[ 7]); - - /* round 2: */ - s0= (Td0[t0 >> 24] ^ Td1[(t3 >> 16) & 0xff] ^ Td2[(t2 >> 8) & 0xff] - ^ Td3[t1 & 0xff] ^ rk[ 8]); - s1= (Td0[t1 >> 24] ^ Td1[(t0 >> 16) & 0xff] ^ Td2[(t3 >> 8) & 0xff] - ^ Td3[t2 & 0xff] ^ rk[ 9]); - s2= (Td0[t2 >> 24] ^ Td1[(t1 >> 16) & 0xff] ^ Td2[(t0 >> 8) & 0xff] - ^ Td3[t3 & 0xff] ^ rk[10]); - s3= (Td0[t3 >> 24] ^ Td1[(t2 >> 16) & 0xff] ^ Td2[(t1 >> 8) & 0xff] - ^ Td3[t0 & 0xff] ^ rk[11]); - - /* round 3: */ - t0= (Td0[s0 >> 24] ^ Td1[(s3 >> 16) & 0xff] ^ Td2[(s2 >> 8) & 0xff] - ^ Td3[s1 & 0xff] ^ rk[12]); - t1= (Td0[s1 >> 24] ^ Td1[(s0 >> 16) & 0xff] ^ Td2[(s3 >> 8) & 0xff] - ^ Td3[s2 & 0xff] ^ rk[13]); - t2= (Td0[s2 >> 24] ^ Td1[(s1 >> 16) & 0xff] ^ Td2[(s0 >> 8) & 0xff] - ^ Td3[s3 & 0xff] ^ rk[14]); - t3= (Td0[s3 >> 24] ^ Td1[(s2 >> 16) & 0xff] ^ Td2[(s1 >> 8) & 0xff] - ^ Td3[s0 & 0xff] ^ rk[15]); - - /* round 4: */ - s0= (Td0[t0 >> 24] ^ Td1[(t3 >> 16) & 0xff] ^ Td2[(t2 >> 8) & 0xff] - ^ Td3[t1 & 0xff] ^ rk[16]); - s1= (Td0[t1 >> 24] ^ Td1[(t0 >> 16) & 0xff] ^ Td2[(t3 >> 8) & 0xff] - ^ Td3[t2 & 0xff] ^ rk[17]); - s2= (Td0[t2 >> 24] ^ Td1[(t1 >> 16) & 0xff] ^ Td2[(t0 >> 8) & 0xff] - ^ Td3[t3 & 0xff] ^ rk[18]); - s3= (Td0[t3 >> 24] ^ Td1[(t2 >> 16) & 0xff] ^ Td2[(t1 >> 8) & 0xff] - ^ Td3[t0 & 0xff] ^ rk[19]); - - /* round 5: */ - t0= (Td0[s0 >> 24] ^ Td1[(s3 >> 16) & 0xff] ^ Td2[(s2 >> 8) & 0xff] - ^ Td3[s1 & 0xff] ^ rk[20]); - t1= (Td0[s1 >> 24] ^ Td1[(s0 >> 16) & 0xff] ^ Td2[(s3 >> 8) & 0xff] - ^ Td3[s2 & 0xff] ^ rk[21]); - t2= (Td0[s2 >> 24] ^ Td1[(s1 >> 16) & 0xff] ^ Td2[(s0 >> 8) & 0xff] - ^ Td3[s3 & 0xff] ^ rk[22]); - t3= (Td0[s3 >> 24] ^ Td1[(s2 >> 16) & 0xff] ^ Td2[(s1 >> 8) & 0xff] - ^ Td3[s0 & 0xff] ^ rk[23]); - - /* round 6: */ - s0= (Td0[t0 >> 24] ^ Td1[(t3 >> 16) & 0xff] ^ Td2[(t2 >> 8) & 0xff] - ^ Td3[t1 & 0xff] ^ rk[24]); - s1= (Td0[t1 >> 24] ^ Td1[(t0 >> 16) & 0xff] ^ Td2[(t3 >> 8) & 0xff] - ^ Td3[t2 & 0xff] ^ rk[25]); - s2= (Td0[t2 >> 24] ^ Td1[(t1 >> 16) & 0xff] ^ Td2[(t0 >> 8) & 0xff] - ^ Td3[t3 & 0xff] ^ rk[26]); - s3= (Td0[t3 >> 24] ^ Td1[(t2 >> 16) & 0xff] ^ Td2[(t1 >> 8) & 0xff] - ^ Td3[t0 & 0xff] ^ rk[27]); - - /* round 7: */ - t0= (Td0[s0 >> 24] ^ Td1[(s3 >> 16) & 0xff] ^ Td2[(s2 >> 8) & 0xff] - ^ Td3[s1 & 0xff] ^ rk[28]); - t1= (Td0[s1 >> 24] ^ Td1[(s0 >> 16) & 0xff] ^ Td2[(s3 >> 8) & 0xff] - ^ Td3[s2 & 0xff] ^ rk[29]); - t2= (Td0[s2 >> 24] ^ Td1[(s1 >> 16) & 0xff] ^ Td2[(s0 >> 8) & 0xff] - ^ Td3[s3 & 0xff] ^ rk[30]); - t3= (Td0[s3 >> 24] ^ Td1[(s2 >> 16) & 0xff] ^ Td2[(s1 >> 8) & 0xff] - ^ Td3[s0 & 0xff] ^ rk[31]); - - /* round 8: */ - s0= (Td0[t0 >> 24] ^ Td1[(t3 >> 16) & 0xff] ^ Td2[(t2 >> 8) & 0xff] - ^ Td3[t1 & 0xff] ^ rk[32]); - s1= (Td0[t1 >> 24] ^ Td1[(t0 >> 16) & 0xff] ^ Td2[(t3 >> 8) & 0xff] - ^ Td3[t2 & 0xff] ^ rk[33]); - s2= (Td0[t2 >> 24] ^ Td1[(t1 >> 16) & 0xff] ^ Td2[(t0 >> 8) & 0xff] - ^ Td3[t3 & 0xff] ^ rk[34]); - s3= (Td0[t3 >> 24] ^ Td1[(t2 >> 16) & 0xff] ^ Td2[(t1 >> 8) & 0xff] - ^ Td3[t0 & 0xff] ^ rk[35]); - - /* round 9: */ - t0= (Td0[s0 >> 24] ^ Td1[(s3 >> 16) & 0xff] ^ Td2[(s2 >> 8) & 0xff] - ^ Td3[s1 & 0xff] ^ rk[36]); - t1= (Td0[s1 >> 24] ^ Td1[(s0 >> 16) & 0xff] ^ Td2[(s3 >> 8) & 0xff] - ^ Td3[s2 & 0xff] ^ rk[37]); - t2= (Td0[s2 >> 24] ^ Td1[(s1 >> 16) & 0xff] ^ Td2[(s0 >> 8) & 0xff] - ^ Td3[s3 & 0xff] ^ rk[38]); - t3= (Td0[s3 >> 24] ^ Td1[(s2 >> 16) & 0xff] ^ Td2[(s1 >> 8) & 0xff] - ^ Td3[s0 & 0xff] ^ rk[39]); - - if (Nr > 10) - { - /* round 10: */ - s0= (Td0[t0 >> 24] ^ Td1[(t3 >> 16) & 0xff] ^ Td2[(t2 >> 8) & 0xff] - ^ Td3[t1 & 0xff] ^ rk[40]); - s1= (Td0[t1 >> 24] ^ Td1[(t0 >> 16) & 0xff] ^ Td2[(t3 >> 8) & 0xff] - ^ Td3[t2 & 0xff] ^ rk[41]); - s2= (Td0[t2 >> 24] ^ Td1[(t1 >> 16) & 0xff] ^ Td2[(t0 >> 8) & 0xff] - ^ Td3[t3 & 0xff] ^ rk[42]); - s3= (Td0[t3 >> 24] ^ Td1[(t2 >> 16) & 0xff] ^ Td2[(t1 >> 8) & 0xff] - ^ Td3[t0 & 0xff] ^ rk[43]); - - /* round 11: */ - t0= (Td0[s0 >> 24] ^ Td1[(s3 >> 16) & 0xff] ^ Td2[(s2 >> 8) & 0xff] - ^ Td3[s1 & 0xff] ^ rk[44]); - t1= (Td0[s1 >> 24] ^ Td1[(s0 >> 16) & 0xff] ^ Td2[(s3 >> 8) & 0xff] - ^ Td3[s2 & 0xff] ^ rk[45]); - t2= (Td0[s2 >> 24] ^ Td1[(s1 >> 16) & 0xff] ^ Td2[(s0 >> 8) & 0xff] - ^ Td3[s3 & 0xff] ^ rk[46]); - t3= (Td0[s3 >> 24] ^ Td1[(s2 >> 16) & 0xff] ^ Td2[(s1 >> 8) & 0xff] - ^ Td3[s0 & 0xff] ^ rk[47]); - - if (Nr > 12) - { - /* round 12: */ - s0= (Td0[t0 >> 24] ^ Td1[(t3 >> 16) & 0xff] ^ Td2[(t2 >> 8) & 0xff] - ^ Td3[t1 & 0xff] ^ rk[48]); - s1= (Td0[t1 >> 24] ^ Td1[(t0 >> 16) & 0xff] ^ Td2[(t3 >> 8) & 0xff] - ^ Td3[t2 & 0xff] ^ rk[49]); - s2= (Td0[t2 >> 24] ^ Td1[(t1 >> 16) & 0xff] ^ Td2[(t0 >> 8) & 0xff] - ^ Td3[t3 & 0xff] ^ rk[50]); - s3= (Td0[t3 >> 24] ^ Td1[(t2 >> 16) & 0xff] ^ Td2[(t1 >> 8) & 0xff] - ^ Td3[t0 & 0xff] ^ rk[51]); - - /* round 13: */ - t0= (Td0[s0 >> 24] ^ Td1[(s3 >> 16) & 0xff] ^ Td2[(s2 >> 8) & 0xff] - ^ Td3[s1 & 0xff] ^ rk[52]); - t1= (Td0[s1 >> 24] ^ Td1[(s0 >> 16) & 0xff] ^ Td2[(s3 >> 8) & 0xff] - ^ Td3[s2 & 0xff] ^ rk[53]); - t2= (Td0[s2 >> 24] ^ Td1[(s1 >> 16) & 0xff] ^ Td2[(s0 >> 8) & 0xff] - ^ Td3[s3 & 0xff] ^ rk[54]); - t3= (Td0[s3 >> 24] ^ Td1[(s2 >> 16) & 0xff] ^ Td2[(s1 >> 8) & 0xff] - ^ Td3[s0 & 0xff] ^ rk[55]); - } - } - rk += Nr << 2; -#else /* !FULL_UNROLL */ - - /* Nr - 1 full rounds: */ - r= (Nr >> 1); - for (;;) - { - t0= (Td0[(s0 >> 24) ] ^ - Td1[(s3 >> 16) & 0xff] ^ - Td2[(s2 >> 8) & 0xff] ^ - Td3[(s1 ) & 0xff] ^ - rk[4]); - - t1= (Td0[(s1 >> 24) ] ^ - Td1[(s0 >> 16) & 0xff] ^ - Td2[(s3 >> 8) & 0xff] ^ - Td3[(s2 ) & 0xff] ^ - rk[5]); - - t2= (Td0[(s2 >> 24) ] ^ - Td1[(s1 >> 16) & 0xff] ^ - Td2[(s0 >> 8) & 0xff] ^ - Td3[(s3 ) & 0xff] ^ - rk[6]); - - t3= (Td0[(s3 >> 24) ] ^ - Td1[(s2 >> 16) & 0xff] ^ - Td2[(s1 >> 8) & 0xff] ^ - Td3[(s0 ) & 0xff] ^ - rk[7]); - - rk+= 8; - if (--r == 0) - break; - - s0= (Td0[(t0 >> 24) ] ^ - Td1[(t3 >> 16) & 0xff] ^ - Td2[(t2 >> 8) & 0xff] ^ - Td3[(t1 ) & 0xff] ^ - rk[0]); - - s1= (Td0[(t1 >> 24) ] ^ - Td1[(t0 >> 16) & 0xff] ^ - Td2[(t3 >> 8) & 0xff] ^ - Td3[(t2 ) & 0xff] ^ - rk[1]); - - s2= (Td0[(t2 >> 24) ] ^ - Td1[(t1 >> 16) & 0xff] ^ - Td2[(t0 >> 8) & 0xff] ^ - Td3[(t3 ) & 0xff] ^ - rk[2]); - - s3= (Td0[(t3 >> 24) ] ^ - Td1[(t2 >> 16) & 0xff] ^ - Td2[(t1 >> 8) & 0xff] ^ - Td3[(t0 ) & 0xff] ^ - rk[3]); - } - -#endif /* FULL_UNROLL */ - - /* Apply last round and map cipher state to byte array block: */ - - s0= ((Td4[(t0 >> 24) ] & 0xff000000) ^ - (Td4[(t3 >> 16) & 0xff] & 0x00ff0000) ^ - (Td4[(t2 >> 8) & 0xff] & 0x0000ff00) ^ - (Td4[(t1 ) & 0xff] & 0x000000ff) ^ - rk[0]); - PUTuint32(pt , s0); - - s1= ((Td4[(t1 >> 24) ] & 0xff000000) ^ - (Td4[(t0 >> 16) & 0xff] & 0x00ff0000) ^ - (Td4[(t3 >> 8) & 0xff] & 0x0000ff00) ^ - (Td4[(t2 ) & 0xff] & 0x000000ff) ^ - rk[1]); - PUTuint32(pt + 4, s1); - - s2= ((Td4[(t2 >> 24) ] & 0xff000000) ^ - (Td4[(t1 >> 16) & 0xff] & 0x00ff0000) ^ - (Td4[(t0 >> 8) & 0xff] & 0x0000ff00) ^ - (Td4[(t3 ) & 0xff] & 0x000000ff) ^ - rk[2]); - PUTuint32(pt + 8, s2); - - s3= ((Td4[(t3 >> 24) ] & 0xff000000) ^ - (Td4[(t2 >> 16) & 0xff] & 0x00ff0000) ^ - (Td4[(t1 >> 8) & 0xff] & 0x0000ff00) ^ - (Td4[(t0 ) & 0xff] & 0x000000ff) ^ - rk[3]); - PUTuint32(pt + 12, s3); -} diff --git a/mysys/safemalloc.c b/mysys/safemalloc.c index 31f0333725f..08e43d0a80a 100644 --- a/mysys/safemalloc.c +++ b/mysys/safemalloc.c @@ -58,6 +58,8 @@ struct st_irem #ifdef HAVE_BACKTRACE void *frame[SF_REMEMBER_FRAMES]; /* call stack */ #endif + uint32 flags; /* Flags passed to malloc */ + my_thread_id thread_id; /* Which thread did the allocation */ uint32 marker; /* Underrun marker value */ }; @@ -79,11 +81,21 @@ static int bad_ptr(const char *where, void *ptr); static void free_memory(void *ptr); static void sf_terminate(); +/* Setup default call to get a thread id for the memory */ + +my_thread_id default_sf_malloc_dbug_id(void) +{ + return my_thread_dbug_id(); +} + +my_thread_id (*sf_malloc_dbug_id)(void)= default_sf_malloc_dbug_id; + + /** allocates memory */ -void *sf_malloc(size_t size) +void *sf_malloc(size_t size, myf my_flags) { struct st_irem *irem; uchar *data; @@ -96,8 +108,6 @@ void *sf_malloc(size_t size) if (!init_done) { pthread_mutex_init(&sf_mutex, NULL); - /* disable deadlock detector, because it calls my_malloc() */ - safe_mutex_setflags(&sf_mutex, MYF_NO_DEADLOCK_DETECTION); atexit(sf_terminate); init_done= 1; } @@ -114,7 +124,9 @@ void *sf_malloc(size_t size) data= (uchar*) (irem + 1); irem->datasize= size; irem->prev= 0; + irem->flags= my_flags; irem->marker= MAGICSTART; + irem->thread_id= sf_malloc_dbug_id(); data[size + 0]= MAGICEND0; data[size + 1]= MAGICEND1; data[size + 2]= MAGICEND2; @@ -154,17 +166,17 @@ void *sf_malloc(size_t size) return data; } -void *sf_realloc(void *ptr, size_t size) +void *sf_realloc(void *ptr, size_t size, myf my_flags) { char *data; if (!ptr) - return sf_malloc(size); + return sf_malloc(size, my_flags); if (bad_ptr("Reallocating", ptr)) return 0; - if ((data= sf_malloc(size))) + if ((data= sf_malloc(size, my_flags))) { struct st_irem *irem= (struct st_irem *)ptr - 1; set_if_smaller(size, irem->datasize); @@ -182,28 +194,25 @@ void sf_free(void *ptr) free_memory(ptr); } -static void free_memory(void *ptr) -{ - struct st_irem *irem= (struct st_irem *)ptr - 1; - - pthread_mutex_lock(&sf_mutex); - /* Remove this structure from the linked list */ - if (irem->prev) - irem->prev->next= irem->next; - else - sf_malloc_root= irem->next; +/** + Return size of memory block and if block is thread specific - if (irem->next) - irem->next->prev= irem->prev; + sf_malloc_usable_size() + @param ptr Pointer to malloced block + @param flags We will store 1 here if block is marked as MY_THREAD_SPECIFIC + otherwise 0 - /* Handle the statistics */ - sf_malloc_count--; - pthread_mutex_unlock(&sf_mutex); + @return Size of block +*/ - /* only trash the data and magic values, but keep the stack trace */ - TRASH_FREE((uchar*)(irem + 1) - 4, irem->datasize + 8); - free(irem); - return; +size_t sf_malloc_usable_size(void *ptr, my_bool *is_thread_specific) +{ + struct st_irem *irem= (struct st_irem *)ptr - 1; + DBUG_ENTER("sf_malloc_usable_size"); + *is_thread_specific= MY_TEST(irem->flags & MY_THREAD_SPECIFIC); + DBUG_PRINT("exit", ("size: %lu flags: %lu", (ulong) irem->datasize, + (ulong)irem->flags)); + DBUG_RETURN(irem->datasize); } #ifdef HAVE_BACKTRACE @@ -235,11 +244,46 @@ static void print_stack(void **frame) #define print_stack(X) fprintf(stderr, "???\n") #endif +static void free_memory(void *ptr) +{ + struct st_irem *irem= (struct st_irem *)ptr - 1; + + if ((irem->flags & MY_THREAD_SPECIFIC) && irem->thread_id && + irem->thread_id != sf_malloc_dbug_id()) + { + fprintf(stderr, "Warning: %4lu bytes freed by T@%lu, allocated by T@%lu at ", + (ulong) irem->datasize, + (ulong) sf_malloc_dbug_id(), (ulong) irem->thread_id); + print_stack(irem->frame); + } + + pthread_mutex_lock(&sf_mutex); + /* Remove this structure from the linked list */ + if (irem->prev) + irem->prev->next= irem->next; + else + sf_malloc_root= irem->next; + + if (irem->next) + irem->next->prev= irem->prev; + + /* Handle the statistics */ + sf_malloc_count--; + pthread_mutex_unlock(&sf_mutex); + + /* only trash the data and magic values, but keep the stack trace */ + TRASH_FREE((uchar*)(irem + 1) - 4, irem->datasize + 8); + free(irem); + return; +} + static void warn(const char *format,...) { va_list args; + DBUG_PRINT("error", ("%s", format)); va_start(args,format); vfprintf(stderr, format, args); + fflush(stderr); va_end(args); #ifdef HAVE_BACKTRACE @@ -312,9 +356,11 @@ static int sf_sanity() /** report on all the memory pieces that have not been free'd + + @param id Id of thread to report. 0 if all */ -static void sf_terminate() +void sf_report_leaked_memory(my_thread_id id) { size_t total= 0; struct st_irem *irem; @@ -322,21 +368,31 @@ static void sf_terminate() sf_sanity(); /* Report on all the memory that was allocated but not free'd */ - if (!sf_leaking_memory && sf_malloc_root) + + for (irem= sf_malloc_root; irem; irem= irem->next) { - for (irem= sf_malloc_root; irem; irem= irem->next) + if (!id || (irem->thread_id == id && irem->flags & MY_THREAD_SPECIFIC)) { - fprintf(stderr, "Warning: %4lu bytes lost, allocated at ", - (ulong) irem->datasize); + my_thread_id tid = irem->thread_id && irem->flags & MY_THREAD_SPECIFIC ? + irem->thread_id : 0; + fprintf(stderr, "Warning: %4lu bytes lost at %p, allocated by T@%lu at ", + (ulong) irem->datasize, (char*) (irem + 1), tid); print_stack(irem->frame); total+= irem->datasize; } + } + if (total) fprintf(stderr, "Memory lost: %lu bytes in %d chunks\n", (ulong) total, sf_malloc_count); - } + return; +} + +static void sf_terminate() +{ + if (!sf_leaking_memory) + sf_report_leaked_memory(0); pthread_mutex_destroy(&sf_mutex); - return; } #endif diff --git a/mysys/sha1.c b/mysys/sha1.c deleted file mode 100644 index a247d69f5b1..00000000000 --- a/mysys/sha1.c +++ /dev/null @@ -1,423 +0,0 @@ -/* Copyright (c) 2002, 2004, 2006 MySQL AB - Use is subject to license terms - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; version 2 of the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -*/ - -/* - Original Source from: http://www.faqs.org/rfcs/rfc3174.html - - Copyright (C) The Internet Society (2001). All Rights Reserved. - - This document and translations of it may be copied and furnished to - others, and derivative works that comment on or otherwise explain it - or assist in its implementation may be prepared, copied, published - and distributed, in whole or in part, without restriction of any - kind, provided that the above copyright notice and this paragraph are - included on all such copies and derivative works. However, this - document itself may not be modified in any way, such as by removing - the copyright notice or references to the Internet Society or other - Internet organizations, except as needed for the purpose of - developing Internet standards in which case the procedures for - copyrights defined in the Internet Standards process must be - followed, or as required to translate it into languages other than - English. - - The limited permissions granted above are perpetual and will not be - revoked by the Internet Society or its successors or assigns. - - This document and the information contained herein is provided on an - "AS IS" basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING - TASK FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING - BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION - HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF - MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. - - Acknowledgement - Funding for the RFC Editor function is currently provided by the - Internet Society. - - DESCRIPTION - This file implements the Secure Hashing Algorithm 1 as - defined in FIPS PUB 180-1 published April 17, 1995. - - The SHA-1, produces a 160-bit message digest for a given data - stream. It should take about 2**n steps to find a message with the - same digest as a given message and 2**(n/2) to find any two - messages with the same digest, when n is the digest size in bits. - Therefore, this algorithm can serve as a means of providing a - "fingerprint" for a message. - - PORTABILITY ISSUES - SHA-1 is defined in terms of 32-bit "words". This code uses - <stdint.h> (included via "sha1.h" to define 32 and 8 bit unsigned - integer types. If your C compiler does not support 32 bit unsigned - integers, this code is not appropriate. - - CAVEATS - SHA-1 is designed to work with messages less than 2^64 bits long. - Although SHA-1 allows a message digest to be generated for messages - of any number of bits less than 2^64, this implementation only - works with messages with a length that is a multiple of the size of - an 8-bit character. - - CHANGES - 2002 by Peter Zaitsev to - - fit to new prototypes according to MySQL standard - - Some optimizations - - All checking is now done in debug only mode - - More comments -*/ - -#include "my_global.h" -#include "m_string.h" -#include "sha1.h" - -/* - Define the SHA1 circular left shift macro -*/ - -#define SHA1CircularShift(bits,word) \ - (((word) << (bits)) | ((word) >> (32-(bits)))) - -/* Local Function Prototyptes */ -static void SHA1PadMessage(SHA1_CONTEXT*); -static void SHA1ProcessMessageBlock(SHA1_CONTEXT*); - - -/* - Initialize SHA1Context - - SYNOPSIS - mysql_sha1_reset() - context [in/out] The context to reset. - - DESCRIPTION - This function will initialize the SHA1Context in preparation - for computing a new SHA1 message digest. - - RETURN - SHA_SUCCESS ok - != SHA_SUCCESS sha Error Code. -*/ - - -const uint32 sha_const_key[5]= -{ - 0x67452301, - 0xEFCDAB89, - 0x98BADCFE, - 0x10325476, - 0xC3D2E1F0 -}; - - -int mysql_sha1_reset(SHA1_CONTEXT *context) -{ -#ifndef DBUG_OFF - if (!context) - return SHA_NULL; -#endif - - context->Length = 0; - context->Message_Block_Index = 0; - - context->Intermediate_Hash[0] = sha_const_key[0]; - context->Intermediate_Hash[1] = sha_const_key[1]; - context->Intermediate_Hash[2] = sha_const_key[2]; - context->Intermediate_Hash[3] = sha_const_key[3]; - context->Intermediate_Hash[4] = sha_const_key[4]; - - context->Computed = 0; - context->Corrupted = 0; - - return SHA_SUCCESS; -} - - -/* - Return the 160-bit message digest into the array provided by the caller - - SYNOPSIS - mysql_sha1_result() - context [in/out] The context to use to calculate the SHA-1 hash. - Message_Digest: [out] Where the digest is returned. - - DESCRIPTION - NOTE: The first octet of hash is stored in the 0th element, - the last octet of hash in the 19th element. - - RETURN - SHA_SUCCESS ok - != SHA_SUCCESS sha Error Code. -*/ - -int mysql_sha1_result(SHA1_CONTEXT *context, - uint8 Message_Digest[SHA1_HASH_SIZE]) -{ - int i; - -#ifndef DBUG_OFF - if (!context || !Message_Digest) - return SHA_NULL; - - if (context->Corrupted) - return context->Corrupted; -#endif - - if (!context->Computed) - { - SHA1PadMessage(context); - /* message may be sensitive, clear it out */ - bzero((char*) context->Message_Block,64); - context->Length = 0; /* and clear length */ - context->Computed = 1; - } - - for (i = 0; i < SHA1_HASH_SIZE; i++) - Message_Digest[i] = (int8)((context->Intermediate_Hash[i>>2] >> 8 - * ( 3 - ( i & 0x03 ) ))); - return SHA_SUCCESS; -} - - -/* - Accepts an array of octets as the next portion of the message. - - SYNOPSIS - mysql_sha1_input() - context [in/out] The SHA context to update - message_array An array of characters representing the next portion - of the message. - length The length of the message in message_array - - RETURN - SHA_SUCCESS ok - != SHA_SUCCESS sha Error Code. -*/ - -int mysql_sha1_input(SHA1_CONTEXT *context, const uint8 *message_array, - unsigned length) -{ - if (!length) - return SHA_SUCCESS; - -#ifndef DBUG_OFF - /* We assume client konows what it is doing in non-debug mode */ - if (!context || !message_array) - return SHA_NULL; - if (context->Computed) - return (context->Corrupted= SHA_STATE_ERROR); - if (context->Corrupted) - return context->Corrupted; -#endif - - while (length--) - { - context->Message_Block[context->Message_Block_Index++]= - (*message_array & 0xFF); - context->Length += 8; /* Length is in bits */ - -#ifndef DBUG_OFF - /* - Then we're not debugging we assume we never will get message longer - 2^64 bits. - */ - if (context->Length == 0) - return (context->Corrupted= 1); /* Message is too long */ -#endif - - if (context->Message_Block_Index == 64) - { - SHA1ProcessMessageBlock(context); - } - message_array++; - } - return SHA_SUCCESS; -} - - -/* - Process the next 512 bits of the message stored in the Message_Block array. - - SYNOPSIS - SHA1ProcessMessageBlock() - - DESCRIPTION - Many of the variable names in this code, especially the single - character names, were used because those were the names used in - the publication. -*/ - -/* Constants defined in SHA-1 */ -static const uint32 K[]= -{ - 0x5A827999, - 0x6ED9EBA1, - 0x8F1BBCDC, - 0xCA62C1D6 -}; - - -static void SHA1ProcessMessageBlock(SHA1_CONTEXT *context) -{ - int t; /* Loop counter */ - uint32 temp; /* Temporary word value */ - uint32 W[80]; /* Word sequence */ - uint32 A, B, C, D, E; /* Word buffers */ - int idx; - - /* - Initialize the first 16 words in the array W - */ - - for (t = 0; t < 16; t++) - { - idx=t*4; - W[t] = context->Message_Block[idx] << 24; - W[t] |= context->Message_Block[idx + 1] << 16; - W[t] |= context->Message_Block[idx + 2] << 8; - W[t] |= context->Message_Block[idx + 3]; - } - - - for (t = 16; t < 80; t++) - { - W[t] = SHA1CircularShift(1,W[t-3] ^ W[t-8] ^ W[t-14] ^ W[t-16]); - } - - A = context->Intermediate_Hash[0]; - B = context->Intermediate_Hash[1]; - C = context->Intermediate_Hash[2]; - D = context->Intermediate_Hash[3]; - E = context->Intermediate_Hash[4]; - - for (t = 0; t < 20; t++) - { - temp= SHA1CircularShift(5,A) + ((B & C) | ((~B) & D)) + E + W[t] + K[0]; - E = D; - D = C; - C = SHA1CircularShift(30,B); - B = A; - A = temp; - } - - for (t = 20; t < 40; t++) - { - temp = SHA1CircularShift(5,A) + (B ^ C ^ D) + E + W[t] + K[1]; - E = D; - D = C; - C = SHA1CircularShift(30,B); - B = A; - A = temp; - } - - for (t = 40; t < 60; t++) - { - temp= (SHA1CircularShift(5,A) + ((B & C) | (B & D) | (C & D)) + E + W[t] + - K[2]); - E = D; - D = C; - C = SHA1CircularShift(30,B); - B = A; - A = temp; - } - - for (t = 60; t < 80; t++) - { - temp = SHA1CircularShift(5,A) + (B ^ C ^ D) + E + W[t] + K[3]; - E = D; - D = C; - C = SHA1CircularShift(30,B); - B = A; - A = temp; - } - - context->Intermediate_Hash[0] += A; - context->Intermediate_Hash[1] += B; - context->Intermediate_Hash[2] += C; - context->Intermediate_Hash[3] += D; - context->Intermediate_Hash[4] += E; - - context->Message_Block_Index = 0; -} - - -/* - Pad message - - SYNOPSIS - SHA1PadMessage() - context: [in/out] The context to pad - - DESCRIPTION - According to the standard, the message must be padded to an even - 512 bits. The first padding bit must be a '1'. The last 64 bits - represent the length of the original message. All bits in between - should be 0. This function will pad the message according to - those rules by filling the Message_Block array accordingly. It - will also call the ProcessMessageBlock function provided - appropriately. When it returns, it can be assumed that the message - digest has been computed. - -*/ - -static void SHA1PadMessage(SHA1_CONTEXT *context) -{ - /* - Check to see if the current message block is too small to hold - the initial padding bits and length. If so, we will pad the - block, process it, and then continue padding into a second - block. - */ - - int i=context->Message_Block_Index; - - if (i > 55) - { - context->Message_Block[i++] = 0x80; - bzero((char*) &context->Message_Block[i], - sizeof(context->Message_Block[0])*(64-i)); - context->Message_Block_Index=64; - - /* This function sets context->Message_Block_Index to zero */ - SHA1ProcessMessageBlock(context); - - bzero((char*) &context->Message_Block[0], - sizeof(context->Message_Block[0])*56); - context->Message_Block_Index=56; - } - else - { - context->Message_Block[i++] = 0x80; - bzero((char*) &context->Message_Block[i], - sizeof(context->Message_Block[0])*(56-i)); - context->Message_Block_Index=56; - } - - /* - Store the message length as the last 8 octets - */ - - context->Message_Block[56] = (int8) (context->Length >> 56); - context->Message_Block[57] = (int8) (context->Length >> 48); - context->Message_Block[58] = (int8) (context->Length >> 40); - context->Message_Block[59] = (int8) (context->Length >> 32); - context->Message_Block[60] = (int8) (context->Length >> 24); - context->Message_Block[61] = (int8) (context->Length >> 16); - context->Message_Block[62] = (int8) (context->Length >> 8); - context->Message_Block[63] = (int8) (context->Length); - - SHA1ProcessMessageBlock(context); -} diff --git a/mysys/stacktrace.c b/mysys/stacktrace.c index ae715c04621..0d4d5fce803 100644 --- a/mysys/stacktrace.c +++ b/mysys/stacktrace.c @@ -95,7 +95,7 @@ static int safe_print_str(const char *addr, int max_len) /* Read up to the maximum number of bytes. */ while (total) { - count= min(sizeof(buf), total); + count= MY_MIN(sizeof(buf), total); if ((nbytes= pread(fd, buf, count, offset)) < 0) { @@ -129,13 +129,32 @@ static int safe_print_str(const char *addr, int max_len) #endif -void my_safe_print_str(const char* val, int max_len) +/* + Attempt to print a char * pointer as a string. + + SYNOPSIS + Prints either until the end of string ('\0'), or max_len characters have + been printed. + + RETURN VALUE + 0 Pointer was within the heap address space. + The string was printed fully, or until the end of the heap address space. + 1 Pointer is outside the heap address space. Printed as invalid. + + NOTE + On some systems, we can have valid pointers outside the heap address space. + This is through the use of mmap inside malloc calls. When this function + returns 1, it does not mean 100% that the pointer is corrupted. +*/ + +int my_safe_print_str(const char* val, int max_len) { char *heap_end; #ifdef __linux__ + // Try and make use of /proc filesystem to safely print memory contents. if (!safe_print_str(val, max_len)) - return; + return 0; #endif heap_end= (char*) sbrk(0); @@ -143,12 +162,14 @@ void my_safe_print_str(const char* val, int max_len) if (!PTR_SANE(val)) { my_safe_printf_stderr("%s", "is an invalid pointer"); - return; + return 1; } for (; max_len && PTR_SANE(val) && *val; --max_len) my_write_stderr((val++), 1); my_safe_printf_stderr("%s", "\n"); + + return 0; } #if defined(HAVE_PRINTSTACK) @@ -315,12 +336,11 @@ inline uint32* find_prev_pc(uint32* pc, uchar** fp) void my_print_stacktrace(uchar* stack_bottom, ulong thread_stack) { - uchar** fp; + uchar** UNINIT_VAR(fp); uint frame_count = 0, sigreturn_frame_count; #if defined(__alpha__) && defined(__GNUC__) uint32* pc; #endif - LINT_INIT(fp); #ifdef __i386__ @@ -348,7 +368,7 @@ void my_print_stacktrace(uchar* stack_bottom, ulong thread_stack) if (!stack_bottom || (uchar*) stack_bottom > (uchar*) &fp) { - ulong tmp= min(0x10000,thread_stack); + ulong tmp= MY_MIN(0x10000,thread_stack); /* Assume that the stack starts at the previous even 65K */ stack_bottom= (uchar*) (((ulong) &fp + tmp) & ~(ulong) 0xFFFF); my_safe_printf_stderr("Cannot determine thread, fp=%p, " @@ -728,7 +748,7 @@ void my_write_core(int unused) } -void my_safe_print_str(const char *val, int len) +int my_safe_print_str(const char *val, int len) { __try { @@ -738,6 +758,7 @@ void my_safe_print_str(const char *val, int len) { my_safe_printf_stderr("%s", "is an invalid string pointer"); } + return 0; } #endif /*__WIN__*/ diff --git a/mysys/string.c b/mysys/string.c index a63b1f502e5..a0fa3a02e17 100644 --- a/mysys/string.c +++ b/mysys/string.c @@ -174,10 +174,41 @@ my_bool dynstr_append_os_quoted(DYNAMIC_STRING *str, const char *append, ...) return ret; } +my_bool dynstr_append_quoted(DYNAMIC_STRING *str, + const char *append, size_t len, + char quote) +{ + uint additional= (str->alloc_increment ? str->alloc_increment : 10); + uint lim= additional; + uint i; + if (dynstr_realloc(str, len + additional + 2)) + return TRUE; + str->str[str->length++]= quote; + for (i= 0; i < len; i++) + { + register char c= append[i]; + if (c == quote || c == '\\') + { + if (!lim) + { + if (dynstr_realloc(str, additional)) + return TRUE; + lim= additional; + } + lim--; + str->str[str->length++]= '\\'; + } + str->str[str->length++]= c; + } + str->str[str->length++]= quote; + return FALSE; +} + void dynstr_free(DYNAMIC_STRING *str) { - my_free(str->str); + if (str->str) /* Safety to allow double free */ + my_free(str->str); str->str= NULL; } diff --git a/mysys/testhash.c b/mysys/testhash.c index aa78a2d8c40..96b106bec14 100644 --- a/mysys/testhash.c +++ b/mysys/testhash.c @@ -79,7 +79,7 @@ static int do_test() for (i=0 ; i < recant ; i++) { - n1=rnd(1000); n2=rnd(100); n3=rnd(min(recant*5,MAX_RECORDS)); + n1=rnd(1000); n2=rnd(100); n3=rnd(MY_MIN(recant*5,MAX_RECORDS)); record= (char*) my_malloc(reclength,MYF(MY_FAE)); sprintf(record,"%6d:%4d:%8d:Pos: %4d ",n1,n2,n3,write_count); if (my_hash_insert(&hash,record)) @@ -133,7 +133,7 @@ static int do_test() printf("- Update\n"); for (i=0 ; i < write_count/10 ; i++) { - n1=rnd(1000); n2=rnd(100); n3=rnd(min(recant*2,MAX_RECORDS)); + n1=rnd(1000); n2=rnd(100); n3=rnd(MY_MIN(recant*2,MAX_RECORDS)); for (j=rnd(1000) ; j>0 && key1[j] == 0 ; j--) ; if (j) { diff --git a/mysys/thr_alarm.c b/mysys/thr_alarm.c index d0bb2f1ef4c..61ef3657161 100644 --- a/mysys/thr_alarm.c +++ b/mysys/thr_alarm.c @@ -273,7 +273,7 @@ void thr_end_alarm(thr_alarm_t *alarmed) /* Come here when some alarm in queue is due. Mark all alarms with are finnished in list. - Shedule alarms to be sent again after 1-10 sec (many alarms at once) + Schedule alarms to be sent again after 1-10 sec (many alarms at once) If alarm_aborted is set then all alarms are given and resent every second. */ @@ -425,7 +425,7 @@ void end_thr_alarm(my_bool free_structures) if (alarm_aborted != 1) /* If memory not freed */ { mysql_mutex_lock(&LOCK_alarm); - DBUG_PRINT("info",("Resheduling %d waiting alarms",alarm_queue.elements)); + DBUG_PRINT("info",("Rescheduling %d waiting alarms",alarm_queue.elements)); alarm_aborted= -1; /* mark aborted */ if (alarm_queue.elements || (alarm_thread_running && free_structures)) { @@ -694,7 +694,7 @@ static void *test_thread(void *arg) thread_count--; mysql_cond_signal(&COND_thread_count); /* Tell main we are ready */ mysql_mutex_unlock(&LOCK_thread_count); - free((uchar*) arg); + my_thread_end(); return 0; } @@ -771,7 +771,7 @@ int main(int argc __attribute__((unused)),char **argv __attribute__((unused))) { pthread_t tid; pthread_attr_t thr_attr; - int i,*param,error; + int i, param[2], error; sigset_t set; ALARM_INFO alarm_info; MY_INIT(argv[0]); @@ -815,12 +815,11 @@ int main(int argc __attribute__((unused)),char **argv __attribute__((unused))) printf("Main thread: %s\n",my_thread_name()); for (i=0 ; i < 2 ; i++) { - param=(int*) malloc(sizeof(int)); - *param= i; + param[i]= i; mysql_mutex_lock(&LOCK_thread_count); if ((error= mysql_thread_create(0, &tid, &thr_attr, test_thread, - (void*) param))) + (void*) ¶m[i]))) { printf("Can't create thread %d, error: %d\n",i,error); exit(1); @@ -851,6 +850,9 @@ int main(int argc __attribute__((unused)),char **argv __attribute__((unused))) alarm_info.active_alarms, alarm_info.max_used_alarms, alarm_info.next_alarm_time); printf("Test succeeded\n"); + mysql_cond_destroy(&COND_thread_count); + mysql_mutex_destroy(&LOCK_thread_count); + my_end(MY_CHECK_ERROR); return 0; } diff --git a/mysys/thr_lock.c b/mysys/thr_lock.c index 890140fc34f..5c3437382a5 100644 --- a/mysys/thr_lock.c +++ b/mysys/thr_lock.c @@ -1,6 +1,5 @@ -/* - Copyright (c) 2000, 2011, Oracle and/or its affiliates - Copyright (c) 2012, Monty Program Ab +/* Copyright (c) 2000, 2011, Oracle and/or its affiliates. + Copyright (c) 2012, Monty Program Ab. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -88,6 +87,7 @@ be any number of TL_WRITE_CONCURRENT_INSERT locks aktive at the same time. #include "mysys_priv.h" #include "thr_lock.h" +#include "mysql/psi/mysql_table.h" #include <m_string.h> #include <errno.h> @@ -95,6 +95,24 @@ my_bool thr_lock_inited=0; ulong locks_immediate = 0L, locks_waited = 0L; enum thr_lock_type thr_upgraded_concurrent_insert_lock = TL_WRITE; +#ifdef WITH_WSREP +static wsrep_thd_is_brute_force_fun wsrep_thd_is_brute_force= NULL; +static wsrep_abort_thd_fun wsrep_abort_thd= NULL; +static my_bool wsrep_debug; +static my_bool wsrep_convert_LOCK_to_trx; +static wsrep_on_fun wsrep_on = NULL; + +void wsrep_thr_lock_init( + wsrep_thd_is_brute_force_fun bf_fun, wsrep_abort_thd_fun abort_fun, + my_bool debug, my_bool convert_LOCK_to_trx, wsrep_on_fun on_fun +) { + wsrep_thd_is_brute_force = bf_fun; + wsrep_abort_thd = abort_fun; + wsrep_debug = debug; + wsrep_convert_LOCK_to_trx= convert_LOCK_to_trx; + wsrep_on = on_fun; +} +#endif /* The following constants are only for debug output */ #define MAX_THREADS 1000 #define MAX_LOCKS 1000 @@ -327,15 +345,14 @@ static void check_locks(THR_LOCK *lock, const char *where, "Warning at '%s': Write lock %d waiting while no exclusive read locks\n",where,(int) lock->write_wait.data->type); DBUG_PRINT("warning", ("Warning at '%s': Write lock %d waiting while no exclusive read locks",where,(int) lock->write_wait.data->type)); } - } + } } else { /* We have at least one write lock */ if (lock->write.data->type == TL_WRITE_CONCURRENT_INSERT) { - THR_LOCK_DATA *data; - uint count= 0; + count= 0; for (data=lock->write.data->next; data && count < MAX_LOCKS; data=data->next) @@ -368,7 +385,6 @@ static void check_locks(THR_LOCK *lock, const char *where, } if (lock->read.data) { - THR_LOCK_DATA *data; for (data=lock->read.data ; data ; data=data->next) { if (!thr_lock_owner_equal(lock->write.data->owner, @@ -481,17 +497,6 @@ has_old_lock(THR_LOCK_DATA *data, THR_LOCK_INFO *owner) return 0; } -static inline my_bool have_specific_lock(THR_LOCK_DATA *data, - enum thr_lock_type type) -{ - for ( ; data ; data=data->next) - { - if (data->type == type) - return 1; - } - return 0; -} - static void wake_up_waiters(THR_LOCK *lock); @@ -504,7 +509,7 @@ wait_for_lock(struct st_lock_list *wait, THR_LOCK_DATA *data, mysql_cond_t *cond= &thread_var->suspend; struct timespec wait_timeout; enum enum_thr_lock_result result= THR_LOCK_ABORTED; - const char *old_proc_info; + PSI_stage_info old_stage; my_bool use_wait_callbacks= FALSE; DBUG_ENTER("wait_for_lock"); @@ -512,8 +517,8 @@ wait_for_lock(struct st_lock_list *wait, THR_LOCK_DATA *data, One can use this to signal when a thread is going to wait for a lock. See debug_sync.cc. - Beware of waiting for a signal here. The lock has aquired its mutex. - While waiting on a signal here, the locking thread could not aquire + Beware of waiting for a signal here. The lock has acquired its mutex. + While waiting on a signal here, the locking thread could not acquire the mutex to release the lock. One could lock up the table completely. @@ -544,7 +549,8 @@ wait_for_lock(struct st_lock_list *wait, THR_LOCK_DATA *data, thread_var->current_cond= cond; data->cond= cond; - old_proc_info= proc_info_hook(NULL, "Waiting for table level lock", + proc_info_hook(NULL, &stage_waiting_for_table_level_lock, + &old_stage, __func__, __FILE__, __LINE__); /* @@ -640,11 +646,97 @@ wait_for_lock(struct st_lock_list *wait, THR_LOCK_DATA *data, thread_var->current_cond= 0; mysql_mutex_unlock(&thread_var->mutex); - proc_info_hook(NULL, old_proc_info, __func__, __FILE__, __LINE__); + proc_info_hook(NULL, &old_stage, NULL, __func__, __FILE__, __LINE__); DBUG_RETURN(result); } +#ifdef WITH_WSREP +/* + * If brute force applier would need to wait for a thr lock, + * it needs to make sure that it will get the lock without (too much) + * delay. + * We identify here the owners of blocking locks and ask them to + * abort. We then put our lock request in the first place in the + * wait queue. When lock holders abort (one by one) the lock release + * algorithm should grant the lock to us. We rely on this and proceed + * to wait_for_locks(). + * wsrep_break_locks() should be called in all the cases, where lock + * wait would happen. + * + * TODO: current implementation might not cover all possible lock wait + * situations. This needs an review still. + * TODO: lock release, might favor some other lock (instead our bf). + * This needs an condition to check for bf locks first. + * TODO: we still have a debug fprintf, this should be removed + */ +static my_bool +wsrep_break_lock( + THR_LOCK_DATA *data, struct st_lock_list *lock_queue1, + struct st_lock_list *wait_queue) +{ + if (wsrep_on && wsrep_on(data->owner->mysql_thd) && + wsrep_thd_is_brute_force && + wsrep_thd_is_brute_force(data->owner->mysql_thd, TRUE)) + { + THR_LOCK_DATA *holder; + + /* if locking session conversion to transaction has been enabled, + we know that this conflicting lock must be read lock and furthermore, + lock holder is read-only. It is safe to wait for him. + */ +#ifdef TODO_WHEN_LOCK_TABLES_IS_A_TRANSACTION + if (wsrep_convert_LOCK_to_trx && + (THD*)(data->owner->mysql_thd)->in_lock_tables) + { + if (wsrep_debug) + fprintf(stderr,"WSREP wsrep_break_lock read lock untouched\n"); + return FALSE; + } +#endif + if (wsrep_debug) + fprintf(stderr,"WSREP wsrep_break_lock aborting locks\n"); + + /* aborting lock holder(s) here */ + for (holder=(lock_queue1) ? lock_queue1->data : NULL; + holder; + holder=holder->next) + { + if (!wsrep_thd_is_brute_force(holder->owner->mysql_thd, TRUE)) + { + wsrep_abort_thd(data->owner->mysql_thd, + holder->owner->mysql_thd, FALSE); + } + else + { + if (wsrep_debug) + fprintf(stderr,"WSREP wsrep_break_lock skipping BF lock conflict\n"); + return FALSE; + } + } + + /* Add our lock to the head of the wait queue */ + if (*(wait_queue->last)==wait_queue->data) + { + wait_queue->last=&data->next; + assert(wait_queue->data==0); + } + else + { + assert(wait_queue->data!=0); + wait_queue->data->prev=&data->next; + } + data->next=wait_queue->data; + data->prev=&wait_queue->data; + wait_queue->data=data; + data->cond=get_cond(); + + statistic_increment(locks_immediate,&THR_LOCK_lock); + return TRUE; + } + return FALSE; +} +#endif static enum enum_thr_lock_result thr_lock(THR_LOCK_DATA *data, THR_LOCK_INFO *owner, ulong lock_wait_timeout) @@ -653,12 +745,20 @@ thr_lock(THR_LOCK_DATA *data, THR_LOCK_INFO *owner, ulong lock_wait_timeout) enum enum_thr_lock_result result= THR_LOCK_SUCCESS; struct st_lock_list *wait_queue; enum thr_lock_type lock_type= data->type; +#ifdef WITH_WSREP + my_bool wsrep_lock_inserted= FALSE; +#endif + MYSQL_TABLE_WAIT_VARIABLES(locker, state) /* no ';' */ DBUG_ENTER("thr_lock"); data->next=0; data->cond=0; /* safety */ data->owner= owner; /* Must be reset ! */ data->priority&= ~THR_LOCK_LATE_PRIV; + + MYSQL_START_TABLE_LOCK_WAIT(locker, &state, data->m_psi, + PSI_TABLE_LOCK, lock_type); + mysql_mutex_lock(&lock->mutex); DBUG_PRINT("lock",("data: 0x%lx thread: 0x%lx lock: 0x%lx type: %d", (long) data, data->owner->thread_id, @@ -688,8 +788,8 @@ thr_lock(THR_LOCK_DATA *data, THR_LOCK_INFO *owner, ulong lock_wait_timeout) \ = READ - + = Request can be satisified. - - = Request cannot be satisified. + + = Request can be satisfied. + - = Request cannot be satisfied. READ_NO_INSERT and WRITE_ALLOW_WRITE should in principle be incompatible. However this will cause starvation of @@ -744,6 +844,13 @@ thr_lock(THR_LOCK_DATA *data, THR_LOCK_INFO *owner, ulong lock_wait_timeout) lock but a high priority write waiting in the write_wait queue. In the latter case we should yield the lock to the writer. */ +#ifdef WITH_WSREP + if (wsrep_break_lock(data, &lock->write, &lock->read_wait)) + { + wsrep_lock_inserted= TRUE; + } +#endif + wait_queue= &lock->read_wait; } else /* Request for WRITE lock */ @@ -828,7 +935,8 @@ thr_lock(THR_LOCK_DATA *data, THR_LOCK_INFO *owner, ulong lock_wait_timeout) if (((lock_type == TL_WRITE_ALLOW_WRITE || (lock_type == TL_WRITE_CONCURRENT_INSERT && - lock->allow_multiple_concurrent_insert)) && + lock->allow_multiple_concurrent_insert && + !lock->read_no_write_count)) && ! lock->write_wait.data && lock->write.data->type == lock_type && ! lock->read_no_write_count) || @@ -887,12 +995,26 @@ thr_lock(THR_LOCK_DATA *data, THR_LOCK_INFO *owner, ulong lock_wait_timeout) DBUG_PRINT("lock",("write locked 3 by thread: 0x%lx type: %d", lock->read.data->owner->thread_id, data->type)); } +#ifdef WITH_WSREP + if (wsrep_break_lock(data, &lock->write, &lock->write_wait)) + { + wsrep_lock_inserted= TRUE; + } +#endif + wait_queue= &lock->write_wait; } /* Can't get lock yet; Wait for it */ - DBUG_RETURN(wait_for_lock(wait_queue, data, 0, lock_wait_timeout)); +#ifdef WITH_WSREP + if (wsrep_lock_inserted && wsrep_on(data->owner->mysql_thd)) + DBUG_RETURN(wait_for_lock(wait_queue, data, 1, lock_wait_timeout)); +#endif + result= wait_for_lock(wait_queue, data, 0, lock_wait_timeout); + MYSQL_END_TABLE_LOCK_WAIT(locker); + DBUG_RETURN(result); end: mysql_mutex_unlock(&lock->mutex); + MYSQL_END_TABLE_LOCK_WAIT(locker); DBUG_RETURN(result); } @@ -1124,7 +1246,7 @@ end: /* Get all locks in a specific order to avoid dead-locks - Sort acording to lock position and put write_locks before read_locks if + Sort according to lock position and put write_locks before read_locks if lock on same lock. Locks on MERGE tables has lower priority than other locks of the same type. See comment for lock_priority. */ @@ -1716,7 +1838,7 @@ static void *test_thread(void *arg) thread_count--; mysql_cond_signal(&COND_thread_count); /* Tell main we are ready */ mysql_mutex_unlock(&LOCK_thread_count); - free((uchar*) arg); + my_thread_end(); return 0; } @@ -1725,7 +1847,7 @@ int main(int argc __attribute__((unused)),char **argv __attribute__((unused))) { pthread_t tid; pthread_attr_t thr_attr; - int *param,error; + int param[array_elements(lock_counts)], error; uint i; MY_INIT(argv[0]); if (argc > 1 && argv[1][0] == '-' && argv[1][1] == '#') @@ -1781,8 +1903,7 @@ int main(int argc __attribute__((unused)),char **argv __attribute__((unused))) #endif for (i=0 ; i < array_elements(lock_counts) ; i++) { - param=(int*) malloc(sizeof(int)); - *param=i; + param[i]= i; if ((error= mysql_mutex_lock(&LOCK_thread_count))) { @@ -1792,7 +1913,7 @@ int main(int argc __attribute__((unused)),char **argv __attribute__((unused))) } if ((error= mysql_thread_create(0, &tid, &thr_attr, test_thread, - (void*) param))) + (void*) ¶m[i]))) { fprintf(stderr, "Got error: %d from mysql_thread_create (errno: %d)\n", error, errno); @@ -1821,6 +1942,9 @@ int main(int argc __attribute__((unused)),char **argv __attribute__((unused))) else #endif printf("Test succeeded\n"); + mysql_cond_destroy(&COND_thread_count); + mysql_mutex_destroy(&LOCK_thread_count); + my_end(MY_CHECK_ERROR); return 0; } diff --git a/mysys/thr_mutex.c b/mysys/thr_mutex.c index 17cda782b30..18af5f47b10 100644 --- a/mysys/thr_mutex.c +++ b/mysys/thr_mutex.c @@ -132,7 +132,7 @@ void safe_mutex_global_init(void) #ifdef SAFE_MUTEX_DETECT_DESTROY safe_mutex_create_root= 0; -#endif +#endif /* SAFE_MUTEX_DETECT_DESTROY */ } static inline void remove_from_active_list(safe_mutex_t *mp) @@ -174,12 +174,12 @@ static int safe_mutex_lazy_init_deadlock_detection(safe_mutex_t *mp) 128, offsetof(safe_mutex_deadlock_t, id), sizeof(mp->id), - 0, 0, HASH_UNIQUE); + 0, 0, 0, HASH_UNIQUE); my_hash_init2(mp->used_mutex, 64, &my_charset_bin, 128, offsetof(safe_mutex_t, id), sizeof(mp->id), - 0, 0, HASH_UNIQUE); + 0, 0, 0, HASH_UNIQUE); return 0; } @@ -553,7 +553,7 @@ int safe_cond_timedwait(pthread_cond_t *cond, safe_mutex_t *mp, "on %s at %s, line %d\n", error, errno, mp->name, file, line); } -#endif +#endif /* EXTRA_DEBUG */ pthread_mutex_lock(&mp->global); /* Restore state as it was before */ mp->thread= save_state.thread; @@ -612,7 +612,7 @@ int safe_mutex_destroy(safe_mutex_t *mp, const char *file, uint line) error=1; if (pthread_mutex_destroy(&mp->mutex)) error=1; -#endif +#endif /* __WIN__ */ mp->file= 0; /* Mark destroyed */ #ifdef SAFE_MUTEX_DETECT_DESTROY @@ -838,7 +838,7 @@ static void print_deadlock_warning(safe_mutex_t *new_mutex, DBUG_VOID_RETURN; } -#elif defined(MY_PTHREAD_FASTMUTEX) +#elif defined(MY_PTHREAD_FASTMUTEX) /* !SAFE_MUTEX_DEFINED */ static ulong mutex_delay(ulong delayloops) { @@ -922,4 +922,4 @@ void fastmutex_global_init(void) #endif } -#endif /* defined(MY_PTHREAD_FASTMUTEX) */ +#endif /* defined(MY_PTHREAD_FASTMUTEX) && defined(SAFE_MUTEX_DEFINED) */ diff --git a/mysys/thr_timer.c b/mysys/thr_timer.c new file mode 100644 index 00000000000..b8726617f44 --- /dev/null +++ b/mysys/thr_timer.c @@ -0,0 +1,570 @@ +/* + Copyright (c) 2012 Monty Program Ab + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 or later of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +/* + Implementation if OS independent timers. + This is done based on pthread primitives, especially pthread_cond_timedwait() +*/ + +#include "mysys_priv.h" +#include "thr_timer.h" +#include <m_string.h> +#include <queues.h> +#ifdef HAVE_TIMER_CREATE +#include <sys/syscall.h> +#endif + +struct timespec next_timer_expire_time; + +static my_bool thr_timer_inited= 0; +static mysql_mutex_t LOCK_timer; +static mysql_cond_t COND_timer; +static QUEUE timer_queue; +pthread_t timer_thread; + +#define set_max_time(abs_time) \ + { (abs_time)->MY_tv_sec= INT_MAX32; (abs_time)->MY_tv_nsec= 0; } + + +static void *timer_handler(void *arg __attribute__((unused))); + +/* + Compare two timespecs +*/ + +static int compare_timespec(void *not_used __attribute__((unused)), + uchar *a_ptr, uchar *b_ptr) +{ + return cmp_timespec((*(struct timespec*) a_ptr), + (*(struct timespec*) b_ptr)); +} + + +/** + Initialize timer variables and create timer thread + + @param alloc_timers Init allocation of timers. Will be autoextended + if needed + @return 0 ok + @return 1 error; Can't create thread +*/ + +static thr_timer_t max_timer_data; + +my_bool init_thr_timer(uint alloc_timers) +{ + pthread_attr_t thr_attr; + my_bool res= 0; + DBUG_ENTER("init_thr_timer"); + + init_queue(&timer_queue, alloc_timers+2, offsetof(thr_timer_t,expire_time), + 0, compare_timespec, NullS, + offsetof(thr_timer_t, index_in_queue)+1, 1); + mysql_mutex_init(key_LOCK_timer, &LOCK_timer, MY_MUTEX_INIT_FAST); + mysql_cond_init(key_COND_timer, &COND_timer, NULL); + + /* Set dummy element with max time into the queue to simplify usage */ + bzero(&max_timer_data, sizeof(max_timer_data)); + set_max_time(&max_timer_data.expire_time); + queue_insert(&timer_queue, (uchar*) &max_timer_data); + next_timer_expire_time= max_timer_data.expire_time; + + /* Create a thread to handle timers */ + pthread_attr_init(&thr_attr); + pthread_attr_setscope(&thr_attr,PTHREAD_SCOPE_PROCESS); + pthread_attr_setstacksize(&thr_attr,8196); + thr_timer_inited= 1; + if (mysql_thread_create(key_thread_timer, &timer_thread, &thr_attr, + timer_handler, NULL)) + { + thr_timer_inited= 0; + res= 1; + mysql_mutex_destroy(&LOCK_timer); + mysql_cond_destroy(&COND_timer); + delete_queue(&timer_queue); + } + pthread_attr_destroy(&thr_attr); + + DBUG_RETURN(res); +} + + +void end_thr_timer(void) +{ + DBUG_ENTER("end_thr_timer"); + + if (!thr_timer_inited) + DBUG_VOID_RETURN; + + mysql_mutex_lock(&LOCK_timer); + thr_timer_inited= 0; /* Signal abort */ + mysql_cond_signal(&COND_timer); + mysql_mutex_unlock(&LOCK_timer); + pthread_join(timer_thread, NULL); + + mysql_mutex_destroy(&LOCK_timer); + mysql_cond_destroy(&COND_timer); + delete_queue(&timer_queue); + DBUG_VOID_RETURN; +} + + +/* + Initialize a timer object + + @param timer_data Timer structure + @param function Function to be called when getting timeout + @param argument Argument for function +*/ + +void thr_timer_init(thr_timer_t *timer_data, void(*function)(void*), + void *arg) +{ + DBUG_ENTER("thr_timer_init"); + bzero(timer_data, sizeof(*timer_data)); + timer_data->func= function; + timer_data->func_arg= arg; + timer_data->expired= 1; /* Not active */ + DBUG_VOID_RETURN; +} + + +/* + Request timer after X milliseconds + + SYNOPSIS + thr_timer() + timer_data Pointer to timer structure + micro_seconds; Number of microseconds until timer + + RETURN VALUES + 0 ok + 1 If no more timers are allowed (aborted by process) + + Stores in first argument a pointer to a non-zero int which is set to 0 + when the timer has been given +*/ + +my_bool thr_timer_settime(thr_timer_t *timer_data, ulonglong micro_seconds) +{ + int reschedule; + DBUG_ENTER("thr_timer_settime"); + DBUG_PRINT("enter",("thread: %s micro_seconds: %llu",my_thread_name(), + micro_seconds)); + + DBUG_ASSERT(timer_data->expired == 1); + + set_timespec_nsec(timer_data->expire_time, micro_seconds*1000); + timer_data->expired= 0; + + mysql_mutex_lock(&LOCK_timer); /* Lock from threads & timers */ + if (queue_insert_safe(&timer_queue,(uchar*) timer_data)) + { + DBUG_PRINT("info", ("timer queue full")); + fprintf(stderr,"Warning: thr_timer queue is full\n"); + timer_data->expired= 1; + mysql_mutex_unlock(&LOCK_timer); + DBUG_RETURN(1); + } + + /* Reschedule timer if the current one has more time left than new one */ + reschedule= cmp_timespec(next_timer_expire_time, timer_data->expire_time); + mysql_mutex_unlock(&LOCK_timer); + if (reschedule > 0) + { +#if defined(MAIN) + printf("reschedule\n"); fflush(stdout); +#endif + DBUG_PRINT("info", ("reschedule")); + mysql_cond_signal(&COND_timer); + } + + DBUG_RETURN(0); +} + + +/* + Remove timer from list of timers + + notes: Timer will be marked as expired +*/ + +void thr_timer_end(thr_timer_t *timer_data) +{ + DBUG_ENTER("thr_timer_end"); + + mysql_mutex_lock(&LOCK_timer); + if (!timer_data->expired) + { + DBUG_ASSERT(timer_data->index_in_queue != 0); + DBUG_ASSERT(queue_element(&timer_queue, timer_data->index_in_queue) == + (uchar*) timer_data); + queue_remove(&timer_queue, timer_data->index_in_queue); + /* Mark as expired for asserts to work */ + timer_data->expired= 1; + } + mysql_mutex_unlock(&LOCK_timer); + DBUG_VOID_RETURN; +} + + +/* + Come here when some timer in queue is due. +*/ + +static sig_handler process_timers(struct timespec *now) +{ + thr_timer_t *timer_data; + DBUG_ENTER("process_timers"); + DBUG_PRINT("info",("active timers: %d", timer_queue.elements - 1)); + +#if defined(MAIN) + printf("process_timer\n"); fflush(stdout); +#endif + + /* We can safely remove the first one as it has already expired */ + for (;;) + { + void (*function)(void*); + void *func_arg; + + timer_data= (thr_timer_t*) queue_top(&timer_queue); + function= timer_data->func; + func_arg= timer_data->func_arg; + timer_data->expired= 1; /* Mark expired */ + /* + We remove timer before calling timer function to allow thread to + delete it's timer data any time. + */ + queue_remove_top(&timer_queue); /* Remove timer */ + (*function)(func_arg); /* Inform thread of timeout */ + + /* Check if next one has also expired */ + timer_data= (thr_timer_t*) queue_top(&timer_queue); + if (cmp_timespec(timer_data->expire_time, (*now)) > 0) + break; /* All data processed */ + } + DBUG_VOID_RETURN; +} + + +/* + set up a timer thread to handle timeouts + This will be killed when thr_timer_inited is set to false. +*/ + +static void *timer_handler(void *arg __attribute__((unused))) +{ + my_thread_init(); + + mysql_mutex_lock(&LOCK_timer); + while (likely(thr_timer_inited)) + { + int error; + struct timespec *top_time; + struct timespec now, abstime; + + set_timespec(now, 0); + + top_time= &(((thr_timer_t*) queue_top(&timer_queue))->expire_time); + + if (cmp_timespec((*top_time), now) <= 0) + { + process_timers(&now); + top_time= &(((thr_timer_t*) queue_top(&timer_queue))->expire_time); + } + + abstime= *top_time; + next_timer_expire_time= *top_time; + if ((error= mysql_cond_timedwait(&COND_timer, &LOCK_timer, &abstime)) && + error != ETIME && error != ETIMEDOUT) + { +#ifdef MAIN + printf("Got error: %d from ptread_cond_timedwait (errno: %d)\n", + error,errno); +#endif + } + } + mysql_mutex_unlock(&LOCK_timer); + my_thread_end(); + pthread_exit(0); + return 0; /* Impossible */ +} + + +/**************************************************************************** + Testing of thr_timer (when compiled with -DMAIN) +***************************************************************************/ + +#ifdef MAIN + +static mysql_cond_t COND_thread_count; +static mysql_mutex_t LOCK_thread_count; +static uint thread_count, benchmark_runs, test_to_run= 1; + +static void send_signal(void *arg) +{ + struct st_my_thread_var *current_my_thread_var= arg; +#if defined(MAIN) + printf("sending signal\n"); fflush(stdout); +#endif + mysql_mutex_lock(¤t_my_thread_var->mutex); + mysql_cond_signal(¤t_my_thread_var->suspend); + mysql_mutex_unlock(¤t_my_thread_var->mutex); +} + + +static void run_thread_test(int param) +{ + int i,wait_time,retry; + my_hrtime_t start_time; + thr_timer_t timer_data; + struct st_my_thread_var *current_my_thread_var; + DBUG_ENTER("run_thread_test"); + + current_my_thread_var= my_thread_var; + thr_timer_init(&timer_data, send_signal, current_my_thread_var); + + for (i=1 ; i <= 10 ; i++) + { + wait_time=param ? 11-i : i; + start_time= my_hrtime(); + + mysql_mutex_lock(¤t_my_thread_var->mutex); + if (thr_timer_settime(&timer_data, wait_time * 1000000)) + { + printf("Thread: %s timers aborted\n",my_thread_name()); + break; + } + if (wait_time == 3) + { + printf("Thread: %s Simulation of no timer needed\n",my_thread_name()); + fflush(stdout); + } + else + { + for (retry=0 ; !timer_data.expired && retry < 10 ; retry++) + { + printf("Thread: %s Waiting %d sec\n",my_thread_name(),wait_time); + mysql_cond_wait(¤t_my_thread_var->suspend, + ¤t_my_thread_var->mutex); + + } + if (!timer_data.expired) + { + printf("Thread: %s didn't get an timer. Aborting!\n", + my_thread_name()); + break; + } + } + mysql_mutex_unlock(¤t_my_thread_var->mutex); + printf("Thread: %s Slept for %g (%d) sec\n",my_thread_name(), + (int) (my_hrtime().val-start_time.val)/1000000.0, wait_time); + fflush(stdout); + thr_timer_end(&timer_data); + fflush(stdout); + } + DBUG_VOID_RETURN; +} + + +static void run_thread_benchmark(int param) +{ + int i; + struct st_my_thread_var *current_my_thread_var; + thr_timer_t timer_data; + DBUG_ENTER("run_thread_benchmark"); + + current_my_thread_var= my_thread_var; + thr_timer_init(&timer_data, send_signal, current_my_thread_var); + + for (i=1 ; i <= param ; i++) + { + if (thr_timer_settime(&timer_data, 1000000)) + { + printf("Thread: %s timers aborted\n",my_thread_name()); + break; + } + thr_timer_end(&timer_data); + } + DBUG_VOID_RETURN; +} + + +#ifdef HAVE_TIMER_CREATE + +/* Test for benchmarking posix timers against thr_timer */ + +#ifndef sigev_notify_thread_id +#define sigev_notify_thread_id _sigev_un._tid +#endif + +static void run_timer_benchmark(int param) +{ + int i; + timer_t timerid; + struct sigevent sigev; + pid_t thread_id= (pid_t) syscall(SYS_gettid); + DBUG_ENTER("run_timer_benchmark"); + + /* Setup a signal that will never be signaled */ + sigev.sigev_value.sival_ptr= 0; + sigev.sigev_signo= SIGRTMIN; /* First free signal */ + sigev.sigev_notify= SIGEV_SIGNAL | SIGEV_THREAD_ID; + sigev.sigev_notify_thread_id= thread_id; + + if (timer_create(CLOCK_MONOTONIC, &sigev, &timerid)) + { + printf("Could not create timer\n"); + exit(1); + } + + for (i=1 ; i <= param ; i++) + { + struct itimerspec abstime; + abstime.it_interval.tv_sec= 0; + abstime.it_interval.tv_nsec= 0; + abstime.it_value.tv_sec= 1; + abstime.it_value.tv_nsec= 0; + + if (timer_settime(timerid, 0, &abstime, NULL)) + { + printf("Thread: %s timers aborted\n",my_thread_name()); + break; + } + abstime.it_interval.tv_sec= 0; + abstime.it_interval.tv_nsec= 0; + abstime.it_value.tv_sec= 0; + abstime.it_value.tv_nsec= 0; + timer_settime(timerid, 0, &abstime, NULL); + } + timer_delete(timerid); + DBUG_VOID_RETURN; +} +#endif /* HAVE_TIMER_CREATE */ + + +static void *start_thread(void *arg) +{ + my_thread_init(); + printf("Thread %d (%s) started\n",*((int*) arg),my_thread_name()); + fflush(stdout); + + switch (test_to_run) { + case 1: + run_thread_test(*((int*) arg)); + break; + case 2: + run_thread_benchmark(benchmark_runs); + break; + case 3: +#ifdef HAVE_TIMER_CREATE + run_timer_benchmark(benchmark_runs); +#endif + break; + } + free((uchar*) arg); + mysql_mutex_lock(&LOCK_thread_count); + thread_count--; + mysql_cond_signal(&COND_thread_count); /* Tell main we are ready */ + mysql_mutex_unlock(&LOCK_thread_count); + my_thread_end(); + return 0; +} + + +/* Start a lot of threads that will run with timers */ + +static void run_test() +{ + pthread_t tid; + pthread_attr_t thr_attr; + int i,*param,error; + DBUG_ENTER("run_test"); + + if (init_thr_timer(5)) + { + printf("Can't initialize timers\n"); + exit(1); + } + + mysql_mutex_init(0, &LOCK_thread_count, MY_MUTEX_INIT_FAST); + mysql_cond_init(0, &COND_thread_count, NULL); + + thr_setconcurrency(3); + pthread_attr_init(&thr_attr); + pthread_attr_setscope(&thr_attr,PTHREAD_SCOPE_PROCESS); + printf("Main thread: %s\n",my_thread_name()); + for (i=0 ; i < 2 ; i++) + { + param=(int*) malloc(sizeof(int)); + *param= i; + mysql_mutex_lock(&LOCK_thread_count); + if ((error= mysql_thread_create(0, + &tid, &thr_attr, start_thread, + (void*) param))) + { + printf("Can't create thread %d, error: %d\n",i,error); + exit(1); + } + thread_count++; + mysql_mutex_unlock(&LOCK_thread_count); + } + + pthread_attr_destroy(&thr_attr); + mysql_mutex_lock(&LOCK_thread_count); + while (thread_count) + { + mysql_cond_wait(&COND_thread_count, &LOCK_thread_count); + } + mysql_mutex_unlock(&LOCK_thread_count); + DBUG_ASSERT(timer_queue.elements == 1); + end_thr_timer(); + printf("Test succeeded\n"); + DBUG_VOID_RETURN; +} + + +int main(int argc __attribute__((unused)),char **argv __attribute__((unused))) +{ + MY_INIT(argv[0]); + + if (argc > 1 && argv[1][0] == '-') + { + switch (argv[1][1]) { + case '#': + test_to_run= 1; + DBUG_PUSH(argv[1]+2); + break; + case 'b': + test_to_run= 2; + benchmark_runs= atoi(argv[1]+2); + break; + case 't': + test_to_run= 3; + benchmark_runs= atoi(argv[1]+2); + break; + } + } + if (!benchmark_runs) + benchmark_runs= 1000000; + + run_test(); + my_end(1); + return 0; +} + +#endif /* MAIN */ diff --git a/mysys/tree.c b/mysys/tree.c index 4184fc4634a..83be0a0d20f 100644 --- a/mysys/tree.c +++ b/mysys/tree.c @@ -30,8 +30,8 @@ 3) if key_size is given to init_tree then each node will continue the key and calls to insert_key may increase length of key. if key_size > sizeof(pointer) and key_size is a multiple of 8 (double - allign) then key will be put on a 8 alligned adress. Else - the key will be on adress (element+1). This is transparent for user + align) then key will be put on a 8 aligned address. Else + the key will be on address (element+1). This is transparent for user compare and search functions uses a pointer to given key-argument. - If you use a free function for tree-elements and you are freeing @@ -77,6 +77,7 @@ static void rb_insert(TREE *tree,TREE_ELEMENT ***parent, TREE_ELEMENT *leaf); static void rb_delete_fixup(TREE *tree,TREE_ELEMENT ***parent); +static TREE_ELEMENT null_element= { NULL, NULL, 0, BLACK }; /* The actual code for handling binary trees */ @@ -85,8 +86,9 @@ static int test_rb_tree(TREE_ELEMENT *element); #endif void init_tree(TREE *tree, size_t default_alloc_size, size_t memory_limit, - int size, qsort_cmp2 compare, my_bool with_delete, - tree_element_free free_element, void *custom_arg) + int size, qsort_cmp2 compare, + tree_element_free free_element, void *custom_arg, + myf my_flags) { DBUG_ENTER("init_tree"); DBUG_PRINT("enter",("tree: 0x%lx size: %d", (long) tree, size)); @@ -94,8 +96,7 @@ void init_tree(TREE *tree, size_t default_alloc_size, size_t memory_limit, if (default_alloc_size < DEFAULT_ALLOC_SIZE) default_alloc_size= DEFAULT_ALLOC_SIZE; default_alloc_size= MY_ALIGN(default_alloc_size, DEFAULT_ALIGN_SIZE); - bzero((uchar*) &tree->null_element,sizeof(tree->null_element)); - tree->root= &tree->null_element; + tree->root= &null_element; tree->compare=compare; tree->size_of_element= size > 0 ? (uint) size : 0; tree->memory_limit=memory_limit; @@ -103,8 +104,7 @@ void init_tree(TREE *tree, size_t default_alloc_size, size_t memory_limit, tree->allocated=0; tree->elements_in_tree=0; tree->custom_arg = custom_arg; - tree->null_element.colour=BLACK; - tree->null_element.left=tree->null_element.right=0; + tree->my_flags= my_flags; tree->flag= 0; if (!free_element && size >= 0 && ((uint) size <= sizeof(void*) || ((uint) size & (sizeof(void*)-1)))) @@ -126,10 +126,10 @@ void init_tree(TREE *tree, size_t default_alloc_size, size_t memory_limit, tree->offset_to_key=0; /* use key through pointer */ tree->size_of_element+=sizeof(void*); } - if (!(tree->with_delete=with_delete)) + if (!(tree->with_delete= MY_TEST(my_flags & MY_TREE_WITH_DELETE))) { - init_alloc_root(&tree->mem_root, default_alloc_size, 0); - tree->mem_root.min_malloc=(sizeof(TREE_ELEMENT)+tree->size_of_element); + init_alloc_root(&tree->mem_root, default_alloc_size, 0, MYF(my_flags)); + tree->mem_root.min_malloc= sizeof(TREE_ELEMENT)+tree->size_of_element; } DBUG_VOID_RETURN; } @@ -156,7 +156,7 @@ static void free_tree(TREE *tree, myf free_flags) free_root(&tree->mem_root, free_flags); } } - tree->root= &tree->null_element; + tree->root= &null_element; tree->elements_in_tree=0; tree->allocated=0; @@ -177,7 +177,7 @@ void reset_tree(TREE* tree) static void delete_tree_element(TREE *tree, TREE_ELEMENT *element) { - if (element != &tree->null_element) + if (element != &null_element) { delete_tree_element(tree,element->left); if (tree->free) @@ -207,7 +207,7 @@ TREE_ELEMENT *tree_insert(TREE *tree, void *key, uint key_size, *parent = &tree->root; element= tree->root; for (;;) { - if (element == &tree->null_element || + if (element == &null_element || (cmp = (*tree->compare)(custom_arg, ELEMENT_KEY(tree,element), key)) == 0) break; @@ -220,11 +220,11 @@ TREE_ELEMENT *tree_insert(TREE *tree, void *key, uint key_size, *++parent = &element->left; element= element->left; } } - if (element == &tree->null_element) + if (element == &null_element) { uint alloc_size; if (tree->flag & TREE_ONLY_DUPS) - return((TREE_ELEMENT *) 1); + return TREE_ELEMENT_UNIQUE; alloc_size=sizeof(TREE_ELEMENT)+key_size+tree->size_of_element; tree->allocated+=alloc_size; @@ -237,13 +237,14 @@ TREE_ELEMENT *tree_insert(TREE *tree, void *key, uint key_size, key_size+=tree->size_of_element; if (tree->with_delete) - element=(TREE_ELEMENT *) my_malloc(alloc_size, MYF(MY_WME)); + element=(TREE_ELEMENT *) my_malloc(alloc_size, + MYF(tree->my_flags | MY_WME)); else element=(TREE_ELEMENT *) alloc_root(&tree->mem_root,alloc_size); if (!element) return(NULL); **parent=element; - element->left=element->right= &tree->null_element; + element->left=element->right= &null_element; if (!tree->offset_to_key) { if (key_size == sizeof(void*)) /* no length, save pointer */ @@ -285,7 +286,7 @@ int tree_delete(TREE *tree, void *key, uint key_size, void *custom_arg) *parent= &tree->root; element= tree->root; for (;;) { - if (element == &tree->null_element) + if (element == &null_element) return 1; /* Was not in tree */ if ((cmp = (*tree->compare)(custom_arg, ELEMENT_KEY(tree,element), key)) == 0) @@ -299,12 +300,12 @@ int tree_delete(TREE *tree, void *key, uint key_size, void *custom_arg) *++parent = &element->left; element= element->left; } } - if (element->left == &tree->null_element) + if (element->left == &null_element) { (**parent)=element->right; remove_colour= element->colour; } - else if (element->right == &tree->null_element) + else if (element->right == &null_element) { (**parent)=element->left; remove_colour= element->colour; @@ -313,7 +314,7 @@ int tree_delete(TREE *tree, void *key, uint key_size, void *custom_arg) { org_parent= parent; *++parent= &element->right; nod= element->right; - while (nod->left != &tree->null_element) + while (nod->left != &null_element) { *++parent= &nod->left; nod= nod->left; } @@ -343,7 +344,7 @@ void *tree_search(TREE *tree, void *key, void *custom_arg) for (;;) { - if (element == &tree->null_element) + if (element == &null_element) return (void*) 0; if ((cmp = (*tree->compare)(custom_arg, ELEMENT_KEY(tree,element), key)) == 0) @@ -368,8 +369,8 @@ void *tree_search_key(TREE *tree, const void *key, TODO: support for HA_READ_KEY_OR_PREV, HA_READ_PREFIX flags if needed. */ - *parents = &tree->null_element; - while (element != &tree->null_element) + *parents = &null_element; + while (element != &null_element) { *++parents= element; if ((cmp= (*tree->compare)(custom_arg, ELEMENT_KEY(tree, element), @@ -440,14 +441,14 @@ void *tree_search_edge(TREE *tree, TREE_ELEMENT **parents, { TREE_ELEMENT *element= tree->root; - *parents= &tree->null_element; - while (element != &tree->null_element) + *parents= &null_element; + while (element != &null_element) { *++parents= element; element= ELEMENT_CHILD(element, child_offs); } *last_pos= parents; - return **last_pos != &tree->null_element ? + return **last_pos != &null_element ? ELEMENT_KEY(tree, **last_pos) : NULL; } @@ -456,11 +457,11 @@ void *tree_search_next(TREE *tree, TREE_ELEMENT ***last_pos, int l_offs, { TREE_ELEMENT *x= **last_pos; - if (ELEMENT_CHILD(x, r_offs) != &tree->null_element) + if (ELEMENT_CHILD(x, r_offs) != &null_element) { x= ELEMENT_CHILD(x, r_offs); *++*last_pos= x; - while (ELEMENT_CHILD(x, l_offs) != &tree->null_element) + while (ELEMENT_CHILD(x, l_offs) != &null_element) { x= ELEMENT_CHILD(x, l_offs); *++*last_pos= x; @@ -470,12 +471,12 @@ void *tree_search_next(TREE *tree, TREE_ELEMENT ***last_pos, int l_offs, else { TREE_ELEMENT *y= *--*last_pos; - while (y != &tree->null_element && x == ELEMENT_CHILD(y, r_offs)) + while (y != &null_element && x == ELEMENT_CHILD(y, r_offs)) { x= y; y= *--*last_pos; } - return y == &tree->null_element ? NULL : ELEMENT_KEY(tree, y); + return y == &null_element ? NULL : ELEMENT_KEY(tree, y); } } @@ -491,7 +492,7 @@ ha_rows tree_record_pos(TREE *tree, const void *key, double left= 1; double right= tree->elements_in_tree; - while (element != &tree->null_element) + while (element != &null_element) { if ((cmp= (*tree->compare)(custom_arg, ELEMENT_KEY(tree, element), key)) == 0) diff --git a/mysys/typelib.c b/mysys/typelib.c index 2c36c97c6d6..9ca0847570f 100644 --- a/mysys/typelib.c +++ b/mysys/typelib.c @@ -1,4 +1,5 @@ /* Copyright (c) 2000, 2011, Oracle and/or its affiliates. All rights reserved. + Copyright (c) 2009, 2013, Monty Program Ab. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -44,18 +45,6 @@ int find_type_with_warning(const char *x, TYPELIB *typelib, const char *option) } -int find_type_or_exit(const char *x, TYPELIB *typelib, const char *option) -{ - int res; - if ((res= find_type_with_warning(x, typelib, option)) <= 0) - { - sf_leaking_memory= 1; /* no memory leak reports here */ - exit(1); - } - return res; -} - - /** Search after a string in a list of strings. Endspace in x is not compared. @@ -181,7 +170,7 @@ const char *get_type(TYPELIB *typelib, uint nr) /** - Create an integer value to represent the supplied comma-seperated + Create an integer value to represent the supplied comma-separated string where each string in the TYPELIB denotes a bit position. @param x string to decompose @@ -218,7 +207,7 @@ my_ulonglong find_typeset(char *x, TYPELIB *lib, int *err) x++; if ((find= find_type(i, lib, FIND_TYPE_COMMA_TERM) - 1) < 0) DBUG_RETURN(0); - result|= (ULL(1) << find); + result|= (1ULL << find); } *err= 0; DBUG_RETURN(result); diff --git a/mysys/waiting_threads.c b/mysys/waiting_threads.c index 1eecb8e99f9..35f19aa7edb 100644 --- a/mysys/waiting_threads.c +++ b/mysys/waiting_threads.c @@ -1,4 +1,5 @@ /* Copyright (C) 2008 MySQL AB, 2008-2009 Sun Microsystems, Inc. + Copyright (c) 2011, 2013, Monty Program Ab. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -135,7 +136,7 @@ Status ^^^^^^ - We calculate the number of successfull waits (WT_OK returned from + We calculate the number of successful waits (WT_OK returned from wt_thd_cond_timedwait()), a number of timeouts, a deadlock cycle length distribution - number of deadlocks with every length from 1 to WT_CYCLE_STATS, and a wait time distribution - number @@ -191,19 +192,12 @@ uint32 wt_wait_stats[WT_WAIT_STATS+1]; uint32 wt_cycle_stats[2][WT_CYCLE_STATS+1]; uint32 wt_success_stats; -static my_atomic_rwlock_t cycle_stats_lock, wait_stats_lock, success_stats_lock; - #ifdef HAVE_PSI_INTERFACE extern PSI_cond_key key_WT_RESOURCE_cond; #endif #ifdef SAFE_STATISTICS -#define incr(VAR, LOCK) \ - do { \ - my_atomic_rwlock_wrlock(&(LOCK)); \ - my_atomic_add32(&(VAR), 1); \ - my_atomic_rwlock_wrunlock(&(LOCK)); \ - } while(0) +#define incr(VAR, LOCK) do { my_atomic_add32(&(VAR), 1); } while(0) #else #define incr(VAR,LOCK) do { (VAR)++; } while(0) #endif @@ -259,11 +253,7 @@ struct st_wt_resource { #ifndef DBUG_OFF mysql_mutex_t *cond_mutex; /* a mutex for the 'cond' below */ #endif - /* - before the 'lock' all elements are mutable, after (and including) - - immutable in the sense that lf_hash_insert() won't memcpy() over them. - See wt_init(). - */ + #ifdef WT_RWLOCKS_USE_MUTEXES /* we need a special rwlock-like 'lock' to allow readers bypass @@ -395,15 +385,15 @@ static LF_HASH reshash; It's called from lf_hash and takes a pointer to an LF_SLIST instance. WT_RESOURCE is located at arg+sizeof(LF_SLIST) */ -static void wt_resource_init(uchar *arg) +static void wt_resource_create(uchar *arg) { WT_RESOURCE *rc= (WT_RESOURCE*)(arg+LF_HASH_OVERHEAD); - DBUG_ENTER("wt_resource_init"); + DBUG_ENTER("wt_resource_create"); bzero(rc, sizeof(*rc)); rc_rwlock_init(rc); mysql_cond_init(key_WT_RESOURCE_cond, &rc->cond, 0); - my_init_dynamic_array(&rc->owners, sizeof(WT_THD *), 0, 5); + my_init_dynamic_array(&rc->owners, sizeof(WT_THD *), 0, 5, MYF(0)); DBUG_VOID_RETURN; } @@ -425,25 +415,37 @@ static void wt_resource_destroy(uchar *arg) DBUG_VOID_RETURN; } +/** + WT_RESOURCE initializer + + It's called from lf_hash when an element is inserted. +*/ +static void wt_resource_init(LF_HASH *hash __attribute__((unused)), + WT_RESOURCE *rc, WT_RESOURCE_ID *id) +{ + DBUG_ENTER("wt_resource_init"); + rc->id= *id; + rc->waiter_count= 0; + rc->state= ACTIVE; +#ifndef DBUG_OFF + rc->cond_mutex= 0; +#endif + DBUG_VOID_RETURN; +} + static int wt_init_done; void wt_init() { DBUG_ENTER("wt_init"); - DBUG_ASSERT(reshash.alloc.constructor != wt_resource_init); + DBUG_ASSERT(reshash.alloc.constructor != wt_resource_create); lf_hash_init(&reshash, sizeof(WT_RESOURCE), LF_HASH_UNIQUE, 0, sizeof_WT_RESOURCE_ID, 0, 0); - reshash.alloc.constructor= wt_resource_init; + reshash.alloc.constructor= wt_resource_create; reshash.alloc.destructor= wt_resource_destroy; - /* - Note a trick: we initialize the hash with the real element size, - but fix it later to a shortened element size. This way - the allocator will allocate elements correctly, but - lf_hash_insert() will only overwrite part of the element with memcpy(). - lock, condition, and dynamic array will be intact. - */ - reshash.element_size= offsetof(WT_RESOURCE, lock); + reshash.initializer= (lf_hash_initializer) wt_resource_init; + bzero(wt_wait_stats, sizeof(wt_wait_stats)); bzero(wt_cycle_stats, sizeof(wt_cycle_stats)); wt_success_stats= 0; @@ -457,9 +459,6 @@ void wt_init() DBUG_ASSERT(i == 0 || wt_wait_table[i-1] != wt_wait_table[i]); } } - my_atomic_rwlock_init(&cycle_stats_lock); - my_atomic_rwlock_init(&success_stats_lock); - my_atomic_rwlock_init(&wait_stats_lock); wt_init_done= 1; DBUG_VOID_RETURN; } @@ -472,9 +471,6 @@ void wt_end() DBUG_ASSERT(reshash.count == 0); lf_hash_destroy(&reshash); - my_atomic_rwlock_destroy(&cycle_stats_lock); - my_atomic_rwlock_destroy(&success_stats_lock); - my_atomic_rwlock_destroy(&wait_stats_lock); reshash.alloc.constructor= NULL; wt_init_done= 0; DBUG_VOID_RETURN; @@ -510,7 +506,7 @@ void wt_thd_lazy_init(WT_THD *thd, const ulong *ds, const ulong *ts, thd->deadlock_search_depth_long= dl; thd->timeout_long= tl; /* dynamic array is also initialized lazily - without memory allocations */ - my_init_dynamic_array(&thd->my_resources, sizeof(WT_RESOURCE *), 0, 5); + my_init_dynamic_array(&thd->my_resources, sizeof(WT_RESOURCE *), 0, 5, MYF(0)); #ifndef DBUG_OFF thd->name= my_thread_name(); #endif @@ -560,7 +556,7 @@ my_bool wt_resource_id_memcmp(const void *a, const void *b) { /* we use the fact that there's no padding in the middle of WT_RESOURCE_ID */ compile_time_assert(offsetof(WT_RESOURCE_ID, type) == sizeof(ulonglong)); - return memcmp(a, b, sizeof_WT_RESOURCE_ID); + return MY_TEST(memcmp(a, b, sizeof_WT_RESOURCE_ID)); } /** @@ -604,8 +600,6 @@ static int deadlock_search(struct deadlock_arg *arg, WT_THD *blocker, DBUG_PRINT("wt", ("enter: thd=%s, blocker=%s, depth=%u", arg->thd->name, blocker->name, depth)); - LF_REQUIRE_PINS(1); - arg->last_locked_rc= 0; if (depth > arg->max_depth) @@ -923,8 +917,6 @@ int wt_thd_will_wait_for(WT_THD *thd, WT_THD *blocker, WT_RESOURCE *rc; DBUG_ENTER("wt_thd_will_wait_for"); - LF_REQUIRE_PINS(3); - DBUG_PRINT("wt", ("enter: thd=%s, blocker=%s, resid=%lu", thd->name, blocker->name, (ulong)resid->value)); @@ -946,14 +938,9 @@ int wt_thd_will_wait_for(WT_THD *thd, WT_THD *blocker, retry: while ((rc= lf_hash_search(&reshash, thd->pins, key, keylen)) == 0) { - WT_RESOURCE tmp; - DBUG_PRINT("wt", ("failed to find rc in hash, inserting")); - bzero(&tmp, sizeof(tmp)); - tmp.id= *resid; - tmp.state= ACTIVE; - if (lf_hash_insert(&reshash, thd->pins, &tmp) == -1) /* if OOM */ + if (lf_hash_insert(&reshash, thd->pins, resid) == -1) /* if OOM */ DBUG_RETURN(WT_DEADLOCK); /* Two cases: either lf_hash_insert() failed - because another thread @@ -1069,7 +1056,7 @@ int wt_thd_cond_timedwait(WT_THD *thd, mysql_mutex_t *mutex) ret= WT_OK; rc_unlock(rc); - end_wait_time= starttime.val *1000 + (*thd->timeout_short)*ULL(1000000); + end_wait_time= starttime.val *1000 + (*thd->timeout_short)*1000000ULL; set_timespec_time_nsec(timeout, end_wait_time); if (ret == WT_TIMEOUT && !thd->killed) ret= mysql_cond_timedwait(&rc->cond, mutex, &timeout); @@ -1082,7 +1069,7 @@ int wt_thd_cond_timedwait(WT_THD *thd, mysql_mutex_t *mutex) ret= WT_DEADLOCK; else if (*thd->timeout_long > *thd->timeout_short) { - end_wait_time= starttime.val *1000 + (*thd->timeout_long)*ULL(1000000); + end_wait_time= starttime.val *1000 + (*thd->timeout_long)*1000000ULL; set_timespec_time_nsec(timeout, end_wait_time); if (!thd->killed) ret= mysql_cond_timedwait(&rc->cond, mutex, &timeout); diff --git a/mysys/wqueue.c b/mysys/wqueue.c index 0c4f799c78f..6e4081a26e1 100644 --- a/mysys/wqueue.c +++ b/mysys/wqueue.c @@ -211,7 +211,7 @@ void wqueue_release_one_locktype_from_queue(WQUEUE *wqueue) /* Add thread and wait - SYNOPSYS + SYNOPSIS wqueue_add_and_wait() wqueue queue to add to thread thread which is waiting |