diff options
Diffstat (limited to 'mysys')
77 files changed, 7282 insertions, 1615 deletions
diff --git a/mysys/CMakeLists.txt b/mysys/CMakeLists.txt index 82deafa95bd..92269538c58 100644 --- a/mysys/CMakeLists.txt +++ b/mysys/CMakeLists.txt @@ -16,7 +16,7 @@ INCLUDE_DIRECTORIES(${ZLIB_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/include ${CMAKE_SOURCE_DIR}/mysys) -SET(MYSYS_SOURCES array.c charset-def.c charset.c checksum.c my_default.c +SET(MYSYS_SOURCES array.c charset-def.c charset.c my_default.c get_password.c errors.c hash.c list.c mf_cache.c mf_dirname.c mf_fn_ext.c @@ -39,17 +39,18 @@ SET(MYSYS_SOURCES array.c charset-def.c charset.c checksum.c my_default.c tree.c typelib.c base64.c my_memmem.c my_getpagesize.c guess_malloc_library.c - lf_alloc-pin.c lf_dynarray.c lf_hash.c + lf_alloc-pin.c lf_dynarray.c lf_hash.cc safemalloc.c my_new.cc my_getncpus.c my_safehash.c my_chmod.c my_rnd.c my_uuid.c wqueue.c waiting_threads.c ma_dyncol.c ../sql-common/my_time.c my_rdtsc.c my_context.c psi_noop.c - my_atomic_writes.c my_cpu.c my_likely.c - file_logger.c my_dlerror.c) + my_atomic_writes.c my_cpu.c my_likely.c my_largepage.c + file_logger.c my_dlerror.c crc32/crc32c.cc) IF (WIN32) SET (MYSYS_SOURCES ${MYSYS_SOURCES} my_winthread.c + my_wintoken.c my_wincond.c my_winerr.c my_winfile.c @@ -57,6 +58,98 @@ IF (WIN32) my_win_popen.cc) ENDIF() +IF(MSVC) + SET(MYSYS_SOURCES ${MYSYS_SOURCES} crc32/crc32_x86.c) + IF(CMAKE_SIZEOF_VOID_P EQUAL 8) + SET (MYSYS_SOURCES ${MYSYS_SOURCES} crc32/crc32c_amd64.cc) + ENDIF() + ADD_DEFINITIONS(-DHAVE_SSE42 -DHAVE_PCLMUL) + IF(CLANG_CL) + SET_SOURCE_FILES_PROPERTIES(crc32/crc32_x86.c PROPERTIES COMPILE_FLAGS "-msse4.2 -mpclmul") + ENDIF() +ELSEIF(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|amd64|i386|i686") + MY_CHECK_CXX_COMPILER_FLAG(-msse4.2) + MY_CHECK_CXX_COMPILER_FLAG(-mpclmul) + CHECK_INCLUDE_FILE(cpuid.h HAVE_CPUID_H) + CHECK_INCLUDE_FILE(x86intrin.h HAVE_X86INTRIN_H) + IF(have_CXX__msse4.2 AND HAVE_CPUID_H) + ADD_DEFINITIONS(-DHAVE_SSE42) + IF (have_CXX__mpclmul AND HAVE_X86INTRIN_H) + ADD_DEFINITIONS(-DHAVE_PCLMUL) + SET(MYSYS_SOURCES ${MYSYS_SOURCES} crc32/crc32_x86.c) + SET_SOURCE_FILES_PROPERTIES(crc32/crc32_x86.c PROPERTIES COMPILE_FLAGS "-msse4.2 -mpclmul") + IF(CMAKE_SIZEOF_VOID_P EQUAL 8) + SET(MYSYS_SOURCES ${MYSYS_SOURCES} crc32/crc32c_amd64.cc) + SET_SOURCE_FILES_PROPERTIES(crc32/crc32c_amd64.cc PROPERTIES COMPILE_FLAGS "-msse4.2 -mpclmul") + ENDIF() + ENDIF() + ENDIF() +ELSEIF(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|AARCH64") + IF(CMAKE_COMPILER_IS_GNUCC) + include(CheckCXXSourceCompiles) + + CHECK_CXX_SOURCE_COMPILES(" + #define CRC32CX(crc, value) __asm__(\"crc32cx %w[c], %w[c], %x[v]\":[c]\"+r\"(crc):[v]\"r\"(value)) + asm(\".arch_extension crc\"); + unsigned int foo(unsigned int ret) { + CRC32CX(ret, 0); + return ret; + } + #include <sys/auxv.h> + int main() { foo(0); + #ifdef __linux__ + getauxval(AT_HWCAP); + #else + unsigned long v; + elf_aux_info(AT_HWCAP, &v, sizeof(v)); + #endif + }" HAVE_ARMV8_CRC) + + CHECK_CXX_SOURCE_COMPILES(" + asm(\".arch_extension crypto\"); + unsigned int foo(unsigned int ret) { + __asm__(\"pmull v2.1q, v2.1d, v1.1d\"); + return ret; + } + #include <sys/auxv.h> + int main() { foo(0); + #ifdef __linux__ + getauxval(AT_HWCAP); + #else + unsigned long v; + elf_aux_info(AT_HWCAP, &v, sizeof(v)); + #endif + }" HAVE_ARMV8_CRYPTO) + + CHECK_C_COMPILER_FLAG(-march=armv8-a+crc+crypto HAVE_ARMV8_CRC_CRYPTO_MARCH) + + IF(HAVE_ARMV8_CRC_CRYPTO_MARCH) + CHECK_INCLUDE_FILE(arm_acle.h HAVE_ARM_ACLE_H -march=armv8-a+crc+crypto) + IF(HAVE_ARM_ACLE_H) + ADD_DEFINITIONS(-DHAVE_ARMV8_CRC_CRYPTO_INTRINSICS) + ENDIF() + IF(HAVE_ARMV8_CRC) + ADD_DEFINITIONS(-DHAVE_ARMV8_CRC) + ENDIF() + IF(HAVE_ARMV8_CRYPTO) + ADD_DEFINITIONS(-DHAVE_ARMV8_CRYPTO) + ENDIF() + SET(MYSYS_SOURCES ${MYSYS_SOURCES} crc32/crc32_arm64.c) + SET_SOURCE_FILES_PROPERTIES(crc32/crc32_arm64.c PROPERTIES + COMPILE_FLAGS "-march=armv8-a+crc+crypto") + ENDIF() + ENDIF() +ENDIF() + +IF(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64|powerpc64" OR CMAKE_SYSTEM_NAME MATCHES AIX) + SET(MYSYS_SOURCES ${MYSYS_SOURCES} crc32/crc32_ppc64.c crc32/crc32c_ppc.c) + SET_SOURCE_FILES_PROPERTIES(crc32/crc32_ppc64.c crc32/crc32c_ppc.c PROPERTIES + COMPILE_FLAGS "${COMPILE_FLAGS} -maltivec -mvsx -mpower8-vector -mcrypto -mpower8-vector") + ADD_DEFINITIONS(-DHAVE_POWER8 -DHAS_ALTIVEC) +ELSE() + SET (MYSYS_SOURCES ${MYSYS_SOURCES} crc32ieee.cc) +ENDIF() + IF(UNIX) SET (MYSYS_SOURCES ${MYSYS_SOURCES} my_addr_resolve.c my_setuser.c) ENDIF() @@ -65,17 +158,14 @@ IF(HAVE_ALARM) SET(MYSYS_SOURCES ${MYSYS_SOURCES} my_alarm.c) ENDIF() -IF(HAVE_LINUX_LARGE_PAGES) - SET(MYSYS_SOURCES ${MYSYS_SOURCES} my_largepage.c) -ENDIF() - IF(HAVE_MLOCK) SET(MYSYS_SOURCES ${MYSYS_SOURCES} my_lockmem.c) ENDIF() ADD_CONVENIENCE_LIBRARY(mysys ${MYSYS_SOURCES}) +MAYBE_DISABLE_IPO(mysys) TARGET_LINK_LIBRARIES(mysys dbug strings ${ZLIB_LIBRARY} - ${LIBNSL} ${LIBM} ${LIBRT} ${CMAKE_DL_LIBS} ${LIBSOCKET} ${LIBEXECINFO} ${CRC32_LIBRARY}) + ${LIBNSL} ${LIBM} ${LIBRT} ${CMAKE_DL_LIBS} ${LIBSOCKET} ${LIBEXECINFO}) DTRACE_INSTRUMENT(mysys) IF(HAVE_BFD_H) diff --git a/mysys/array.c b/mysys/array.c index e8b253a9e99..59698a4cc7c 100644 --- a/mysys/array.c +++ b/mysys/array.c @@ -40,9 +40,9 @@ FALSE Ok */ -my_bool init_dynamic_array2(DYNAMIC_ARRAY *array, uint element_size, - void *init_buffer, uint init_alloc, - uint alloc_increment, myf my_flags) +my_bool init_dynamic_array2(PSI_memory_key psi_key, DYNAMIC_ARRAY *array, + uint element_size, void *init_buffer, + uint init_alloc, uint alloc_increment, myf my_flags) { DBUG_ENTER("init_dynamic_array2"); if (!alloc_increment) @@ -55,6 +55,7 @@ my_bool init_dynamic_array2(DYNAMIC_ARRAY *array, uint element_size, array->max_element=init_alloc; array->alloc_increment=alloc_increment; array->size_of_element=element_size; + array->m_psi_key= psi_key; array->malloc_flags= my_flags; DBUG_ASSERT((my_flags & MY_INIT_BUFFER_USED) == 0); if ((array->buffer= init_buffer)) @@ -67,7 +68,7 @@ my_bool init_dynamic_array2(DYNAMIC_ARRAY *array, uint element_size, should not throw an error */ if (init_alloc && - !(array->buffer= (uchar*) my_malloc(element_size*init_alloc, + !(array->buffer= (uchar*) my_malloc(psi_key, element_size*init_alloc, MYF(my_flags)))) array->max_element=0; DBUG_RETURN(FALSE); @@ -133,7 +134,8 @@ void *alloc_dynamic(DYNAMIC_ARRAY *array) In this scenario, the buffer is statically preallocated, so we have to create an all-new malloc since we overflowed */ - if (!(new_ptr= (char *) my_malloc((array->max_element+ + if (!(new_ptr= (char *) my_malloc(array->m_psi_key, + (array->max_element+ array->alloc_increment) * array->size_of_element, MYF(array->malloc_flags | MY_WME)))) @@ -144,8 +146,8 @@ void *alloc_dynamic(DYNAMIC_ARRAY *array) array->malloc_flags&= ~MY_INIT_BUFFER_USED; } else if (!(new_ptr=(char*) - my_realloc(array->buffer,(array->max_element+ - array->alloc_increment)* + my_realloc(array->m_psi_key, array->buffer, + (array->max_element+ array->alloc_increment) * array->size_of_element, MYF(MY_WME | MY_ALLOW_ZERO_PTR | array->malloc_flags)))) @@ -242,7 +244,7 @@ my_bool allocate_dynamic(DYNAMIC_ARRAY *array, uint max_elements) In this senerio, the buffer is statically preallocated, so we have to create an all-new malloc since we overflowed */ - if (!(new_ptr= (uchar *) my_malloc(size * + if (!(new_ptr= (uchar *) my_malloc(array->m_psi_key, size * array->size_of_element, MYF(array->malloc_flags | MY_WME)))) DBUG_RETURN(0); @@ -250,7 +252,8 @@ my_bool allocate_dynamic(DYNAMIC_ARRAY *array, uint max_elements) array->elements * array->size_of_element); array->malloc_flags&= ~MY_INIT_BUFFER_USED; } - else if (!(new_ptr= (uchar*) my_realloc(array->buffer,size* + else if (!(new_ptr= (uchar*) my_realloc(array->m_psi_key, + array->buffer,size * array->size_of_element, MYF(MY_WME | MY_ALLOW_ZERO_PTR | array->malloc_flags)))) @@ -363,8 +366,8 @@ void freeze_size(DYNAMIC_ARRAY *array) elements= MY_MAX(array->elements, 1); if (array->buffer && array->max_element > elements) { - array->buffer=(uchar*) my_realloc(array->buffer, - elements*array->size_of_element, + array->buffer=(uchar*) my_realloc(array->m_psi_key, array->buffer, + elements * array->size_of_element, MYF(MY_WME | array->malloc_flags)); array->max_element= elements; } diff --git a/mysys/charset-def.c b/mysys/charset-def.c index b4317806762..259b7af9a60 100644 --- a/mysys/charset-def.c +++ b/mysys/charset-def.c @@ -116,37 +116,37 @@ extern struct charset_info_st my_charset_utf16_unicode_520_nopad_ci; #endif /* HAVE_CHARSET_utf16 */ -#ifdef HAVE_CHARSET_utf8 -extern struct charset_info_st my_charset_utf8_german2_uca_ci; -extern struct charset_info_st my_charset_utf8_icelandic_uca_ci; -extern struct charset_info_st my_charset_utf8_latvian_uca_ci; -extern struct charset_info_st my_charset_utf8_romanian_uca_ci; -extern struct charset_info_st my_charset_utf8_slovenian_uca_ci; -extern struct charset_info_st my_charset_utf8_polish_uca_ci; -extern struct charset_info_st my_charset_utf8_estonian_uca_ci; -extern struct charset_info_st my_charset_utf8_spanish_uca_ci; -extern struct charset_info_st my_charset_utf8_swedish_uca_ci; -extern struct charset_info_st my_charset_utf8_turkish_uca_ci; -extern struct charset_info_st my_charset_utf8_czech_uca_ci; -extern struct charset_info_st my_charset_utf8_danish_uca_ci; -extern struct charset_info_st my_charset_utf8_lithuanian_uca_ci; -extern struct charset_info_st my_charset_utf8_slovak_uca_ci; -extern struct charset_info_st my_charset_utf8_spanish2_uca_ci; -extern struct charset_info_st my_charset_utf8_roman_uca_ci; -extern struct charset_info_st my_charset_utf8_persian_uca_ci; -extern struct charset_info_st my_charset_utf8_esperanto_uca_ci; -extern struct charset_info_st my_charset_utf8_hungarian_uca_ci; -extern struct charset_info_st my_charset_utf8_croatian_mysql561_uca_ci; -extern struct charset_info_st my_charset_utf8_sinhala_uca_ci; -extern struct charset_info_st my_charset_utf8_unicode_520_ci; -extern struct charset_info_st my_charset_utf8_vietnamese_ci; -extern struct charset_info_st my_charset_utf8_croatian_uca_ci; -extern struct charset_info_st my_charset_utf8_myanmar_uca_ci; -extern struct charset_info_st my_charset_utf8_thai_520_w2; +#ifdef HAVE_CHARSET_utf8mb3 +extern struct charset_info_st my_charset_utf8mb3_german2_uca_ci; +extern struct charset_info_st my_charset_utf8mb3_icelandic_uca_ci; +extern struct charset_info_st my_charset_utf8mb3_latvian_uca_ci; +extern struct charset_info_st my_charset_utf8mb3_romanian_uca_ci; +extern struct charset_info_st my_charset_utf8mb3_slovenian_uca_ci; +extern struct charset_info_st my_charset_utf8mb3_polish_uca_ci; +extern struct charset_info_st my_charset_utf8mb3_estonian_uca_ci; +extern struct charset_info_st my_charset_utf8mb3_spanish_uca_ci; +extern struct charset_info_st my_charset_utf8mb3_swedish_uca_ci; +extern struct charset_info_st my_charset_utf8mb3_turkish_uca_ci; +extern struct charset_info_st my_charset_utf8mb3_czech_uca_ci; +extern struct charset_info_st my_charset_utf8mb3_danish_uca_ci; +extern struct charset_info_st my_charset_utf8mb3_lithuanian_uca_ci; +extern struct charset_info_st my_charset_utf8mb3_slovak_uca_ci; +extern struct charset_info_st my_charset_utf8mb3_spanish2_uca_ci; +extern struct charset_info_st my_charset_utf8mb3_roman_uca_ci; +extern struct charset_info_st my_charset_utf8mb3_persian_uca_ci; +extern struct charset_info_st my_charset_utf8mb3_esperanto_uca_ci; +extern struct charset_info_st my_charset_utf8mb3_hungarian_uca_ci; +extern struct charset_info_st my_charset_utf8mb3_croatian_mysql561_uca_ci; +extern struct charset_info_st my_charset_utf8mb3_sinhala_uca_ci; +extern struct charset_info_st my_charset_utf8mb3_unicode_520_ci; +extern struct charset_info_st my_charset_utf8mb3_vietnamese_ci; +extern struct charset_info_st my_charset_utf8mb3_croatian_uca_ci; +extern struct charset_info_st my_charset_utf8mb3_myanmar_uca_ci; +extern struct charset_info_st my_charset_utf8mb3_thai_520_w2; #ifdef HAVE_UTF8_GENERAL_CS -extern struct charset_info_st my_charset_utf8_general_cs; +extern struct charset_info_st my_charset_utf8mb3_general_cs; #endif -extern struct charset_info_st my_charset_utf8_unicode_520_nopad_ci; +extern struct charset_info_st my_charset_utf8mb3_unicode_520_nopad_ci; #endif #ifdef HAVE_CHARSET_utf8mb4 @@ -304,47 +304,47 @@ my_bool init_compiled_charsets(myf flags __attribute__((unused))) add_compiled_collation(&my_charset_ujis_nopad_bin); #endif -#ifdef HAVE_CHARSET_utf8 - add_compiled_collation(&my_charset_utf8_general_ci); - add_compiled_collation(&my_charset_utf8_general_nopad_ci); - add_compiled_collation(&my_charset_utf8_bin); - add_compiled_collation(&my_charset_utf8_nopad_bin); - add_compiled_collation(&my_charset_utf8_general_mysql500_ci); +#ifdef HAVE_CHARSET_utf8mb3 + add_compiled_collation(&my_charset_utf8mb3_general_ci); + add_compiled_collation(&my_charset_utf8mb3_general_nopad_ci); + add_compiled_collation(&my_charset_utf8mb3_bin); + add_compiled_collation(&my_charset_utf8mb3_nopad_bin); + add_compiled_collation(&my_charset_utf8mb3_general_mysql500_ci); #ifdef HAVE_UTF8_GENERAL_CS - add_compiled_collation(&my_charset_utf8_general_cs); + add_compiled_collation(&my_charset_utf8mb3_general_cs); #endif #ifdef HAVE_UCA_COLLATIONS - add_compiled_collation(&my_charset_utf8_unicode_ci); - add_compiled_collation(&my_charset_utf8_german2_uca_ci); - add_compiled_collation(&my_charset_utf8_icelandic_uca_ci); - add_compiled_collation(&my_charset_utf8_latvian_uca_ci); - add_compiled_collation(&my_charset_utf8_romanian_uca_ci); - add_compiled_collation(&my_charset_utf8_slovenian_uca_ci); - add_compiled_collation(&my_charset_utf8_polish_uca_ci); - add_compiled_collation(&my_charset_utf8_estonian_uca_ci); - add_compiled_collation(&my_charset_utf8_spanish_uca_ci); - add_compiled_collation(&my_charset_utf8_swedish_uca_ci); - add_compiled_collation(&my_charset_utf8_turkish_uca_ci); - add_compiled_collation(&my_charset_utf8_czech_uca_ci); - add_compiled_collation(&my_charset_utf8_danish_uca_ci); - add_compiled_collation(&my_charset_utf8_lithuanian_uca_ci); - add_compiled_collation(&my_charset_utf8_slovak_uca_ci); - add_compiled_collation(&my_charset_utf8_spanish2_uca_ci); - add_compiled_collation(&my_charset_utf8_roman_uca_ci); - add_compiled_collation(&my_charset_utf8_persian_uca_ci); - add_compiled_collation(&my_charset_utf8_esperanto_uca_ci); - add_compiled_collation(&my_charset_utf8_hungarian_uca_ci); - add_compiled_collation(&my_charset_utf8_croatian_mysql561_uca_ci); - add_compiled_collation(&my_charset_utf8_sinhala_uca_ci); - add_compiled_collation(&my_charset_utf8_unicode_520_ci); - add_compiled_collation(&my_charset_utf8_vietnamese_ci); - add_compiled_collation(&my_charset_utf8_croatian_uca_ci); - add_compiled_collation(&my_charset_utf8_myanmar_uca_ci); - add_compiled_collation(&my_charset_utf8_thai_520_w2); - add_compiled_collation(&my_charset_utf8_unicode_nopad_ci); - add_compiled_collation(&my_charset_utf8_unicode_520_nopad_ci); + add_compiled_collation(&my_charset_utf8mb3_unicode_ci); + add_compiled_collation(&my_charset_utf8mb3_german2_uca_ci); + add_compiled_collation(&my_charset_utf8mb3_icelandic_uca_ci); + add_compiled_collation(&my_charset_utf8mb3_latvian_uca_ci); + add_compiled_collation(&my_charset_utf8mb3_romanian_uca_ci); + add_compiled_collation(&my_charset_utf8mb3_slovenian_uca_ci); + add_compiled_collation(&my_charset_utf8mb3_polish_uca_ci); + add_compiled_collation(&my_charset_utf8mb3_estonian_uca_ci); + add_compiled_collation(&my_charset_utf8mb3_spanish_uca_ci); + add_compiled_collation(&my_charset_utf8mb3_swedish_uca_ci); + add_compiled_collation(&my_charset_utf8mb3_turkish_uca_ci); + add_compiled_collation(&my_charset_utf8mb3_czech_uca_ci); + add_compiled_collation(&my_charset_utf8mb3_danish_uca_ci); + add_compiled_collation(&my_charset_utf8mb3_lithuanian_uca_ci); + add_compiled_collation(&my_charset_utf8mb3_slovak_uca_ci); + add_compiled_collation(&my_charset_utf8mb3_spanish2_uca_ci); + add_compiled_collation(&my_charset_utf8mb3_roman_uca_ci); + add_compiled_collation(&my_charset_utf8mb3_persian_uca_ci); + add_compiled_collation(&my_charset_utf8mb3_esperanto_uca_ci); + add_compiled_collation(&my_charset_utf8mb3_hungarian_uca_ci); + add_compiled_collation(&my_charset_utf8mb3_croatian_mysql561_uca_ci); + add_compiled_collation(&my_charset_utf8mb3_sinhala_uca_ci); + add_compiled_collation(&my_charset_utf8mb3_unicode_520_ci); + add_compiled_collation(&my_charset_utf8mb3_vietnamese_ci); + add_compiled_collation(&my_charset_utf8mb3_croatian_uca_ci); + add_compiled_collation(&my_charset_utf8mb3_myanmar_uca_ci); + add_compiled_collation(&my_charset_utf8mb3_thai_520_w2); + add_compiled_collation(&my_charset_utf8mb3_unicode_nopad_ci); + add_compiled_collation(&my_charset_utf8mb3_unicode_520_nopad_ci); #endif -#endif /* HAVE_CHARSET_utf8 */ +#endif /* HAVE_CHARSET_utf8mb3 */ #ifdef HAVE_CHARSET_utf8mb4 @@ -469,7 +469,7 @@ my_bool init_compiled_charsets(myf flags __attribute__((unused))) /* Copy compiled charsets */ for (cs=compiled_charsets; cs->name; cs++) - add_compiled_collation((struct charset_info_st *) cs); + add_compiled_extra_collation((struct charset_info_st *) cs); return FALSE; } diff --git a/mysys/charset.c b/mysys/charset.c index f44dc7606c1..32cfeb56e2d 100644 --- a/mysys/charset.c +++ b/mysys/charset.c @@ -1,5 +1,6 @@ /* Copyright (c) 2000, 2011, Oracle and/or its affiliates + Copyright (c) 2009, 2020, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -19,6 +20,7 @@ #include <m_ctype.h> #include <m_string.h> #include <my_dir.h> +#include <hash.h> #include <my_xml.h> #ifdef HAVE_LANGINFO_H #include <langinfo.h> @@ -27,6 +29,8 @@ #include <locale.h> #endif +extern HASH charset_name_hash; + /* The code below implements this functionality: @@ -37,15 +41,10 @@ - Setting server default character set */ -my_bool my_charset_same(CHARSET_INFO *cs1, CHARSET_INFO *cs2) -{ - return ((cs1 == cs2) || !strcmp(cs1->csname,cs2->csname)); -} - - static uint get_collation_number_internal(const char *name) { + CHARSET_INFO **cs; for (cs= all_charsets; cs < all_charsets + array_elements(all_charsets); @@ -61,7 +60,7 @@ get_collation_number_internal(const char *name) static my_bool is_multi_byte_ident(CHARSET_INFO *cs, uchar ch) { - int chlen= my_charlen(cs, (const char *) &ch, (const char *) &ch + 1); + int chlen= my_ci_charlen(cs, &ch, &ch + 1); return MY_CS_IS_TOOSMALL(chlen) ? TRUE : FALSE; } @@ -71,11 +70,10 @@ static my_bool init_state_maps(struct charset_info_st *cs) uchar *state_map; uchar *ident_map; - if (!(cs->state_map= state_map= (uchar*) my_once_alloc(256, MYF(MY_WME)))) + if (!(cs->state_map= state_map= (uchar*) my_once_alloc(256*2, MYF(MY_WME)))) return 1; - if (!(cs->ident_map= ident_map= (uchar*) my_once_alloc(256, MYF(MY_WME)))) - return 1; + cs->ident_map= ident_map= state_map + 256; /* Fill state_map with states to get a faster parser */ for (i=0; i < 256 ; i++) @@ -152,7 +150,8 @@ static int cs_copy_data(struct charset_info_st *to, CHARSET_INFO *from) { to->number= from->number ? from->number : to->number; - if (from->csname) + /* Don't replace csname if already set */ + if (from->csname && !to->csname) if (!(to->csname= my_once_strdup(from->csname,MYF(MY_WME)))) goto err; @@ -164,11 +163,11 @@ static int cs_copy_data(struct charset_info_st *to, CHARSET_INFO *from) if (!(to->comment= my_once_strdup(from->comment,MYF(MY_WME)))) goto err; - if (from->ctype) + if (from->m_ctype) { - if (!(to->ctype= (uchar*) my_once_memdup((char*) from->ctype, - MY_CS_CTYPE_TABLE_SIZE, - MYF(MY_WME)))) + if (!(to->m_ctype= (uchar*) my_once_memdup((char*) from->m_ctype, + MY_CS_CTYPE_TABLE_SIZE, + MYF(MY_WME)))) goto err; if (init_state_maps(to)) goto err; @@ -212,7 +211,7 @@ err: static my_bool simple_8bit_charset_data_is_full(CHARSET_INFO *cs) { - return cs->ctype && cs->to_upper && cs->to_lower && cs->tab_to_uni; + return cs->m_ctype && cs->to_upper && cs->to_lower && cs->tab_to_uni; } @@ -228,8 +227,8 @@ inherit_charset_data(struct charset_info_st *cs, CHARSET_INFO *refcs) cs->to_upper= refcs->to_upper; if (!cs->to_lower) cs->to_lower= refcs->to_lower; - if (!cs->ctype) - cs->ctype= refcs->ctype; + if (!cs->m_ctype) + cs->m_ctype= refcs->m_ctype; if (!cs->tab_to_uni) cs->tab_to_uni= refcs->tab_to_uni; } @@ -262,7 +261,7 @@ static my_bool simple_cs_is_full(CHARSET_INFO *cs) } -#if defined(HAVE_UCA_COLLATIONS) && (defined(HAVE_CHARSET_ucs2) || defined(HAVE_CHARSET_utf8)) +#if defined(HAVE_UCA_COLLATIONS) && (defined(HAVE_CHARSET_ucs2) || defined(HAVE_CHARSET_utf8mb3)) /** Initialize a loaded collation. @param [OUT] to - The new charset_info_st structure to initialize. @@ -321,7 +320,21 @@ static int add_collation(struct charset_info_st *cs) return MY_XML_ERROR; bzero(newcs,sizeof(CHARSET_INFO)); } - + else + { + /* Don't allow change of csname */ + if (newcs->csname && strcmp(newcs->csname, cs->csname)) + { + my_error(EE_DUPLICATE_CHARSET, MYF(ME_WARNING), + cs->number, cs->csname, newcs->csname); + /* + Continue parsing rest of Index.xml. We got an warning in the log + so the user can fix the wrong character set definition. + */ + return MY_XML_OK; + } + } + if (cs->primary_number == cs->number) cs->state |= MY_CS_PRIMARY; @@ -350,12 +363,12 @@ static int add_collation(struct charset_info_st *cs) } else if (!strcmp(cs->csname, "utf8") || !strcmp(cs->csname, "utf8mb3")) { -#if defined (HAVE_CHARSET_utf8) && defined(HAVE_UCA_COLLATIONS) +#if defined (HAVE_CHARSET_utf8mb3) && defined(HAVE_UCA_COLLATIONS) copy_uca_collation(newcs, newcs->state & MY_CS_NOPAD ? - &my_charset_utf8_unicode_nopad_ci : - &my_charset_utf8_unicode_ci, + &my_charset_utf8mb3_unicode_nopad_ci : + &my_charset_utf8mb3_unicode_ci, cs); - newcs->ctype= my_charset_utf8_unicode_ci.ctype; + newcs->m_ctype= my_charset_utf8mb3_unicode_ci.m_ctype; if (init_state_maps(newcs)) return MY_XML_ERROR; #endif @@ -367,7 +380,7 @@ static int add_collation(struct charset_info_st *cs) &my_charset_utf8mb4_unicode_nopad_ci : &my_charset_utf8mb4_unicode_ci, cs); - newcs->ctype= my_charset_utf8mb4_unicode_ci.ctype; + newcs->m_ctype= my_charset_utf8mb4_unicode_ci.m_ctype; newcs->state|= MY_CS_AVAILABLE | MY_CS_LOADED; #endif } @@ -401,8 +414,8 @@ static int add_collation(struct charset_info_st *cs) { newcs->state |= MY_CS_LOADED; } - newcs->state|= MY_CS_AVAILABLE; } + add_compiled_extra_collation(newcs); } else { @@ -419,7 +432,7 @@ static int add_collation(struct charset_info_st *cs) if (cs->comment) if (!(newcs->comment= my_once_strdup(cs->comment,MYF(MY_WME)))) return MY_XML_ERROR; - if (cs->csname) + if (cs->csname && ! newcs->csname) if (!(newcs->csname= my_once_strdup(cs->csname,MYF(MY_WME)))) return MY_XML_ERROR; if (cs->name) @@ -462,12 +475,12 @@ my_once_alloc_c(size_t size) static void * my_malloc_c(size_t size) -{ return my_malloc(size, MYF(MY_WME)); } +{ return my_malloc(key_memory_charset_loader, size, MYF(MY_WME)); } static void * my_realloc_c(void *old, size_t size) -{ return my_realloc(old, size, MYF(MY_WME|MY_ALLOW_ZERO_PTR)); } +{ return my_realloc(key_memory_charset_loader, old, size, MYF(MY_WME|MY_ALLOW_ZERO_PTR)); } /** @@ -505,7 +518,7 @@ my_read_charset_file(MY_CHARSET_LOADER *loader, if (!my_stat(filename, &stat_info, MYF(myflags)) || ((len= (uint)stat_info.st_size) > MY_MAX_ALLOWED_BUF) || - !(buf= (uchar*) my_malloc(len,myflags))) + !(buf= (uchar*) my_malloc(key_memory_charset_loader,len,myflags))) return TRUE; if ((fd= mysql_file_open(key_file_charset, filename, O_RDONLY, myflags)) < 0) @@ -556,14 +569,55 @@ char *get_charsets_dir(char *buf) CHARSET_INFO *all_charsets[MY_ALL_CHARSETS_SIZE]={NULL}; CHARSET_INFO *default_charset_info = &my_charset_latin1; + +/* + Add standard character set compiled into the application + All related character sets should share same cname +*/ + void add_compiled_collation(struct charset_info_st *cs) { DBUG_ASSERT(cs->number < array_elements(all_charsets)); all_charsets[cs->number]= cs; cs->state|= MY_CS_AVAILABLE; + if ((my_hash_insert(&charset_name_hash, (uchar*) cs))) + { +#ifndef DBUG_OFF + CHARSET_INFO *org= (CHARSET_INFO*) my_hash_search(&charset_name_hash, + (uchar*) cs->csname, + strlen(cs->csname)); + DBUG_ASSERT(org); + DBUG_ASSERT(org->csname == cs->csname); +#endif + } } +/* + Add optional characters sets from ctype-extra.c + + If cname is already in use, replace csname in new object with a pointer to + the already used csname to ensure that all csname's points to the same string + for the same character set. +*/ + + +void add_compiled_extra_collation(struct charset_info_st *cs) +{ + DBUG_ASSERT(cs->number < array_elements(all_charsets)); + all_charsets[cs->number]= cs; + cs->state|= MY_CS_AVAILABLE; + if ((my_hash_insert(&charset_name_hash, (uchar*) cs))) + { + CHARSET_INFO *org= (CHARSET_INFO*) my_hash_search(&charset_name_hash, + (uchar*) cs->csname, + strlen(cs->csname)); + cs->csname= org->csname; + } +} + + + static my_pthread_once_t charsets_initialized= MY_PTHREAD_ONCE_INIT; static my_pthread_once_t charsets_template= MY_PTHREAD_ONCE_INIT; @@ -611,14 +665,31 @@ const char *my_collation_get_tailoring(uint id) } +HASH charset_name_hash; + +static uchar *get_charset_key(const uchar *object, + size_t *size, + my_bool not_used __attribute__((unused))) +{ + CHARSET_INFO *cs= (CHARSET_INFO*) object; + *size= strlen(cs->csname); + return (uchar*) cs->csname; +} + static void init_available_charsets(void) { char fname[FN_REFLEN + sizeof(MY_CHARSET_INDEX)]; struct charset_info_st **cs; MY_CHARSET_LOADER loader; + DBUG_ENTER("init_available_charsets"); bzero((char*) &all_charsets,sizeof(all_charsets)); bzero((char*) &my_collation_statistics, sizeof(my_collation_statistics)); + + my_hash_init2(key_memory_charsets, &charset_name_hash, 16, + &my_charset_latin1, 64, 0, 0, get_charset_key, + 0, 0, HASH_UNIQUE); + init_compiled_charsets(MYF(0)); /* Copy compiled charsets */ @@ -630,7 +701,7 @@ static void init_available_charsets(void) if (*cs) { DBUG_ASSERT(cs[0]->mbmaxlen <= MY_CS_MBMAXLEN); - if (cs[0]->ctype) + if (cs[0]->m_ctype) if (init_state_maps(*cs)) *cs= NULL; } @@ -639,12 +710,14 @@ static void init_available_charsets(void) my_charset_loader_init_mysys(&loader); strmov(get_charsets_dir(fname), MY_CHARSET_INDEX); my_read_charset_file(&loader, fname, MYF(0)); + DBUG_VOID_RETURN; } void free_charsets(void) { charsets_initialized= charsets_template; + my_hash_free(&charset_name_hash); } @@ -807,8 +880,8 @@ get_internal_charset(MY_CHARSET_LOADER *loader, uint cs_number, myf flags) inherit_collation_data(cs, refcl); } - if ((cs->cset->init && cs->cset->init(cs, loader)) || - (cs->coll->init && cs->coll->init(cs, loader))) + if (my_ci_init_charset(cs, loader) || + my_ci_init_collation(cs, loader)) { cs= NULL; } @@ -1036,8 +1109,7 @@ size_t escape_string_for_mysql(CHARSET_INFO *charset_info, { char escape= 0; #ifdef USE_MB - int tmp_length= use_mb(charset_info) ? my_charlen(charset_info, from, end) : - 1; + int tmp_length= my_ci_charlen(charset_info, (const uchar *) from, (const uchar *) end); if (tmp_length > 1) { if (to + tmp_length > to_end) @@ -1174,7 +1246,7 @@ size_t escape_quotes_for_mysql(CHARSET_INFO *charset_info, const char *end, *to_end=to_start + (to_length ? to_length-1 : 2*length); my_bool overflow= FALSE; #ifdef USE_MB - my_bool use_mb_flag= use_mb(charset_info); + my_bool use_mb_flag= my_ci_use_mb(charset_info); #endif for (end= from + length; from < end; from++) { @@ -1431,4 +1503,4 @@ const char* my_default_csname() csname = my_os_charset_to_mysql_charset(csname); #endif return csname ? csname : MYSQL_DEFAULT_CHARSET_NAME; -}
\ No newline at end of file +} diff --git a/mysys/checksum.c b/mysys/checksum.c deleted file mode 100644 index 91e681d0db4..00000000000 --- a/mysys/checksum.c +++ /dev/null @@ -1,42 +0,0 @@ -/* Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved. - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; version 2 of the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ - - -#include <my_global.h> -#include <my_sys.h> -#include <zlib.h> - -/* - Calculate a long checksum for a memoryblock. - - SYNOPSIS - my_checksum() - crc start value for crc - pos pointer to memory block - length length of the block -*/ - -ha_checksum my_checksum(ha_checksum crc, const uchar *pos, size_t length) -{ -#ifdef HAVE_CRC32_VPMSUM - extern unsigned int crc32ieee_vpmsum(unsigned int crc, const unsigned char *p, - unsigned long len); - crc= (ha_checksum) crc32ieee_vpmsum((uint) crc, pos, (uint) length); -#else - crc= (ha_checksum) crc32((uint)crc, pos, (uint) length); -#endif - DBUG_PRINT("info", ("crc: %lu", (ulong) crc)); - return crc; -} diff --git a/mysys/crc32/clang_workaround.h b/mysys/crc32/clang_workaround.h new file mode 100644 index 00000000000..915f7e5282f --- /dev/null +++ b/mysys/crc32/clang_workaround.h @@ -0,0 +1,87 @@ +#ifndef CLANG_WORKAROUNDS_H +#define CLANG_WORKAROUNDS_H + +/* + * These stubs fix clang incompatibilities with GCC builtins. + */ + +#ifndef __builtin_crypto_vpmsumw +#define __builtin_crypto_vpmsumw __builtin_crypto_vpmsumb +#endif +#ifndef __builtin_crypto_vpmsumd +#define __builtin_crypto_vpmsumd __builtin_crypto_vpmsumb +#endif + +static inline +__vector unsigned long long __attribute__((overloadable)) +vec_ld(int __a, const __vector unsigned long long* __b) +{ + return (__vector unsigned long long)__builtin_altivec_lvx(__a, __b); +} + +/* + * GCC __builtin_pack_vector_int128 returns a vector __int128_t but Clang + * does not recognize this type. On GCC this builtin is translated to a + * xxpermdi instruction that only moves the registers __a, __b instead generates + * a load. + * + * Clang has vec_xxpermdi intrinsics. It was implemented in 4.0.0. + */ +static inline +__vector unsigned long long __builtin_pack_vector (unsigned long __a, + unsigned long __b) +{ + #if defined(__BIG_ENDIAN__) + __vector unsigned long long __v = {__a, __b}; + #else + __vector unsigned long long __v = {__b, __a}; + #endif + return __v; +} + +/* + * Clang 7 changed the behavior of vec_xxpermdi in order to provide the same + * behavior of GCC. That means code adapted to Clang >= 7 does not work on + * Clang <= 6. So, fallback to __builtin_unpack_vector() on Clang <= 6. + */ +#if !defined vec_xxpermdi || __clang_major__ <= 6 + +static inline +unsigned long __builtin_unpack_vector (__vector unsigned long long __v, + int __o) +{ + return __v[__o]; +} + +#if defined(__BIG_ENDIAN__) +#define __builtin_unpack_vector_0(a) __builtin_unpack_vector ((a), 0) +#define __builtin_unpack_vector_1(a) __builtin_unpack_vector ((a), 1) +#else +#define __builtin_unpack_vector_0(a) __builtin_unpack_vector ((a), 1) +#define __builtin_unpack_vector_1(a) __builtin_unpack_vector ((a), 0) +#endif + +#else + +static inline +unsigned long __builtin_unpack_vector_0 (__vector unsigned long long __v) +{ + #if defined(__BIG_ENDIAN__) + return vec_xxpermdi(__v, __v, 0x0)[0]; + #else + return vec_xxpermdi(__v, __v, 0x3)[0]; + #endif +} + +static inline +unsigned long __builtin_unpack_vector_1 (__vector unsigned long long __v) +{ + #if defined(__BIG_ENDIAN__) + return vec_xxpermdi(__v, __v, 0x3)[0]; + #else + return vec_xxpermdi(__v, __v, 0x0)[0]; + #endif +} +#endif /* vec_xxpermdi */ + +#endif diff --git a/mysys/crc32/crc32_arm64.c b/mysys/crc32/crc32_arm64.c new file mode 100644 index 00000000000..0e70c21812a --- /dev/null +++ b/mysys/crc32/crc32_arm64.c @@ -0,0 +1,340 @@ +#include <my_global.h> +#include <string.h> +#include <stdint.h> + +static int pmull_supported; + +#if defined(HAVE_ARMV8_CRC) + +#if defined(__APPLE__) +#include <sys/sysctl.h> + +int crc32_aarch64_available(void) +{ + int ret; + size_t len = sizeof(ret); + if (sysctlbyname("hw.optional.armv8_crc32", &ret, &len, NULL, 0) == -1) + return 0; + return ret; +} + +const char *crc32c_aarch64_available(void) +{ + if (crc32_aarch64_available() == 0) + return NULL; + pmull_supported = 1; + return "Using ARMv8 crc32 + pmull instructions"; +} + +#else +#include <sys/auxv.h> +#if defined(__FreeBSD__) +static unsigned long getauxval(unsigned int key) +{ + unsigned long val; + if (elf_aux_info(key, (void *)&val, (int)sizeof(val) != 0) + return 0ul; + return val; +} +#else +# include <asm/hwcap.h> +#endif + +#ifndef HWCAP_CRC32 +# define HWCAP_CRC32 (1 << 7) +#endif + +#ifndef HWCAP_PMULL +# define HWCAP_PMULL (1 << 4) +#endif + +/* ARM made crc32 default from ARMv8.1 but optional in ARMv8A + * Runtime check API. + */ +int crc32_aarch64_available(void) +{ + unsigned long auxv= getauxval(AT_HWCAP); + return (auxv & HWCAP_CRC32) != 0; +} + +const char *crc32c_aarch64_available(void) +{ + unsigned long auxv= getauxval(AT_HWCAP); + + if (!(auxv & HWCAP_CRC32)) + return NULL; + + pmull_supported= (auxv & HWCAP_PMULL) != 0; + if (pmull_supported) + return "Using ARMv8 crc32 + pmull instructions"; + else + return "Using ARMv8 crc32 instructions"; +} + +#endif /* __APPLE__ */ +#endif /* HAVE_ARMV8_CRC */ + +#ifndef HAVE_ARMV8_CRC_CRYPTO_INTRINSICS + +/* Request crc extension capabilities from the assembler */ +asm(".arch_extension crc"); + +# ifdef HAVE_ARMV8_CRYPTO +/* crypto extension */ +asm(".arch_extension crypto"); +# endif + +#define CRC32CX(crc, value) __asm__("crc32cx %w[c], %w[c], %x[v]":[c]"+r"(crc):[v]"r"(value)) +#define CRC32CW(crc, value) __asm__("crc32cw %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value)) +#define CRC32CH(crc, value) __asm__("crc32ch %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value)) +#define CRC32CB(crc, value) __asm__("crc32cb %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value)) + +#define CRC32X(crc, value) __asm__("crc32x %w[c], %w[c], %x[v]":[c]"+r"(crc):[v]"r"(value)) +#define CRC32W(crc, value) __asm__("crc32w %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value)) +#define CRC32H(crc, value) __asm__("crc32h %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value)) +#define CRC32B(crc, value) __asm__("crc32b %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value)) + + +#define CRC32C3X8(buffer, ITR) \ + __asm__("crc32cx %w[c1], %w[c1], %x[v]":[c1]"+r"(crc1):[v]"r"(*((const uint64_t *)buffer + 42*1 + (ITR))));\ + __asm__("crc32cx %w[c2], %w[c2], %x[v]":[c2]"+r"(crc2):[v]"r"(*((const uint64_t *)buffer + 42*2 + (ITR))));\ + __asm__("crc32cx %w[c0], %w[c0], %x[v]":[c0]"+r"(crc0):[v]"r"(*((const uint64_t *)buffer + 42*0 + (ITR)))); + +#else /* HAVE_ARMV8_CRC_CRYPTO_INTRINSICS */ + +/* Intrinsics header*/ +#include <arm_acle.h> +#include <arm_neon.h> + +#define CRC32CX(crc, value) (crc) = __crc32cd((crc), (value)) +#define CRC32CW(crc, value) (crc) = __crc32cw((crc), (value)) +#define CRC32CH(crc, value) (crc) = __crc32ch((crc), (value)) +#define CRC32CB(crc, value) (crc) = __crc32cb((crc), (value)) + +#define CRC32X(crc, value) (crc) = __crc32d((crc), (value)) +#define CRC32W(crc, value) (crc) = __crc32w((crc), (value)) +#define CRC32H(crc, value) (crc) = __crc32h((crc), (value)) +#define CRC32B(crc, value) (crc) = __crc32b((crc), (value)) + +#define CRC32C3X8(buffer, ITR) \ + crc1 = __crc32cd(crc1, *((const uint64_t *)buffer + 42*1 + (ITR)));\ + crc2 = __crc32cd(crc2, *((const uint64_t *)buffer + 42*2 + (ITR)));\ + crc0 = __crc32cd(crc0, *((const uint64_t *)buffer + 42*0 + (ITR))); + +#endif /* HAVE_ARMV8_CRC_CRYPTO_INTRINSICS */ + +#define CRC32C7X3X8(buffer, ITR) do {\ + CRC32C3X8(buffer, ((ITR) * 7 + 0)) \ + CRC32C3X8(buffer, ((ITR) * 7 + 1)) \ + CRC32C3X8(buffer, ((ITR) * 7 + 2)) \ + CRC32C3X8(buffer, ((ITR) * 7 + 3)) \ + CRC32C3X8(buffer, ((ITR) * 7 + 4)) \ + CRC32C3X8(buffer, ((ITR) * 7 + 5)) \ + CRC32C3X8(buffer, ((ITR) * 7 + 6)) \ +} while(0) + +#define PREF4X64L1(buffer, PREF_OFFSET, ITR) \ + __asm__("PRFM PLDL1KEEP, [%x[v],%[c]]"::[v]"r"(buffer), [c]"I"((PREF_OFFSET) + ((ITR) + 0)*64));\ + __asm__("PRFM PLDL1KEEP, [%x[v],%[c]]"::[v]"r"(buffer), [c]"I"((PREF_OFFSET) + ((ITR) + 1)*64));\ + __asm__("PRFM PLDL1KEEP, [%x[v],%[c]]"::[v]"r"(buffer), [c]"I"((PREF_OFFSET) + ((ITR) + 2)*64));\ + __asm__("PRFM PLDL1KEEP, [%x[v],%[c]]"::[v]"r"(buffer), [c]"I"((PREF_OFFSET) + ((ITR) + 3)*64)); + +#define PREF1KL1(buffer, PREF_OFFSET) \ + PREF4X64L1(buffer,(PREF_OFFSET), 0) \ + PREF4X64L1(buffer,(PREF_OFFSET), 4) \ + PREF4X64L1(buffer,(PREF_OFFSET), 8) \ + PREF4X64L1(buffer,(PREF_OFFSET), 12) + +#define PREF4X64L2(buffer, PREF_OFFSET, ITR) \ + __asm__("PRFM PLDL2KEEP, [%x[v],%[c]]"::[v]"r"(buffer), [c]"I"((PREF_OFFSET) + ((ITR) + 0)*64));\ + __asm__("PRFM PLDL2KEEP, [%x[v],%[c]]"::[v]"r"(buffer), [c]"I"((PREF_OFFSET) + ((ITR) + 1)*64));\ + __asm__("PRFM PLDL2KEEP, [%x[v],%[c]]"::[v]"r"(buffer), [c]"I"((PREF_OFFSET) + ((ITR) + 2)*64));\ + __asm__("PRFM PLDL2KEEP, [%x[v],%[c]]"::[v]"r"(buffer), [c]"I"((PREF_OFFSET) + ((ITR) + 3)*64)); + +#define PREF1KL2(buffer, PREF_OFFSET) \ + PREF4X64L2(buffer,(PREF_OFFSET), 0) \ + PREF4X64L2(buffer,(PREF_OFFSET), 4) \ + PREF4X64L2(buffer,(PREF_OFFSET), 8) \ + PREF4X64L2(buffer,(PREF_OFFSET), 12) + +uint32_t crc32c_aarch64(uint32_t crc, const unsigned char *buffer, uint64_t len) +{ + uint32_t crc0, crc1, crc2; + int64_t length= (int64_t)len; + + crc^= 0xffffffff; + + /* Pmull runtime check here. + * Raspberry Pi 4 supports crc32 but doesn't support pmull (MDEV-23030). + * + * Consider the condition that the target platform does support hardware crc32 + * but not support PMULL. In this condition, it should leverage the aarch64 + * crc32 instruction (__crc32c) and just only skip parallel computation (pmull/vmull) + * rather than skip all hardware crc32 instruction of computation. + */ + if (pmull_supported) + { +/* The following Macro (HAVE_ARMV8_CRYPTO) is used for compiling check */ +#ifdef HAVE_ARMV8_CRYPTO + +/* Crypto extension Support + * Parallel computation with 1024 Bytes (per block) + * Intrinsics Support + */ +# ifdef HAVE_ARMV8_CRC_CRYPTO_INTRINSICS + const poly64_t k1= 0xe417f38a, k2= 0x8f158014; + uint64_t t0, t1; + + /* Process per block size of 1024 Bytes + * A block size = 8 + 42*3*sizeof(uint64_t) + 8 + */ + while ((length-= 1024) >= 0) + { + /* Prefetch 3*1024 data for avoiding L2 cache miss */ + PREF1KL2(buffer, 1024*3); + /* Do first 8 bytes here for better pipelining */ + crc0= __crc32cd(crc, *(const uint64_t *)buffer); + crc1= 0; + crc2= 0; + buffer+= sizeof(uint64_t); + + /* Process block inline + * Process crc0 last to avoid dependency with above + */ + CRC32C7X3X8(buffer, 0); + CRC32C7X3X8(buffer, 1); + CRC32C7X3X8(buffer, 2); + CRC32C7X3X8(buffer, 3); + CRC32C7X3X8(buffer, 4); + CRC32C7X3X8(buffer, 5); + + buffer+= 42*3*sizeof(uint64_t); + /* Prefetch data for following block to avoid L1 cache miss */ + PREF1KL1(buffer, 1024); + + /* Last 8 bytes + * Merge crc0 and crc1 into crc2 + * crc1 multiply by K2 + * crc0 multiply by K1 + */ + t1= (uint64_t)vmull_p64(crc1, k2); + t0= (uint64_t)vmull_p64(crc0, k1); + crc= __crc32cd(crc2, *(const uint64_t *)buffer); + crc1= __crc32cd(0, t1); + crc^= crc1; + crc0= __crc32cd(0, t0); + crc^= crc0; + + buffer+= sizeof(uint64_t); + } + +# else /* HAVE_ARMV8_CRC_CRYPTO_INTRINSICS */ + + /*No intrinsics*/ + __asm__("mov x16, #0xf38a \n\t" + "movk x16, #0xe417, lsl 16 \n\t" + "mov v1.2d[0], x16 \n\t" + "mov x16, #0x8014 \n\t" + "movk x16, #0x8f15, lsl 16 \n\t" + "mov v0.2d[0], x16 \n\t" + :::"x16"); + + while ((length-= 1024) >= 0) + { + PREF1KL2(buffer, 1024*3); + __asm__("crc32cx %w[c0], %w[c], %x[v]\n\t" + :[c0]"=r"(crc0):[c]"r"(crc), [v]"r"(*(const uint64_t *)buffer):); + crc1= 0; + crc2= 0; + buffer+= sizeof(uint64_t); + + CRC32C7X3X8(buffer, 0); + CRC32C7X3X8(buffer, 1); + CRC32C7X3X8(buffer, 2); + CRC32C7X3X8(buffer, 3); + CRC32C7X3X8(buffer, 4); + CRC32C7X3X8(buffer, 5); + + buffer+= 42*3*sizeof(uint64_t); + PREF1KL1(buffer, 1024); + __asm__("mov v2.2d[0], %x[c1] \n\t" + "pmull v2.1q, v2.1d, v0.1d \n\t" + "mov v3.2d[0], %x[c0] \n\t" + "pmull v3.1q, v3.1d, v1.1d \n\t" + "crc32cx %w[c], %w[c2], %x[v] \n\t" + "mov %x[c1], v2.2d[0] \n\t" + "crc32cx %w[c1], wzr, %x[c1] \n\t" + "eor %w[c], %w[c], %w[c1] \n\t" + "mov %x[c0], v3.2d[0] \n\t" + "crc32cx %w[c0], wzr, %x[c0] \n\t" + "eor %w[c], %w[c], %w[c0] \n\t" + :[c1]"+r"(crc1), [c0]"+r"(crc0), [c2]"+r"(crc2), [c]"+r"(crc) + :[v]"r"(*((const uint64_t *)buffer))); + buffer+= sizeof(uint64_t); + } +# endif /* HAVE_ARMV8_CRC_CRYPTO_INTRINSICS */ + + /* Done if Input data size is aligned with 1024 */ + if (!(length+= 1024)) + return ~crc; + +#endif /* HAVE_ARMV8_CRYPTO */ + + } // end if pmull_supported + + while ((length-= sizeof(uint64_t)) >= 0) + { + CRC32CX(crc, *(uint64_t *)buffer); + buffer+= sizeof(uint64_t); + } + + /* The following is more efficient than the straight loop */ + if (length & sizeof(uint32_t)) + { + CRC32CW(crc, *(uint32_t *)buffer); + buffer+= sizeof(uint32_t); + } + + if (length & sizeof(uint16_t)) + { + CRC32CH(crc, *(uint16_t *)buffer); + buffer+= sizeof(uint16_t); + } + + if (length & sizeof(uint8_t)) + CRC32CB(crc, *buffer); + + return ~crc; +} + +/* There are multiple approaches to calculate crc. +Approach-1: Process 8 bytes then 4 bytes then 2 bytes and then 1 bytes +Approach-2: Process 8 bytes and remaining workload using 1 bytes +Apporach-3: Process 64 bytes at once by issuing 8 crc call and remaining + using 8/1 combination. + +Based on micro-benchmark testing we found that Approach-2 works best especially +given small chunk of variable data. */ +unsigned int crc32_aarch64(unsigned int crc, const void *buf, size_t len) +{ + const uint8_t *buf1= buf; + const uint64_t *buf8= (const uint64_t *) (((uintptr_t) buf + 7) & ~7); + + crc= ~crc; + + /* if start pointer is not 8 bytes aligned */ + while ((buf1 != (const uint8_t *) buf8) && len) + { + CRC32B(crc, *buf1++); + len--; + } + + for (; len >= 8; len-= 8) + CRC32X(crc, *buf8++); + + buf1= (const uint8_t *) buf8; + while (len--) + CRC32B(crc, *buf1++); + + return ~crc; +} diff --git a/mysys/crc32/crc32_ppc64.c b/mysys/crc32/crc32_ppc64.c new file mode 100644 index 00000000000..76df88ee231 --- /dev/null +++ b/mysys/crc32/crc32_ppc64.c @@ -0,0 +1,5 @@ +#define CRC32_FUNCTION my_checksum +#define CRC_TABLE +#define POWER8_INTRINSICS +#include "pcc_crc32_constants.h" +#include "crc_ppc64.h" diff --git a/mysys/crc32/crc32_x86.c b/mysys/crc32/crc32_x86.c new file mode 100644 index 00000000000..f077399caca --- /dev/null +++ b/mysys/crc32/crc32_x86.c @@ -0,0 +1,334 @@ +/* Copyright (c) 2020, 2021, MariaDB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* + Implementation of CRC32 (Ethernet) uing Intel PCLMULQDQ + Ported from Intels work, see https://github.com/intel/soft-crc +*/ + +/******************************************************************************* + Copyright (c) 2009-2018, Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + + +#include <my_global.h> +#include <my_compiler.h> + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <stdint.h> +#include <stddef.h> + +#ifdef __GNUC__ +#include <x86intrin.h> +#elif defined(_MSC_VER) +#include <intrin.h> +#else +#error "unknown compiler" +#endif + +/** + * @brief Shifts left 128 bit register by specified number of bytes + * + * @param reg 128 bit value + * @param num number of bytes to shift left \a reg by (0-16) + * + * @return \a reg << (\a num * 8) + */ +static inline __m128i xmm_shift_left(__m128i reg, const unsigned int num) +{ + static const MY_ALIGNED(16) uint8_t crc_xmm_shift_tab[48]= { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}; + + const __m128i *p= (const __m128i *) (crc_xmm_shift_tab + 16 - num); + + return _mm_shuffle_epi8(reg, _mm_loadu_si128(p)); +} + +struct crcr_pclmulqdq_ctx +{ + uint64_t rk1; + uint64_t rk2; + uint64_t rk5; + uint64_t rk6; + uint64_t rk7; + uint64_t rk8; +}; + +/** + * @brief Performs one folding round + * + * Logically function operates as follows: + * DATA = READ_NEXT_16BYTES(); + * F1 = LSB8(FOLD) + * F2 = MSB8(FOLD) + * T1 = CLMUL(F1, RK1) + * T2 = CLMUL(F2, RK2) + * FOLD = XOR(T1, T2, DATA) + * + * @param data_block 16 byte data block + * @param precomp precomputed rk1 constanst + * @param fold running 16 byte folded data + * + * @return New 16 byte folded data + */ +static inline __m128i crcr32_folding_round(const __m128i data_block, + const __m128i precomp, const __m128i fold) +{ + __m128i tmp0= _mm_clmulepi64_si128(fold, precomp, 0x01); + __m128i tmp1= _mm_clmulepi64_si128(fold, precomp, 0x10); + + return _mm_xor_si128(tmp1, _mm_xor_si128(data_block, tmp0)); +} + +/** + * @brief Performs reduction from 128 bits to 64 bits + * + * @param data128 128 bits data to be reduced + * @param precomp rk5 and rk6 precomputed constants + * + * @return data reduced to 64 bits + */ +static inline __m128i crcr32_reduce_128_to_64(__m128i data128, const __m128i precomp) +{ + __m128i tmp0, tmp1, tmp2; + + /* 64b fold */ + tmp0= _mm_clmulepi64_si128(data128, precomp, 0x00); + tmp1= _mm_srli_si128(data128, 8); + tmp0= _mm_xor_si128(tmp0, tmp1); + + /* 32b fold */ + tmp2= _mm_slli_si128(tmp0, 4); + tmp1= _mm_clmulepi64_si128(tmp2, precomp, 0x10); + + return _mm_xor_si128(tmp1, tmp0); +} + +/** + * @brief Performs Barret's reduction from 64 bits to 32 bits + * + * @param data64 64 bits data to be reduced + * @param precomp rk7 precomputed constant + * + * @return data reduced to 32 bits + */ +static inline uint32_t crcr32_reduce_64_to_32(__m128i data64, const __m128i precomp) +{ + static const MY_ALIGNED(16) uint32_t mask1[4]= { + 0xffffffff, 0xffffffff, 0x00000000, 0x00000000}; + static const MY_ALIGNED(16) uint32_t mask2[4]= { + 0x00000000, 0xffffffff, 0xffffffff, 0xffffffff}; + __m128i tmp0, tmp1, tmp2; + + tmp0= _mm_and_si128(data64, _mm_load_si128((__m128i *) mask2)); + + tmp1= _mm_clmulepi64_si128(tmp0, precomp, 0x00); + tmp1= _mm_xor_si128(tmp1, tmp0); + tmp1= _mm_and_si128(tmp1, _mm_load_si128((__m128i *) mask1)); + + tmp2= _mm_clmulepi64_si128(tmp1, precomp, 0x10); + tmp2= _mm_xor_si128(tmp2, tmp1); + tmp2= _mm_xor_si128(tmp2, tmp0); + + return _mm_extract_epi32(tmp2, 2); +} + +/** + * @brief Calculates reflected 32-bit CRC for given \a data block + * by applying folding and reduction methods. + * + * Algorithm operates on 32 bit CRCs. + * Polynomials and initial values may need to be promoted to + * 32 bits where required. + * + * @param crc initial CRC value (32 bit value) + * @param data pointer to data block + * @param data_len length of \a data block in bytes + * @param params pointer to PCLMULQDQ CRC calculation context + * + * @return CRC for given \a data block (32 bits wide). + */ +static inline uint32_t crcr32_calc_pclmulqdq(const uint8_t *data, uint32_t data_len, + uint32_t crc, + const struct crcr_pclmulqdq_ctx *params) +{ + __m128i temp, fold, k; + uint32_t n; + + DBUG_ASSERT(data != NULL || data_len == 0); + DBUG_ASSERT(params); + + if (unlikely(data_len == 0)) + return crc; + + /** + * Get CRC init value + */ + temp= _mm_insert_epi32(_mm_setzero_si128(), crc, 0); + + /** + * ------------------------------------------------- + * Folding all data into single 16 byte data block + * Assumes: \a fold holds first 16 bytes of data + */ + + if (unlikely(data_len < 32)) + { + if (unlikely(data_len == 16)) + { + /* 16 bytes */ + fold= _mm_loadu_si128((__m128i *) data); + fold= _mm_xor_si128(fold, temp); + goto reduction_128_64; + } + if (unlikely(data_len < 16)) + { + /* 0 to 15 bytes */ + MY_ALIGNED(16) uint8_t buffer[16]; + + memset(buffer, 0, sizeof(buffer)); + memcpy(buffer, data, data_len); + + fold= _mm_load_si128((__m128i *) buffer); + fold= _mm_xor_si128(fold, temp); + if ((data_len < 4)) + { + fold= xmm_shift_left(fold, 8 - data_len); + goto barret_reduction; + } + fold= xmm_shift_left(fold, 16 - data_len); + goto reduction_128_64; + } + /* 17 to 31 bytes */ + fold= _mm_loadu_si128((__m128i *) data); + fold= _mm_xor_si128(fold, temp); + n= 16; + k= _mm_load_si128((__m128i *) (¶ms->rk1)); + goto partial_bytes; + } + + /** + * At least 32 bytes in the buffer + */ + + /** + * Apply CRC initial value + */ + fold= _mm_loadu_si128((const __m128i *) data); + fold= _mm_xor_si128(fold, temp); + + /** + * Main folding loop + * - the last 16 bytes is processed separately + */ + k= _mm_load_si128((__m128i *) (¶ms->rk1)); + for (n= 16; (n + 16) <= data_len; n+= 16) + { + temp= _mm_loadu_si128((__m128i *) &data[n]); + fold= crcr32_folding_round(temp, k, fold); + } + +partial_bytes: + if (likely(n < data_len)) + { + static const MY_ALIGNED(16) uint32_t mask3[4]= {0x80808080, 0x80808080, + 0x80808080, 0x80808080}; + static const MY_ALIGNED(16) uint8_t shf_table[32]= { + 0x00, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, + 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, + 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f}; + __m128i last16, a, b; + + last16= _mm_loadu_si128((const __m128i *) &data[data_len - 16]); + + temp= _mm_loadu_si128((const __m128i *) &shf_table[data_len & 15]); + a= _mm_shuffle_epi8(fold, temp); + + temp= _mm_xor_si128(temp, _mm_load_si128((const __m128i *) mask3)); + b= _mm_shuffle_epi8(fold, temp); + b= _mm_blendv_epi8(b, last16, temp); + + /* k = rk1 & rk2 */ + temp= _mm_clmulepi64_si128(a, k, 0x01); + fold= _mm_clmulepi64_si128(a, k, 0x10); + + fold= _mm_xor_si128(fold, temp); + fold= _mm_xor_si128(fold, b); + } + + /** + * ------------------------------------------------- + * Reduction 128 -> 32 + * Assumes: \a fold holds 128bit folded data + */ +reduction_128_64: + k= _mm_load_si128((__m128i *) (¶ms->rk5)); + fold= crcr32_reduce_128_to_64(fold, k); + +barret_reduction: + k= _mm_load_si128((__m128i *) (¶ms->rk7)); + n= crcr32_reduce_64_to_32(fold, k); + return n; +} + +static const MY_ALIGNED(16) struct crcr_pclmulqdq_ctx ether_crc32_clmul= { + 0xccaa009e, /**< rk1 */ + 0x1751997d0, /**< rk2 */ + 0xccaa009e, /**< rk5 */ + 0x163cd6124, /**< rk6 */ + 0x1f7011640, /**< rk7 */ + 0x1db710641 /**< rk8 */ +}; + +/** + * @brief Calculates Ethernet CRC32 using PCLMULQDQ method. + * + * @param data pointer to data block to calculate CRC for + * @param data_len size of data block + * + * @return New CRC value + */ +unsigned int crc32_pclmul(unsigned int crc32, const void *buf, size_t len) +{ + return ~crcr32_calc_pclmulqdq(buf, (uint32_t)len, ~crc32, ðer_crc32_clmul); +} diff --git a/mysys/crc32/crc32c.cc b/mysys/crc32/crc32c.cc new file mode 100644 index 00000000000..b48e744a663 --- /dev/null +++ b/mysys/crc32/crc32c.cc @@ -0,0 +1,598 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// A portable implementation of crc32c, optimized to handle +// four bytes at a time. + +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + + +#include <stddef.h> +#include <stdint.h> +#include <string> +#include <my_global.h> +#include <my_byteorder.h> +static inline uint32_t DecodeFixed32(const char *ptr) +{ + return uint4korr(ptr); +} + +#include <stdint.h> +#ifdef _MSC_VER +#include <intrin.h> +#endif + +#ifdef HAVE_SSE42 +# ifdef __GNUC__ +# include <cpuid.h> +# if __GNUC__ < 5 && !defined __clang__ +/* the headers do not really work in GCC before version 5 */ +# define _mm_crc32_u8(crc,data) __builtin_ia32_crc32qi(crc,data) +# define _mm_crc32_u32(crc,data) __builtin_ia32_crc32si(crc,data) +# define _mm_crc32_u64(crc,data) __builtin_ia32_crc32di(crc,data) +# else +# include <nmmintrin.h> +# endif +# define USE_SSE42 __attribute__((target("sse4.2"))) +# else +# define USE_SSE42 /* nothing */ +# endif +#endif + + +#ifdef __powerpc64__ +#include "crc32c_ppc.h" + +#if __linux__ +#include <sys/auxv.h> + +#ifndef PPC_FEATURE2_VEC_CRYPTO +#define PPC_FEATURE2_VEC_CRYPTO 0x02000000 +#endif + +#ifndef AT_HWCAP2 +#define AT_HWCAP2 26 +#endif + +#endif /* __linux__ */ + +#endif + +namespace mysys_namespace { +namespace crc32c { + +#if defined(HAVE_POWER8) && defined(HAS_ALTIVEC) +#ifdef __powerpc64__ +static int arch_ppc_crc32 = 0; +#endif /* __powerpc64__ */ +#endif + +static const uint32_t table0_[256] = { + 0x00000000, 0xf26b8303, 0xe13b70f7, 0x1350f3f4, + 0xc79a971f, 0x35f1141c, 0x26a1e7e8, 0xd4ca64eb, + 0x8ad958cf, 0x78b2dbcc, 0x6be22838, 0x9989ab3b, + 0x4d43cfd0, 0xbf284cd3, 0xac78bf27, 0x5e133c24, + 0x105ec76f, 0xe235446c, 0xf165b798, 0x030e349b, + 0xd7c45070, 0x25afd373, 0x36ff2087, 0xc494a384, + 0x9a879fa0, 0x68ec1ca3, 0x7bbcef57, 0x89d76c54, + 0x5d1d08bf, 0xaf768bbc, 0xbc267848, 0x4e4dfb4b, + 0x20bd8ede, 0xd2d60ddd, 0xc186fe29, 0x33ed7d2a, + 0xe72719c1, 0x154c9ac2, 0x061c6936, 0xf477ea35, + 0xaa64d611, 0x580f5512, 0x4b5fa6e6, 0xb93425e5, + 0x6dfe410e, 0x9f95c20d, 0x8cc531f9, 0x7eaeb2fa, + 0x30e349b1, 0xc288cab2, 0xd1d83946, 0x23b3ba45, + 0xf779deae, 0x05125dad, 0x1642ae59, 0xe4292d5a, + 0xba3a117e, 0x4851927d, 0x5b016189, 0xa96ae28a, + 0x7da08661, 0x8fcb0562, 0x9c9bf696, 0x6ef07595, + 0x417b1dbc, 0xb3109ebf, 0xa0406d4b, 0x522bee48, + 0x86e18aa3, 0x748a09a0, 0x67dafa54, 0x95b17957, + 0xcba24573, 0x39c9c670, 0x2a993584, 0xd8f2b687, + 0x0c38d26c, 0xfe53516f, 0xed03a29b, 0x1f682198, + 0x5125dad3, 0xa34e59d0, 0xb01eaa24, 0x42752927, + 0x96bf4dcc, 0x64d4cecf, 0x77843d3b, 0x85efbe38, + 0xdbfc821c, 0x2997011f, 0x3ac7f2eb, 0xc8ac71e8, + 0x1c661503, 0xee0d9600, 0xfd5d65f4, 0x0f36e6f7, + 0x61c69362, 0x93ad1061, 0x80fde395, 0x72966096, + 0xa65c047d, 0x5437877e, 0x4767748a, 0xb50cf789, + 0xeb1fcbad, 0x197448ae, 0x0a24bb5a, 0xf84f3859, + 0x2c855cb2, 0xdeeedfb1, 0xcdbe2c45, 0x3fd5af46, + 0x7198540d, 0x83f3d70e, 0x90a324fa, 0x62c8a7f9, + 0xb602c312, 0x44694011, 0x5739b3e5, 0xa55230e6, + 0xfb410cc2, 0x092a8fc1, 0x1a7a7c35, 0xe811ff36, + 0x3cdb9bdd, 0xceb018de, 0xdde0eb2a, 0x2f8b6829, + 0x82f63b78, 0x709db87b, 0x63cd4b8f, 0x91a6c88c, + 0x456cac67, 0xb7072f64, 0xa457dc90, 0x563c5f93, + 0x082f63b7, 0xfa44e0b4, 0xe9141340, 0x1b7f9043, + 0xcfb5f4a8, 0x3dde77ab, 0x2e8e845f, 0xdce5075c, + 0x92a8fc17, 0x60c37f14, 0x73938ce0, 0x81f80fe3, + 0x55326b08, 0xa759e80b, 0xb4091bff, 0x466298fc, + 0x1871a4d8, 0xea1a27db, 0xf94ad42f, 0x0b21572c, + 0xdfeb33c7, 0x2d80b0c4, 0x3ed04330, 0xccbbc033, + 0xa24bb5a6, 0x502036a5, 0x4370c551, 0xb11b4652, + 0x65d122b9, 0x97baa1ba, 0x84ea524e, 0x7681d14d, + 0x2892ed69, 0xdaf96e6a, 0xc9a99d9e, 0x3bc21e9d, + 0xef087a76, 0x1d63f975, 0x0e330a81, 0xfc588982, + 0xb21572c9, 0x407ef1ca, 0x532e023e, 0xa145813d, + 0x758fe5d6, 0x87e466d5, 0x94b49521, 0x66df1622, + 0x38cc2a06, 0xcaa7a905, 0xd9f75af1, 0x2b9cd9f2, + 0xff56bd19, 0x0d3d3e1a, 0x1e6dcdee, 0xec064eed, + 0xc38d26c4, 0x31e6a5c7, 0x22b65633, 0xd0ddd530, + 0x0417b1db, 0xf67c32d8, 0xe52cc12c, 0x1747422f, + 0x49547e0b, 0xbb3ffd08, 0xa86f0efc, 0x5a048dff, + 0x8ecee914, 0x7ca56a17, 0x6ff599e3, 0x9d9e1ae0, + 0xd3d3e1ab, 0x21b862a8, 0x32e8915c, 0xc083125f, + 0x144976b4, 0xe622f5b7, 0xf5720643, 0x07198540, + 0x590ab964, 0xab613a67, 0xb831c993, 0x4a5a4a90, + 0x9e902e7b, 0x6cfbad78, 0x7fab5e8c, 0x8dc0dd8f, + 0xe330a81a, 0x115b2b19, 0x020bd8ed, 0xf0605bee, + 0x24aa3f05, 0xd6c1bc06, 0xc5914ff2, 0x37faccf1, + 0x69e9f0d5, 0x9b8273d6, 0x88d28022, 0x7ab90321, + 0xae7367ca, 0x5c18e4c9, 0x4f48173d, 0xbd23943e, + 0xf36e6f75, 0x0105ec76, 0x12551f82, 0xe03e9c81, + 0x34f4f86a, 0xc69f7b69, 0xd5cf889d, 0x27a40b9e, + 0x79b737ba, 0x8bdcb4b9, 0x988c474d, 0x6ae7c44e, + 0xbe2da0a5, 0x4c4623a6, 0x5f16d052, 0xad7d5351 +}; +static const uint32_t table1_[256] = { + 0x00000000, 0x13a29877, 0x274530ee, 0x34e7a899, + 0x4e8a61dc, 0x5d28f9ab, 0x69cf5132, 0x7a6dc945, + 0x9d14c3b8, 0x8eb65bcf, 0xba51f356, 0xa9f36b21, + 0xd39ea264, 0xc03c3a13, 0xf4db928a, 0xe7790afd, + 0x3fc5f181, 0x2c6769f6, 0x1880c16f, 0x0b225918, + 0x714f905d, 0x62ed082a, 0x560aa0b3, 0x45a838c4, + 0xa2d13239, 0xb173aa4e, 0x859402d7, 0x96369aa0, + 0xec5b53e5, 0xfff9cb92, 0xcb1e630b, 0xd8bcfb7c, + 0x7f8be302, 0x6c297b75, 0x58ced3ec, 0x4b6c4b9b, + 0x310182de, 0x22a31aa9, 0x1644b230, 0x05e62a47, + 0xe29f20ba, 0xf13db8cd, 0xc5da1054, 0xd6788823, + 0xac154166, 0xbfb7d911, 0x8b507188, 0x98f2e9ff, + 0x404e1283, 0x53ec8af4, 0x670b226d, 0x74a9ba1a, + 0x0ec4735f, 0x1d66eb28, 0x298143b1, 0x3a23dbc6, + 0xdd5ad13b, 0xcef8494c, 0xfa1fe1d5, 0xe9bd79a2, + 0x93d0b0e7, 0x80722890, 0xb4958009, 0xa737187e, + 0xff17c604, 0xecb55e73, 0xd852f6ea, 0xcbf06e9d, + 0xb19da7d8, 0xa23f3faf, 0x96d89736, 0x857a0f41, + 0x620305bc, 0x71a19dcb, 0x45463552, 0x56e4ad25, + 0x2c896460, 0x3f2bfc17, 0x0bcc548e, 0x186eccf9, + 0xc0d23785, 0xd370aff2, 0xe797076b, 0xf4359f1c, + 0x8e585659, 0x9dface2e, 0xa91d66b7, 0xbabffec0, + 0x5dc6f43d, 0x4e646c4a, 0x7a83c4d3, 0x69215ca4, + 0x134c95e1, 0x00ee0d96, 0x3409a50f, 0x27ab3d78, + 0x809c2506, 0x933ebd71, 0xa7d915e8, 0xb47b8d9f, + 0xce1644da, 0xddb4dcad, 0xe9537434, 0xfaf1ec43, + 0x1d88e6be, 0x0e2a7ec9, 0x3acdd650, 0x296f4e27, + 0x53028762, 0x40a01f15, 0x7447b78c, 0x67e52ffb, + 0xbf59d487, 0xacfb4cf0, 0x981ce469, 0x8bbe7c1e, + 0xf1d3b55b, 0xe2712d2c, 0xd69685b5, 0xc5341dc2, + 0x224d173f, 0x31ef8f48, 0x050827d1, 0x16aabfa6, + 0x6cc776e3, 0x7f65ee94, 0x4b82460d, 0x5820de7a, + 0xfbc3faf9, 0xe861628e, 0xdc86ca17, 0xcf245260, + 0xb5499b25, 0xa6eb0352, 0x920cabcb, 0x81ae33bc, + 0x66d73941, 0x7575a136, 0x419209af, 0x523091d8, + 0x285d589d, 0x3bffc0ea, 0x0f186873, 0x1cbaf004, + 0xc4060b78, 0xd7a4930f, 0xe3433b96, 0xf0e1a3e1, + 0x8a8c6aa4, 0x992ef2d3, 0xadc95a4a, 0xbe6bc23d, + 0x5912c8c0, 0x4ab050b7, 0x7e57f82e, 0x6df56059, + 0x1798a91c, 0x043a316b, 0x30dd99f2, 0x237f0185, + 0x844819fb, 0x97ea818c, 0xa30d2915, 0xb0afb162, + 0xcac27827, 0xd960e050, 0xed8748c9, 0xfe25d0be, + 0x195cda43, 0x0afe4234, 0x3e19eaad, 0x2dbb72da, + 0x57d6bb9f, 0x447423e8, 0x70938b71, 0x63311306, + 0xbb8de87a, 0xa82f700d, 0x9cc8d894, 0x8f6a40e3, + 0xf50789a6, 0xe6a511d1, 0xd242b948, 0xc1e0213f, + 0x26992bc2, 0x353bb3b5, 0x01dc1b2c, 0x127e835b, + 0x68134a1e, 0x7bb1d269, 0x4f567af0, 0x5cf4e287, + 0x04d43cfd, 0x1776a48a, 0x23910c13, 0x30339464, + 0x4a5e5d21, 0x59fcc556, 0x6d1b6dcf, 0x7eb9f5b8, + 0x99c0ff45, 0x8a626732, 0xbe85cfab, 0xad2757dc, + 0xd74a9e99, 0xc4e806ee, 0xf00fae77, 0xe3ad3600, + 0x3b11cd7c, 0x28b3550b, 0x1c54fd92, 0x0ff665e5, + 0x759baca0, 0x663934d7, 0x52de9c4e, 0x417c0439, + 0xa6050ec4, 0xb5a796b3, 0x81403e2a, 0x92e2a65d, + 0xe88f6f18, 0xfb2df76f, 0xcfca5ff6, 0xdc68c781, + 0x7b5fdfff, 0x68fd4788, 0x5c1aef11, 0x4fb87766, + 0x35d5be23, 0x26772654, 0x12908ecd, 0x013216ba, + 0xe64b1c47, 0xf5e98430, 0xc10e2ca9, 0xd2acb4de, + 0xa8c17d9b, 0xbb63e5ec, 0x8f844d75, 0x9c26d502, + 0x449a2e7e, 0x5738b609, 0x63df1e90, 0x707d86e7, + 0x0a104fa2, 0x19b2d7d5, 0x2d557f4c, 0x3ef7e73b, + 0xd98eedc6, 0xca2c75b1, 0xfecbdd28, 0xed69455f, + 0x97048c1a, 0x84a6146d, 0xb041bcf4, 0xa3e32483 +}; +static const uint32_t table2_[256] = { + 0x00000000, 0xa541927e, 0x4f6f520d, 0xea2ec073, + 0x9edea41a, 0x3b9f3664, 0xd1b1f617, 0x74f06469, + 0x38513ec5, 0x9d10acbb, 0x773e6cc8, 0xd27ffeb6, + 0xa68f9adf, 0x03ce08a1, 0xe9e0c8d2, 0x4ca15aac, + 0x70a27d8a, 0xd5e3eff4, 0x3fcd2f87, 0x9a8cbdf9, + 0xee7cd990, 0x4b3d4bee, 0xa1138b9d, 0x045219e3, + 0x48f3434f, 0xedb2d131, 0x079c1142, 0xa2dd833c, + 0xd62de755, 0x736c752b, 0x9942b558, 0x3c032726, + 0xe144fb14, 0x4405696a, 0xae2ba919, 0x0b6a3b67, + 0x7f9a5f0e, 0xdadbcd70, 0x30f50d03, 0x95b49f7d, + 0xd915c5d1, 0x7c5457af, 0x967a97dc, 0x333b05a2, + 0x47cb61cb, 0xe28af3b5, 0x08a433c6, 0xade5a1b8, + 0x91e6869e, 0x34a714e0, 0xde89d493, 0x7bc846ed, + 0x0f382284, 0xaa79b0fa, 0x40577089, 0xe516e2f7, + 0xa9b7b85b, 0x0cf62a25, 0xe6d8ea56, 0x43997828, + 0x37691c41, 0x92288e3f, 0x78064e4c, 0xdd47dc32, + 0xc76580d9, 0x622412a7, 0x880ad2d4, 0x2d4b40aa, + 0x59bb24c3, 0xfcfab6bd, 0x16d476ce, 0xb395e4b0, + 0xff34be1c, 0x5a752c62, 0xb05bec11, 0x151a7e6f, + 0x61ea1a06, 0xc4ab8878, 0x2e85480b, 0x8bc4da75, + 0xb7c7fd53, 0x12866f2d, 0xf8a8af5e, 0x5de93d20, + 0x29195949, 0x8c58cb37, 0x66760b44, 0xc337993a, + 0x8f96c396, 0x2ad751e8, 0xc0f9919b, 0x65b803e5, + 0x1148678c, 0xb409f5f2, 0x5e273581, 0xfb66a7ff, + 0x26217bcd, 0x8360e9b3, 0x694e29c0, 0xcc0fbbbe, + 0xb8ffdfd7, 0x1dbe4da9, 0xf7908dda, 0x52d11fa4, + 0x1e704508, 0xbb31d776, 0x511f1705, 0xf45e857b, + 0x80aee112, 0x25ef736c, 0xcfc1b31f, 0x6a802161, + 0x56830647, 0xf3c29439, 0x19ec544a, 0xbcadc634, + 0xc85da25d, 0x6d1c3023, 0x8732f050, 0x2273622e, + 0x6ed23882, 0xcb93aafc, 0x21bd6a8f, 0x84fcf8f1, + 0xf00c9c98, 0x554d0ee6, 0xbf63ce95, 0x1a225ceb, + 0x8b277743, 0x2e66e53d, 0xc448254e, 0x6109b730, + 0x15f9d359, 0xb0b84127, 0x5a968154, 0xffd7132a, + 0xb3764986, 0x1637dbf8, 0xfc191b8b, 0x595889f5, + 0x2da8ed9c, 0x88e97fe2, 0x62c7bf91, 0xc7862def, + 0xfb850ac9, 0x5ec498b7, 0xb4ea58c4, 0x11abcaba, + 0x655baed3, 0xc01a3cad, 0x2a34fcde, 0x8f756ea0, + 0xc3d4340c, 0x6695a672, 0x8cbb6601, 0x29faf47f, + 0x5d0a9016, 0xf84b0268, 0x1265c21b, 0xb7245065, + 0x6a638c57, 0xcf221e29, 0x250cde5a, 0x804d4c24, + 0xf4bd284d, 0x51fcba33, 0xbbd27a40, 0x1e93e83e, + 0x5232b292, 0xf77320ec, 0x1d5de09f, 0xb81c72e1, + 0xccec1688, 0x69ad84f6, 0x83834485, 0x26c2d6fb, + 0x1ac1f1dd, 0xbf8063a3, 0x55aea3d0, 0xf0ef31ae, + 0x841f55c7, 0x215ec7b9, 0xcb7007ca, 0x6e3195b4, + 0x2290cf18, 0x87d15d66, 0x6dff9d15, 0xc8be0f6b, + 0xbc4e6b02, 0x190ff97c, 0xf321390f, 0x5660ab71, + 0x4c42f79a, 0xe90365e4, 0x032da597, 0xa66c37e9, + 0xd29c5380, 0x77ddc1fe, 0x9df3018d, 0x38b293f3, + 0x7413c95f, 0xd1525b21, 0x3b7c9b52, 0x9e3d092c, + 0xeacd6d45, 0x4f8cff3b, 0xa5a23f48, 0x00e3ad36, + 0x3ce08a10, 0x99a1186e, 0x738fd81d, 0xd6ce4a63, + 0xa23e2e0a, 0x077fbc74, 0xed517c07, 0x4810ee79, + 0x04b1b4d5, 0xa1f026ab, 0x4bdee6d8, 0xee9f74a6, + 0x9a6f10cf, 0x3f2e82b1, 0xd50042c2, 0x7041d0bc, + 0xad060c8e, 0x08479ef0, 0xe2695e83, 0x4728ccfd, + 0x33d8a894, 0x96993aea, 0x7cb7fa99, 0xd9f668e7, + 0x9557324b, 0x3016a035, 0xda386046, 0x7f79f238, + 0x0b899651, 0xaec8042f, 0x44e6c45c, 0xe1a75622, + 0xdda47104, 0x78e5e37a, 0x92cb2309, 0x378ab177, + 0x437ad51e, 0xe63b4760, 0x0c158713, 0xa954156d, + 0xe5f54fc1, 0x40b4ddbf, 0xaa9a1dcc, 0x0fdb8fb2, + 0x7b2bebdb, 0xde6a79a5, 0x3444b9d6, 0x91052ba8 +}; +static const uint32_t table3_[256] = { + 0x00000000, 0xdd45aab8, 0xbf672381, 0x62228939, + 0x7b2231f3, 0xa6679b4b, 0xc4451272, 0x1900b8ca, + 0xf64463e6, 0x2b01c95e, 0x49234067, 0x9466eadf, + 0x8d665215, 0x5023f8ad, 0x32017194, 0xef44db2c, + 0xe964b13d, 0x34211b85, 0x560392bc, 0x8b463804, + 0x924680ce, 0x4f032a76, 0x2d21a34f, 0xf06409f7, + 0x1f20d2db, 0xc2657863, 0xa047f15a, 0x7d025be2, + 0x6402e328, 0xb9474990, 0xdb65c0a9, 0x06206a11, + 0xd725148b, 0x0a60be33, 0x6842370a, 0xb5079db2, + 0xac072578, 0x71428fc0, 0x136006f9, 0xce25ac41, + 0x2161776d, 0xfc24ddd5, 0x9e0654ec, 0x4343fe54, + 0x5a43469e, 0x8706ec26, 0xe524651f, 0x3861cfa7, + 0x3e41a5b6, 0xe3040f0e, 0x81268637, 0x5c632c8f, + 0x45639445, 0x98263efd, 0xfa04b7c4, 0x27411d7c, + 0xc805c650, 0x15406ce8, 0x7762e5d1, 0xaa274f69, + 0xb327f7a3, 0x6e625d1b, 0x0c40d422, 0xd1057e9a, + 0xaba65fe7, 0x76e3f55f, 0x14c17c66, 0xc984d6de, + 0xd0846e14, 0x0dc1c4ac, 0x6fe34d95, 0xb2a6e72d, + 0x5de23c01, 0x80a796b9, 0xe2851f80, 0x3fc0b538, + 0x26c00df2, 0xfb85a74a, 0x99a72e73, 0x44e284cb, + 0x42c2eeda, 0x9f874462, 0xfda5cd5b, 0x20e067e3, + 0x39e0df29, 0xe4a57591, 0x8687fca8, 0x5bc25610, + 0xb4868d3c, 0x69c32784, 0x0be1aebd, 0xd6a40405, + 0xcfa4bccf, 0x12e11677, 0x70c39f4e, 0xad8635f6, + 0x7c834b6c, 0xa1c6e1d4, 0xc3e468ed, 0x1ea1c255, + 0x07a17a9f, 0xdae4d027, 0xb8c6591e, 0x6583f3a6, + 0x8ac7288a, 0x57828232, 0x35a00b0b, 0xe8e5a1b3, + 0xf1e51979, 0x2ca0b3c1, 0x4e823af8, 0x93c79040, + 0x95e7fa51, 0x48a250e9, 0x2a80d9d0, 0xf7c57368, + 0xeec5cba2, 0x3380611a, 0x51a2e823, 0x8ce7429b, + 0x63a399b7, 0xbee6330f, 0xdcc4ba36, 0x0181108e, + 0x1881a844, 0xc5c402fc, 0xa7e68bc5, 0x7aa3217d, + 0x52a0c93f, 0x8fe56387, 0xedc7eabe, 0x30824006, + 0x2982f8cc, 0xf4c75274, 0x96e5db4d, 0x4ba071f5, + 0xa4e4aad9, 0x79a10061, 0x1b838958, 0xc6c623e0, + 0xdfc69b2a, 0x02833192, 0x60a1b8ab, 0xbde41213, + 0xbbc47802, 0x6681d2ba, 0x04a35b83, 0xd9e6f13b, + 0xc0e649f1, 0x1da3e349, 0x7f816a70, 0xa2c4c0c8, + 0x4d801be4, 0x90c5b15c, 0xf2e73865, 0x2fa292dd, + 0x36a22a17, 0xebe780af, 0x89c50996, 0x5480a32e, + 0x8585ddb4, 0x58c0770c, 0x3ae2fe35, 0xe7a7548d, + 0xfea7ec47, 0x23e246ff, 0x41c0cfc6, 0x9c85657e, + 0x73c1be52, 0xae8414ea, 0xcca69dd3, 0x11e3376b, + 0x08e38fa1, 0xd5a62519, 0xb784ac20, 0x6ac10698, + 0x6ce16c89, 0xb1a4c631, 0xd3864f08, 0x0ec3e5b0, + 0x17c35d7a, 0xca86f7c2, 0xa8a47efb, 0x75e1d443, + 0x9aa50f6f, 0x47e0a5d7, 0x25c22cee, 0xf8878656, + 0xe1873e9c, 0x3cc29424, 0x5ee01d1d, 0x83a5b7a5, + 0xf90696d8, 0x24433c60, 0x4661b559, 0x9b241fe1, + 0x8224a72b, 0x5f610d93, 0x3d4384aa, 0xe0062e12, + 0x0f42f53e, 0xd2075f86, 0xb025d6bf, 0x6d607c07, + 0x7460c4cd, 0xa9256e75, 0xcb07e74c, 0x16424df4, + 0x106227e5, 0xcd278d5d, 0xaf050464, 0x7240aedc, + 0x6b401616, 0xb605bcae, 0xd4273597, 0x09629f2f, + 0xe6264403, 0x3b63eebb, 0x59416782, 0x8404cd3a, + 0x9d0475f0, 0x4041df48, 0x22635671, 0xff26fcc9, + 0x2e238253, 0xf36628eb, 0x9144a1d2, 0x4c010b6a, + 0x5501b3a0, 0x88441918, 0xea669021, 0x37233a99, + 0xd867e1b5, 0x05224b0d, 0x6700c234, 0xba45688c, + 0xa345d046, 0x7e007afe, 0x1c22f3c7, 0xc167597f, + 0xc747336e, 0x1a0299d6, 0x782010ef, 0xa565ba57, + 0xbc65029d, 0x6120a825, 0x0302211c, 0xde478ba4, + 0x31035088, 0xec46fa30, 0x8e647309, 0x5321d9b1, + 0x4a21617b, 0x9764cbc3, 0xf54642fa, 0x2803e842 +}; + +// Used to fetch a naturally-aligned 32-bit word in little endian byte-order +static inline uint32_t LE_LOAD32(const uint8_t *p) { + return DecodeFixed32(reinterpret_cast<const char*>(p)); +} + +static inline void Slow_CRC32(uint64_t* l, uint8_t const **p) +{ + uint32_t c = static_cast<uint32_t>(*l ^ LE_LOAD32(*p)); + *p += 4; + *l = table3_[c & 0xff] ^ + table2_[(c >> 8) & 0xff] ^ + table1_[(c >> 16) & 0xff] ^ + table0_[c >> 24]; + // DO it twice. + c = static_cast<uint32_t>(*l ^ LE_LOAD32(*p)); + *p += 4; + *l = table3_[c & 0xff] ^ + table2_[(c >> 8) & 0xff] ^ + table1_[(c >> 16) & 0xff] ^ + table0_[c >> 24]; +} + +#ifdef ALIGN +#undef ALIGN +#endif + +// Align n to (1 << m) byte boundary +#define ALIGN(n, m) ((n + ((1 << m) - 1)) & ~((1 << m) - 1)) + +#define STEP1 do { \ + int c = (l & 0xff) ^ *p++; \ + l = table0_[c] ^ (l >> 8); \ +} while (0) + +static uint32_t crc32c_slow(uint32_t crc, const char* buf, size_t size) +{ + const uint8_t *p = reinterpret_cast<const uint8_t *>(buf); + const uint8_t *e = p + size; + uint64_t l = crc ^ 0xffffffffu; + + // Point x at first 16-byte aligned byte in string. This might be + // just past the end of the string. + const uintptr_t pval = reinterpret_cast<uintptr_t>(p); + const uint8_t* x = reinterpret_cast<const uint8_t*>(ALIGN(pval, 4)); + if (x <= e) + // Process bytes until finished or p is 16-byte aligned + while (p != x) + STEP1; + // Process bytes 16 at a time + while ((e-p) >= 16) + { + Slow_CRC32(&l, &p); + Slow_CRC32(&l, &p); + } + // Process bytes 8 at a time + while ((e-p) >= 8) + Slow_CRC32(&l, &p); + // Process the last few bytes + while (p != e) + STEP1; + return static_cast<uint32_t>(l ^ 0xffffffffu); +} + +#if defined HAVE_POWER8 +#elif defined HAVE_ARMV8_CRC +#elif defined HAVE_SSE42 +constexpr uint32_t cpuid_ecx_SSE42= 1U << 20; +constexpr uint32_t cpuid_ecx_SSE42_AND_PCLMUL= cpuid_ecx_SSE42 | 1U<<1; + +static uint32_t cpuid_ecx() +{ +#ifdef __GNUC__ + uint32_t reax= 0, rebx= 0, recx= 0, redx= 0; + __cpuid(1, reax, rebx, recx, redx); + return recx; +#elif defined _MSC_VER + int regs[4]; + __cpuid(regs, 1); + return regs[2]; +#else +# error "unknown compiler" +#endif +} + +extern "C" int crc32_pclmul_enabled(void) +{ + return !(~cpuid_ecx() & cpuid_ecx_SSE42_AND_PCLMUL); +} + +#if SIZEOF_SIZE_T == 8 +extern "C" uint32_t crc32c_3way(uint32_t crc, const char *buf, size_t len); + +USE_SSE42 +static inline uint64_t LE_LOAD64(const uint8_t *ptr) +{ + return uint8korr(reinterpret_cast<const char*>(ptr)); +} +#endif + +USE_SSE42 +static inline void Fast_CRC32(uint64_t* l, uint8_t const **p) +{ +# if (SIZEOF_SIZE_T == 8) + *l = _mm_crc32_u64(*l, LE_LOAD64(*p)); + *p += 8; +# else + *l = _mm_crc32_u32(static_cast<unsigned int>(*l), LE_LOAD32(*p)); + *p += 4; + *l = _mm_crc32_u32(static_cast<unsigned int>(*l), LE_LOAD32(*p)); + *p += 4; +# endif +} + +USE_SSE42 +static uint32_t crc32c_sse42(uint32_t crc, const char* buf, size_t size) +{ + const uint8_t *p = reinterpret_cast<const uint8_t *>(buf); + const uint8_t *e = p + size; + uint64_t l = crc ^ 0xffffffffu; + + // Point x at first 16-byte aligned byte in string. This might be + // just past the end of the string. + const uintptr_t pval = reinterpret_cast<uintptr_t>(p); + const uint8_t* x = reinterpret_cast<const uint8_t*>(ALIGN(pval, 4)); + if (x <= e) + // Process bytes until finished or p is 16-byte aligned + while (p != x) + STEP1; + // Process bytes 16 at a time + while ((e-p) >= 16) + { + Fast_CRC32(&l, &p); + Fast_CRC32(&l, &p); + } + // Process bytes 8 at a time + while ((e-p) >= 8) + Fast_CRC32(&l, &p); + // Process the last few bytes + while (p != e) + STEP1; + return static_cast<uint32_t>(l ^ 0xffffffffu); +} +#endif + +typedef uint32_t (*Function)(uint32_t, const char*, size_t); + +#if defined(HAVE_POWER8) && defined(HAS_ALTIVEC) +uint32_t ExtendPPCImpl(uint32_t crc, const char *buf, size_t size) { + return crc32c_ppc(crc, (const unsigned char *)buf, size); +} + +#if __linux__ +static int arch_ppc_probe(void) { + arch_ppc_crc32 = 0; + +#if defined(__powerpc64__) + if (getauxval(AT_HWCAP2) & PPC_FEATURE2_VEC_CRYPTO) arch_ppc_crc32 = 1; +#endif /* __powerpc64__ */ + + return arch_ppc_crc32; +} +#elif __FreeBSD_version >= 1200000 +#include <machine/cpu.h> +#include <sys/auxv.h> +#include <sys/elf_common.h> +static int arch_ppc_probe(void) { + unsigned long cpufeatures; + arch_ppc_crc32 = 0; + +#if defined(__powerpc64__) + elf_aux_info(AT_HWCAP2, &cpufeatures, sizeof(cpufeatures)); + if (cpufeatures & PPC_FEATURE2_HAS_VEC_CRYPTO) arch_ppc_crc32 = 1; +#endif /* __powerpc64__ */ + + return arch_ppc_crc32; +} +#elif defined(_AIX) || defined(__OpenBSD__) +static int arch_ppc_probe(void) { + arch_ppc_crc32 = 0; + +#if defined(__powerpc64__) + // AIX 7.1+/OpenBSD has vector crypto features on all POWER 8+ + arch_ppc_crc32 = 1; +#endif /* __powerpc64__ */ + + return arch_ppc_crc32; +} +#endif // __linux__ +#endif + +#if defined(HAVE_ARMV8_CRC) +extern "C" const char *crc32c_aarch64_available(void); +extern "C" uint32_t crc32c_aarch64(uint32_t crc, const unsigned char *buffer, uint64_t len); + +static uint32_t ExtendARMImpl(uint32_t crc, const char *buf, size_t size) { + return crc32c_aarch64(crc, (const unsigned char *)buf, (size_t) size); +} +#endif + +static inline Function Choose_Extend() +{ +#if defined HAVE_POWER8 && defined HAS_ALTIVEC + if (arch_ppc_probe()) + return ExtendPPCImpl; +#elif defined(HAVE_ARMV8_CRC) + if (crc32c_aarch64_available()) + return ExtendARMImpl; +#elif HAVE_SSE42 +# if defined HAVE_PCLMUL && SIZEOF_SIZE_T == 8 + switch (cpuid_ecx() & cpuid_ecx_SSE42_AND_PCLMUL) { + case cpuid_ecx_SSE42_AND_PCLMUL: + return crc32c_3way; + case cpuid_ecx_SSE42: + return crc32c_sse42; + } +# else + if (cpuid_ecx() & cpuid_ecx_SSE42) + return crc32c_sse42; +# endif +#endif + return crc32c_slow; +} + +static const Function ChosenExtend= Choose_Extend(); + +static inline uint32_t Extend(uint32_t crc, const char* buf, size_t size) +{ + return ChosenExtend(crc, buf, size); +} + +extern "C" const char *my_crc32c_implementation() +{ +#if defined(HAVE_POWER8) && defined(HAS_ALTIVEC) + if (ChosenExtend == ExtendPPCImpl) + return "Using POWER8 crc32 instructions"; +#elif defined(HAVE_ARMV8_CRC) + if (const char *ret= crc32c_aarch64_available()) + return ret; +#elif HAVE_SSE42 +# if defined HAVE_PCLMUL && SIZEOF_SIZE_T == 8 + if (ChosenExtend == crc32c_3way) + return "Using crc32 + pclmulqdq instructions"; +# endif + if (ChosenExtend == crc32c_sse42) + return "Using SSE4.2 crc32 instructions"; +#endif + return "Using generic crc32 instructions"; +} +} // namespace crc32c +} // namespace mysys_namespace + +extern "C" unsigned my_crc32c(unsigned int crc, const char *buf, size_t size) +{ + return mysys_namespace::crc32c::Extend(crc,buf, size); +} diff --git a/mysys/crc32/crc32c_amd64.cc b/mysys/crc32/crc32c_amd64.cc new file mode 100644 index 00000000000..22c492b457f --- /dev/null +++ b/mysys/crc32/crc32c_amd64.cc @@ -0,0 +1,711 @@ +/* Copyright (c) 2020, 2021, MariaDB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +/* + * Copyright 2016 Ferry Toth, Exalon Delft BV, The Netherlands + * This software is provided 'as-is', without any express or implied + * warranty. In no event will the author be held liable for any damages + * arising from the use of this software. + * Permission is granted to anyone to use this software for any purpose, + * including commercial applications, and to alter it and redistribute it + * freely, subject to the following restrictions: + * 1. The origin of this software must not be misrepresented; you must not + * claim that you wrote the original software. If you use this software + * in a product, an acknowledgment in the product documentation would be + * appreciated but is not required. + * 2. Altered source versions must be plainly marked as such, and must not be + * misrepresented as being the original software. + * 3. This notice may not be removed or altered from any source distribution. + * Ferry Toth + * ftoth@exalondelft.nl + * + * https://github.com/htot/crc32c + * + * Modified by Facebook + * + * Original intel whitepaper: + * "Fast CRC Computation for iSCSI Polynomial Using CRC32 Instruction" + * https://www.intel.com/content/dam/www/public/us/en/documents/white-papers/crc-iscsi-polynomial-crc32-instruction-paper.pdf + * + * This version is from the folly library, created by Dave Watson <davejwatson@fb.com> + * +*/ + +#include <stdint.h> +#include <nmmintrin.h> +#include <wmmintrin.h> + + +#define CRCtriplet(crc, buf, offset) \ + crc##0 = _mm_crc32_u64(crc##0, *(buf##0 + offset)); \ + crc##1 = _mm_crc32_u64(crc##1, *(buf##1 + offset)); \ + crc##2 = _mm_crc32_u64(crc##2, *(buf##2 + offset)); + +#define CRCduplet(crc, buf, offset) \ + crc##0 = _mm_crc32_u64(crc##0, *(buf##0 + offset)); \ + crc##1 = _mm_crc32_u64(crc##1, *(buf##1 + offset)); + +#define CRCsinglet(crc, buf, offset) \ + crc = _mm_crc32_u64(crc, *(uint64_t*)(buf + offset)); + + +// Numbers taken directly from intel whitepaper. +// clang-format off +static const uint64_t clmul_constants alignas(16) [] = { + 0x14cd00bd6, 0x105ec76f0, 0x0ba4fc28e, 0x14cd00bd6, + 0x1d82c63da, 0x0f20c0dfe, 0x09e4addf8, 0x0ba4fc28e, + 0x039d3b296, 0x1384aa63a, 0x102f9b8a2, 0x1d82c63da, + 0x14237f5e6, 0x01c291d04, 0x00d3b6092, 0x09e4addf8, + 0x0c96cfdc0, 0x0740eef02, 0x18266e456, 0x039d3b296, + 0x0daece73e, 0x0083a6eec, 0x0ab7aff2a, 0x102f9b8a2, + 0x1248ea574, 0x1c1733996, 0x083348832, 0x14237f5e6, + 0x12c743124, 0x02ad91c30, 0x0b9e02b86, 0x00d3b6092, + 0x018b33a4e, 0x06992cea2, 0x1b331e26a, 0x0c96cfdc0, + 0x17d35ba46, 0x07e908048, 0x1bf2e8b8a, 0x18266e456, + 0x1a3e0968a, 0x11ed1f9d8, 0x0ce7f39f4, 0x0daece73e, + 0x061d82e56, 0x0f1d0f55e, 0x0d270f1a2, 0x0ab7aff2a, + 0x1c3f5f66c, 0x0a87ab8a8, 0x12ed0daac, 0x1248ea574, + 0x065863b64, 0x08462d800, 0x11eef4f8e, 0x083348832, + 0x1ee54f54c, 0x071d111a8, 0x0b3e32c28, 0x12c743124, + 0x0064f7f26, 0x0ffd852c6, 0x0dd7e3b0c, 0x0b9e02b86, + 0x0f285651c, 0x0dcb17aa4, 0x010746f3c, 0x018b33a4e, + 0x1c24afea4, 0x0f37c5aee, 0x0271d9844, 0x1b331e26a, + 0x08e766a0c, 0x06051d5a2, 0x093a5f730, 0x17d35ba46, + 0x06cb08e5c, 0x11d5ca20e, 0x06b749fb2, 0x1bf2e8b8a, + 0x1167f94f2, 0x021f3d99c, 0x0cec3662e, 0x1a3e0968a, + 0x19329634a, 0x08f158014, 0x0e6fc4e6a, 0x0ce7f39f4, + 0x08227bb8a, 0x1a5e82106, 0x0b0cd4768, 0x061d82e56, + 0x13c2b89c4, 0x188815ab2, 0x0d7a4825c, 0x0d270f1a2, + 0x10f5ff2ba, 0x105405f3e, 0x00167d312, 0x1c3f5f66c, + 0x0f6076544, 0x0e9adf796, 0x026f6a60a, 0x12ed0daac, + 0x1a2adb74e, 0x096638b34, 0x19d34af3a, 0x065863b64, + 0x049c3cc9c, 0x1e50585a0, 0x068bce87a, 0x11eef4f8e, + 0x1524fa6c6, 0x19f1c69dc, 0x16cba8aca, 0x1ee54f54c, + 0x042d98888, 0x12913343e, 0x1329d9f7e, 0x0b3e32c28, + 0x1b1c69528, 0x088f25a3a, 0x02178513a, 0x0064f7f26, + 0x0e0ac139e, 0x04e36f0b0, 0x0170076fa, 0x0dd7e3b0c, + 0x141a1a2e2, 0x0bd6f81f8, 0x16ad828b4, 0x0f285651c, + 0x041d17b64, 0x19425cbba, 0x1fae1cc66, 0x010746f3c, + 0x1a75b4b00, 0x18db37e8a, 0x0f872e54c, 0x1c24afea4, + 0x01e41e9fc, 0x04c144932, 0x086d8e4d2, 0x0271d9844, + 0x160f7af7a, 0x052148f02, 0x05bb8f1bc, 0x08e766a0c, + 0x0a90fd27a, 0x0a3c6f37a, 0x0b3af077a, 0x093a5f730, + 0x04984d782, 0x1d22c238e, 0x0ca6ef3ac, 0x06cb08e5c, + 0x0234e0b26, 0x063ded06a, 0x1d88abd4a, 0x06b749fb2, + 0x04597456a, 0x04d56973c, 0x0e9e28eb4, 0x1167f94f2, + 0x07b3ff57a, 0x19385bf2e, 0x0c9c8b782, 0x0cec3662e, + 0x13a9cba9e, 0x0e417f38a, 0x093e106a4, 0x19329634a, + 0x167001a9c, 0x14e727980, 0x1ddffc5d4, 0x0e6fc4e6a, + 0x00df04680, 0x0d104b8fc, 0x02342001e, 0x08227bb8a, + 0x00a2a8d7e, 0x05b397730, 0x168763fa6, 0x0b0cd4768, + 0x1ed5a407a, 0x0e78eb416, 0x0d2c3ed1a, 0x13c2b89c4, + 0x0995a5724, 0x1641378f0, 0x19b1afbc4, 0x0d7a4825c, + 0x109ffedc0, 0x08d96551c, 0x0f2271e60, 0x10f5ff2ba, + 0x00b0bf8ca, 0x00bf80dd2, 0x123888b7a, 0x00167d312, + 0x1e888f7dc, 0x18dcddd1c, 0x002ee03b2, 0x0f6076544, + 0x183e8d8fe, 0x06a45d2b2, 0x133d7a042, 0x026f6a60a, + 0x116b0f50c, 0x1dd3e10e8, 0x05fabe670, 0x1a2adb74e, + 0x130004488, 0x0de87806c, 0x000bcf5f6, 0x19d34af3a, + 0x18f0c7078, 0x014338754, 0x017f27698, 0x049c3cc9c, + 0x058ca5f00, 0x15e3e77ee, 0x1af900c24, 0x068bce87a, + 0x0b5cfca28, 0x0dd07448e, 0x0ded288f8, 0x1524fa6c6, + 0x059f229bc, 0x1d8048348, 0x06d390dec, 0x16cba8aca, + 0x037170390, 0x0a3e3e02c, 0x06353c1cc, 0x042d98888, + 0x0c4584f5c, 0x0d73c7bea, 0x1f16a3418, 0x1329d9f7e, + 0x0531377e2, 0x185137662, 0x1d8d9ca7c, 0x1b1c69528, + 0x0b25b29f2, 0x18a08b5bc, 0x19fb2a8b0, 0x02178513a, + 0x1a08fe6ac, 0x1da758ae0, 0x045cddf4e, 0x0e0ac139e, + 0x1a91647f2, 0x169cf9eb0, 0x1a0f717c4, 0x0170076fa, +}; + +// Compute the crc32c value for buffer smaller than 8 +static inline void align_to_8( + size_t len, + uint64_t& crc0, // crc so far, updated on return + const unsigned char*& next) { // next data pointer, updated on return + uint32_t crc32bit = static_cast<uint32_t>(crc0); + if (len & 0x04) { + crc32bit = _mm_crc32_u32(crc32bit, *(uint32_t*)next); + next += sizeof(uint32_t); + } + if (len & 0x02) { + crc32bit = _mm_crc32_u16(crc32bit, *(uint16_t*)next); + next += sizeof(uint16_t); + } + if (len & 0x01) { + crc32bit = _mm_crc32_u8(crc32bit, *(next)); + next++; + } + crc0 = crc32bit; +} + +// +// CombineCRC performs pclmulqdq multiplication of 2 partial CRC's and a well +// chosen constant and xor's these with the remaining CRC. +// +static inline uint64_t CombineCRC( + size_t block_size, + uint64_t crc0, + uint64_t crc1, + uint64_t crc2, + const uint64_t* next2) { + const auto multiplier = + *(reinterpret_cast<const __m128i*>(clmul_constants) + block_size - 1); + const auto crc0_xmm = _mm_set_epi64x(0, crc0); + const auto res0 = _mm_clmulepi64_si128(crc0_xmm, multiplier, 0x00); + const auto crc1_xmm = _mm_set_epi64x(0, crc1); + const auto res1 = _mm_clmulepi64_si128(crc1_xmm, multiplier, 0x10); + const auto res = _mm_xor_si128(res0, res1); + crc0 = _mm_cvtsi128_si64(res); + crc0 = crc0 ^ *((uint64_t*)next2 - 1); + crc2 = _mm_crc32_u64(crc2, crc0); + return crc2; +} + +// Compute CRC-32C using the Intel hardware instruction. +extern "C" +uint32_t crc32c_3way(uint32_t crc, const char *buf, size_t len) +{ + const unsigned char* next = (const unsigned char*)buf; + uint64_t count; + uint64_t crc0, crc1, crc2; + crc0 = crc ^ 0xffffffffu; + + + if (len >= 8) { + // if len > 216 then align and use triplets + if (len > 216) { + { + // Work on the bytes (< 8) before the first 8-byte alignment addr starts + auto align_bytes = (8 - (uintptr_t)next) & 7; + len -= align_bytes; + align_to_8(align_bytes, crc0, next); + } + + // Now work on the remaining blocks + count = len / 24; // number of triplets + len %= 24; // bytes remaining + uint64_t n = count >> 7; // #blocks = first block + full blocks + uint64_t block_size = count & 127; + if (block_size == 0) { + block_size = 128; + } else { + n++; + } + // points to the first byte of the next block + const uint64_t* next0 = (uint64_t*)next + block_size; + const uint64_t* next1 = next0 + block_size; + const uint64_t* next2 = next1 + block_size; + + crc1 = crc2 = 0; + // Use Duff's device, a for() loop inside a switch() + // statement. This needs to execute at least once, round len + // down to nearest triplet multiple + switch (block_size) { + case 128: + do { + // jumps here for a full block of len 128 + CRCtriplet(crc, next, -128); + /* fallthrough */ + case 127: + // jumps here or below for the first block smaller + CRCtriplet(crc, next, -127); + /* fallthrough */ + case 126: + CRCtriplet(crc, next, -126); // than 128 + /* fallthrough */ + case 125: + CRCtriplet(crc, next, -125); + /* fallthrough */ + case 124: + CRCtriplet(crc, next, -124); + /* fallthrough */ + case 123: + CRCtriplet(crc, next, -123); + /* fallthrough */ + case 122: + CRCtriplet(crc, next, -122); + /* fallthrough */ + case 121: + CRCtriplet(crc, next, -121); + /* fallthrough */ + case 120: + CRCtriplet(crc, next, -120); + /* fallthrough */ + case 119: + CRCtriplet(crc, next, -119); + /* fallthrough */ + case 118: + CRCtriplet(crc, next, -118); + /* fallthrough */ + case 117: + CRCtriplet(crc, next, -117); + /* fallthrough */ + case 116: + CRCtriplet(crc, next, -116); + /* fallthrough */ + case 115: + CRCtriplet(crc, next, -115); + /* fallthrough */ + case 114: + CRCtriplet(crc, next, -114); + /* fallthrough */ + case 113: + CRCtriplet(crc, next, -113); + /* fallthrough */ + case 112: + CRCtriplet(crc, next, -112); + /* fallthrough */ + case 111: + CRCtriplet(crc, next, -111); + /* fallthrough */ + case 110: + CRCtriplet(crc, next, -110); + /* fallthrough */ + case 109: + CRCtriplet(crc, next, -109); + /* fallthrough */ + case 108: + CRCtriplet(crc, next, -108); + /* fallthrough */ + case 107: + CRCtriplet(crc, next, -107); + /* fallthrough */ + case 106: + CRCtriplet(crc, next, -106); + /* fallthrough */ + case 105: + CRCtriplet(crc, next, -105); + /* fallthrough */ + case 104: + CRCtriplet(crc, next, -104); + /* fallthrough */ + case 103: + CRCtriplet(crc, next, -103); + /* fallthrough */ + case 102: + CRCtriplet(crc, next, -102); + /* fallthrough */ + case 101: + CRCtriplet(crc, next, -101); + /* fallthrough */ + case 100: + CRCtriplet(crc, next, -100); + /* fallthrough */ + case 99: + CRCtriplet(crc, next, -99); + /* fallthrough */ + case 98: + CRCtriplet(crc, next, -98); + /* fallthrough */ + case 97: + CRCtriplet(crc, next, -97); + /* fallthrough */ + case 96: + CRCtriplet(crc, next, -96); + /* fallthrough */ + case 95: + CRCtriplet(crc, next, -95); + /* fallthrough */ + case 94: + CRCtriplet(crc, next, -94); + /* fallthrough */ + case 93: + CRCtriplet(crc, next, -93); + /* fallthrough */ + case 92: + CRCtriplet(crc, next, -92); + /* fallthrough */ + case 91: + CRCtriplet(crc, next, -91); + /* fallthrough */ + case 90: + CRCtriplet(crc, next, -90); + /* fallthrough */ + case 89: + CRCtriplet(crc, next, -89); + /* fallthrough */ + case 88: + CRCtriplet(crc, next, -88); + /* fallthrough */ + case 87: + CRCtriplet(crc, next, -87); + /* fallthrough */ + case 86: + CRCtriplet(crc, next, -86); + /* fallthrough */ + case 85: + CRCtriplet(crc, next, -85); + /* fallthrough */ + case 84: + CRCtriplet(crc, next, -84); + /* fallthrough */ + case 83: + CRCtriplet(crc, next, -83); + /* fallthrough */ + case 82: + CRCtriplet(crc, next, -82); + /* fallthrough */ + case 81: + CRCtriplet(crc, next, -81); + /* fallthrough */ + case 80: + CRCtriplet(crc, next, -80); + /* fallthrough */ + case 79: + CRCtriplet(crc, next, -79); + /* fallthrough */ + case 78: + CRCtriplet(crc, next, -78); + /* fallthrough */ + case 77: + CRCtriplet(crc, next, -77); + /* fallthrough */ + case 76: + CRCtriplet(crc, next, -76); + /* fallthrough */ + case 75: + CRCtriplet(crc, next, -75); + /* fallthrough */ + case 74: + CRCtriplet(crc, next, -74); + /* fallthrough */ + case 73: + CRCtriplet(crc, next, -73); + /* fallthrough */ + case 72: + CRCtriplet(crc, next, -72); + /* fallthrough */ + case 71: + CRCtriplet(crc, next, -71); + /* fallthrough */ + case 70: + CRCtriplet(crc, next, -70); + /* fallthrough */ + case 69: + CRCtriplet(crc, next, -69); + /* fallthrough */ + case 68: + CRCtriplet(crc, next, -68); + /* fallthrough */ + case 67: + CRCtriplet(crc, next, -67); + /* fallthrough */ + case 66: + CRCtriplet(crc, next, -66); + /* fallthrough */ + case 65: + CRCtriplet(crc, next, -65); + /* fallthrough */ + case 64: + CRCtriplet(crc, next, -64); + /* fallthrough */ + case 63: + CRCtriplet(crc, next, -63); + /* fallthrough */ + case 62: + CRCtriplet(crc, next, -62); + /* fallthrough */ + case 61: + CRCtriplet(crc, next, -61); + /* fallthrough */ + case 60: + CRCtriplet(crc, next, -60); + /* fallthrough */ + case 59: + CRCtriplet(crc, next, -59); + /* fallthrough */ + case 58: + CRCtriplet(crc, next, -58); + /* fallthrough */ + case 57: + CRCtriplet(crc, next, -57); + /* fallthrough */ + case 56: + CRCtriplet(crc, next, -56); + /* fallthrough */ + case 55: + CRCtriplet(crc, next, -55); + /* fallthrough */ + case 54: + CRCtriplet(crc, next, -54); + /* fallthrough */ + case 53: + CRCtriplet(crc, next, -53); + /* fallthrough */ + case 52: + CRCtriplet(crc, next, -52); + /* fallthrough */ + case 51: + CRCtriplet(crc, next, -51); + /* fallthrough */ + case 50: + CRCtriplet(crc, next, -50); + /* fallthrough */ + case 49: + CRCtriplet(crc, next, -49); + /* fallthrough */ + case 48: + CRCtriplet(crc, next, -48); + /* fallthrough */ + case 47: + CRCtriplet(crc, next, -47); + /* fallthrough */ + case 46: + CRCtriplet(crc, next, -46); + /* fallthrough */ + case 45: + CRCtriplet(crc, next, -45); + /* fallthrough */ + case 44: + CRCtriplet(crc, next, -44); + /* fallthrough */ + case 43: + CRCtriplet(crc, next, -43); + /* fallthrough */ + case 42: + CRCtriplet(crc, next, -42); + /* fallthrough */ + case 41: + CRCtriplet(crc, next, -41); + /* fallthrough */ + case 40: + CRCtriplet(crc, next, -40); + /* fallthrough */ + case 39: + CRCtriplet(crc, next, -39); + /* fallthrough */ + case 38: + CRCtriplet(crc, next, -38); + /* fallthrough */ + case 37: + CRCtriplet(crc, next, -37); + /* fallthrough */ + case 36: + CRCtriplet(crc, next, -36); + /* fallthrough */ + case 35: + CRCtriplet(crc, next, -35); + /* fallthrough */ + case 34: + CRCtriplet(crc, next, -34); + /* fallthrough */ + case 33: + CRCtriplet(crc, next, -33); + /* fallthrough */ + case 32: + CRCtriplet(crc, next, -32); + /* fallthrough */ + case 31: + CRCtriplet(crc, next, -31); + /* fallthrough */ + case 30: + CRCtriplet(crc, next, -30); + /* fallthrough */ + case 29: + CRCtriplet(crc, next, -29); + /* fallthrough */ + case 28: + CRCtriplet(crc, next, -28); + /* fallthrough */ + case 27: + CRCtriplet(crc, next, -27); + /* fallthrough */ + case 26: + CRCtriplet(crc, next, -26); + /* fallthrough */ + case 25: + CRCtriplet(crc, next, -25); + /* fallthrough */ + case 24: + CRCtriplet(crc, next, -24); + /* fallthrough */ + case 23: + CRCtriplet(crc, next, -23); + /* fallthrough */ + case 22: + CRCtriplet(crc, next, -22); + /* fallthrough */ + case 21: + CRCtriplet(crc, next, -21); + /* fallthrough */ + case 20: + CRCtriplet(crc, next, -20); + /* fallthrough */ + case 19: + CRCtriplet(crc, next, -19); + /* fallthrough */ + case 18: + CRCtriplet(crc, next, -18); + /* fallthrough */ + case 17: + CRCtriplet(crc, next, -17); + /* fallthrough */ + case 16: + CRCtriplet(crc, next, -16); + /* fallthrough */ + case 15: + CRCtriplet(crc, next, -15); + /* fallthrough */ + case 14: + CRCtriplet(crc, next, -14); + /* fallthrough */ + case 13: + CRCtriplet(crc, next, -13); + /* fallthrough */ + case 12: + CRCtriplet(crc, next, -12); + /* fallthrough */ + case 11: + CRCtriplet(crc, next, -11); + /* fallthrough */ + case 10: + CRCtriplet(crc, next, -10); + /* fallthrough */ + case 9: + CRCtriplet(crc, next, -9); + /* fallthrough */ + case 8: + CRCtriplet(crc, next, -8); + /* fallthrough */ + case 7: + CRCtriplet(crc, next, -7); + /* fallthrough */ + case 6: + CRCtriplet(crc, next, -6); + /* fallthrough */ + case 5: + CRCtriplet(crc, next, -5); + /* fallthrough */ + case 4: + CRCtriplet(crc, next, -4); + /* fallthrough */ + case 3: + CRCtriplet(crc, next, -3); + /* fallthrough */ + case 2: + CRCtriplet(crc, next, -2); + /* fallthrough */ + case 1: + CRCduplet(crc, next, -1); // the final triplet is actually only 2 + //{ CombineCRC(); } + crc0 = CombineCRC(block_size, crc0, crc1, crc2, next2); + if (--n > 0) { + crc1 = crc2 = 0; + block_size = 128; + // points to the first byte of the next block + next0 = next2 + 128; + next1 = next0 + 128; // from here on all blocks are 128 long + next2 = next1 + 128; + } + /* fallthrough */ + case 0:; + } while (n > 0); + } + next = (const unsigned char*)next2; + } + uint64_t count2 = len >> 3; // 216 of less bytes is 27 or less singlets + len = len & 7; + next += (count2 * 8); + switch (count2) { + case 27: + CRCsinglet(crc0, next, -27 * 8); + /* fallthrough */ + case 26: + CRCsinglet(crc0, next, -26 * 8); + /* fallthrough */ + case 25: + CRCsinglet(crc0, next, -25 * 8); + /* fallthrough */ + case 24: + CRCsinglet(crc0, next, -24 * 8); + /* fallthrough */ + case 23: + CRCsinglet(crc0, next, -23 * 8); + /* fallthrough */ + case 22: + CRCsinglet(crc0, next, -22 * 8); + /* fallthrough */ + case 21: + CRCsinglet(crc0, next, -21 * 8); + /* fallthrough */ + case 20: + CRCsinglet(crc0, next, -20 * 8); + /* fallthrough */ + case 19: + CRCsinglet(crc0, next, -19 * 8); + /* fallthrough */ + case 18: + CRCsinglet(crc0, next, -18 * 8); + /* fallthrough */ + case 17: + CRCsinglet(crc0, next, -17 * 8); + /* fallthrough */ + case 16: + CRCsinglet(crc0, next, -16 * 8); + /* fallthrough */ + case 15: + CRCsinglet(crc0, next, -15 * 8); + /* fallthrough */ + case 14: + CRCsinglet(crc0, next, -14 * 8); + /* fallthrough */ + case 13: + CRCsinglet(crc0, next, -13 * 8); + /* fallthrough */ + case 12: + CRCsinglet(crc0, next, -12 * 8); + /* fallthrough */ + case 11: + CRCsinglet(crc0, next, -11 * 8); + /* fallthrough */ + case 10: + CRCsinglet(crc0, next, -10 * 8); + /* fallthrough */ + case 9: + CRCsinglet(crc0, next, -9 * 8); + /* fallthrough */ + case 8: + CRCsinglet(crc0, next, -8 * 8); + /* fallthrough */ + case 7: + CRCsinglet(crc0, next, -7 * 8); + /* fallthrough */ + case 6: + CRCsinglet(crc0, next, -6 * 8); + /* fallthrough */ + case 5: + CRCsinglet(crc0, next, -5 * 8); + /* fallthrough */ + case 4: + CRCsinglet(crc0, next, -4 * 8); + /* fallthrough */ + case 3: + CRCsinglet(crc0, next, -3 * 8); + /* fallthrough */ + case 2: + CRCsinglet(crc0, next, -2 * 8); + /* fallthrough */ + case 1: + CRCsinglet(crc0, next, -1 * 8); + /* fallthrough */ + case 0:; + } + } + { + align_to_8(len, crc0, next); + return (uint32_t)crc0 ^ 0xffffffffu; + } +} diff --git a/mysys/crc32/crc32c_ppc.c b/mysys/crc32/crc32c_ppc.c new file mode 100644 index 00000000000..72f24283454 --- /dev/null +++ b/mysys/crc32/crc32c_ppc.c @@ -0,0 +1,5 @@ +#define CRC32_FUNCTION crc32c_ppc +#define CRC_TABLE +#define POWER8_INTRINSICS +#include "pcc_crc32c_constants.h" +#include "crc_ppc64.h" diff --git a/mysys/crc32/crc32c_ppc.h b/mysys/crc32/crc32c_ppc.h new file mode 100644 index 00000000000..c359061c610 --- /dev/null +++ b/mysys/crc32/crc32c_ppc.h @@ -0,0 +1,19 @@ +// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. +// Copyright (c) 2017 International Business Machines Corp. +// All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + +extern uint32_t crc32c_ppc(uint32_t crc, unsigned char const *buffer, + unsigned len); + +#ifdef __cplusplus +} +#endif diff --git a/mysys/crc32/crc_ppc64.h b/mysys/crc32/crc_ppc64.h new file mode 100644 index 00000000000..eb9379abc6c --- /dev/null +++ b/mysys/crc32/crc_ppc64.h @@ -0,0 +1,664 @@ +/* + * Calculate the checksum of data that is 16 byte aligned and a multiple of + * 16 bytes. + * + * The first step is to reduce it to 1024 bits. We do this in 8 parallel + * chunks in order to mask the latency of the vpmsum instructions. If we + * have more than 32 kB of data to checksum we repeat this step multiple + * times, passing in the previous 1024 bits. + * + * The next step is to reduce the 1024 bits to 64 bits. This step adds + * 32 bits of 0s to the end - this matches what a CRC does. We just + * calculate constants that land the data in this 32 bits. + * + * We then use fixed point Barrett reduction to compute a mod n over GF(2) + * for n = CRC using POWER8 instructions. We use x = 32. + * + * http://en.wikipedia.org/wiki/Barrett_reduction + * + * This code uses gcc vector builtins instead using assembly directly. + * + * Copyright (C) 2017 Rogerio Alves <rogealve@br.ibm.com>, IBM + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of either: + * + * a) the GNU General Public License as published by the Free Software + * Foundation; either version 2 of the License, or (at your option) + * any later version, or + * b) the Apache License, Version 2.0 + */ + +#include <altivec.h> + + +#define VMX_ALIGN 16 +#define VMX_ALIGN_MASK (VMX_ALIGN-1) + +#ifdef REFLECT +static unsigned int crc32_align(unsigned int crc, const unsigned char *p, + unsigned long len) +{ + while (len--) + crc = crc_table[(crc ^ *p++) & 0xff] ^ (crc >> 8); + return crc; +} +#else +static unsigned int crc32_align(unsigned int crc, const unsigned char *p, + unsigned long len) +{ + while (len--) + crc = crc_table[((crc >> 24) ^ *p++) & 0xff] ^ (crc << 8); + return crc; +} +#endif + +static unsigned int __attribute__ ((aligned (32))) +__crc32_vpmsum(unsigned int crc, const void* p, unsigned long len); + + +unsigned int CRC32_FUNCTION(unsigned int crc, const unsigned char *p, + unsigned long len) +{ + unsigned int prealign; + unsigned int tail; + +#ifdef CRC_XOR + crc ^= 0xffffffff; +#endif + + if (len < VMX_ALIGN + VMX_ALIGN_MASK) { + crc = crc32_align(crc, p, len); + goto out; + } + + if ((unsigned long)p & VMX_ALIGN_MASK) { + prealign = VMX_ALIGN - ((unsigned long)p & VMX_ALIGN_MASK); + crc = crc32_align(crc, p, prealign); + len -= prealign; + p += prealign; + } + + crc = __crc32_vpmsum(crc, p, len & ~VMX_ALIGN_MASK); + + tail = len & VMX_ALIGN_MASK; + if (tail) { + p += len & ~VMX_ALIGN_MASK; + crc = crc32_align(crc, p, tail); + } + +out: +#ifdef CRC_XOR + crc ^= 0xffffffff; +#endif + + return crc; +} + +#if defined (__clang__) +#include "clang_workaround.h" +#else +#define __builtin_pack_vector(a, b) __builtin_pack_vector_int128 ((a), (b)) +#define __builtin_unpack_vector_0(a) __builtin_unpack_vector_int128 ((vector __int128_t)(a), 0) +#define __builtin_unpack_vector_1(a) __builtin_unpack_vector_int128 ((vector __int128_t)(a), 1) +#endif + +/* When we have a load-store in a single-dispatch group and address overlap + * such that foward is not allowed (load-hit-store) the group must be flushed. + * A group ending NOP prevents the flush. + */ +#define GROUP_ENDING_NOP asm("ori 2,2,0" ::: "memory") + +#if defined(__BIG_ENDIAN__) && defined (REFLECT) +#define BYTESWAP_DATA +#elif defined(__LITTLE_ENDIAN__) && !defined(REFLECT) +#define BYTESWAP_DATA +#endif + +#ifdef BYTESWAP_DATA +#define VEC_PERM(vr, va, vb, vc) vr = vec_perm(va, vb,\ + (__vector unsigned char) vc) +#if defined(__LITTLE_ENDIAN__) +/* Byte reverse permute constant LE. */ +static const __vector unsigned long long vperm_const + __attribute__ ((aligned(16))) = { 0x08090A0B0C0D0E0FUL, + 0x0001020304050607UL }; +#else +static const __vector unsigned long long vperm_const + __attribute__ ((aligned(16))) = { 0x0F0E0D0C0B0A0908UL, + 0X0706050403020100UL }; +#endif +#else +#define VEC_PERM(vr, va, vb, vc) +#endif + +static unsigned int __attribute__ ((aligned (32))) +__crc32_vpmsum(unsigned int crc, const void* p, unsigned long len) { + + const __vector unsigned long long vzero = {0,0}; + const __vector unsigned long long vones = {0xffffffffffffffffUL, + 0xffffffffffffffffUL}; + +#ifdef REFLECT + __vector unsigned char vsht_splat; + const __vector unsigned long long vmask_32bit = + (__vector unsigned long long)vec_sld((__vector unsigned char)vzero, + (__vector unsigned char)vones, 4); +#endif + + const __vector unsigned long long vmask_64bit = + (__vector unsigned long long)vec_sld((__vector unsigned char)vzero, + (__vector unsigned char)vones, 8); + + __vector unsigned long long vcrc; + + __vector unsigned long long vconst1, vconst2; + + /* vdata0-vdata7 will contain our data (p). */ + __vector unsigned long long vdata0, vdata1, vdata2, vdata3, vdata4, + vdata5, vdata6, vdata7; + + /* v0-v7 will contain our checksums */ + __vector unsigned long long v0 = {0,0}; + __vector unsigned long long v1 = {0,0}; + __vector unsigned long long v2 = {0,0}; + __vector unsigned long long v3 = {0,0}; + __vector unsigned long long v4 = {0,0}; + __vector unsigned long long v5 = {0,0}; + __vector unsigned long long v6 = {0,0}; + __vector unsigned long long v7 = {0,0}; + + + /* Vector auxiliary variables. */ + __vector unsigned long long va0, va1, va2, va3, va4, va5, va6, va7; + + unsigned int result = 0; + unsigned int offset; /* Constant table offset. */ + + unsigned long i; /* Counter. */ + unsigned long chunks; + + unsigned long block_size; + int next_block = 0; + + /* Align by 128 bits. The last 128 bit block will be processed at end. */ + unsigned long length = len & 0xFFFFFFFFFFFFFF80UL; + +#ifdef REFLECT + vcrc = (__vector unsigned long long)__builtin_pack_vector(0UL, crc); +#else + vcrc = (__vector unsigned long long)__builtin_pack_vector(crc, 0UL); + + /* Shift into top 32 bits */ + vcrc = (__vector unsigned long long)vec_sld((__vector unsigned char)vcrc, + (__vector unsigned char)vzero, 4); +#endif + + /* Short version. */ + if (len < 256) { + /* Calculate where in the constant table we need to start. */ + offset = 256 - len; + + vconst1 = vec_ld(offset, vcrc_short_const); + vdata0 = vec_ld(0, (__vector unsigned long long*) p); + VEC_PERM(vdata0, vdata0, vconst1, vperm_const); + + /* xor initial value*/ + vdata0 = vec_xor(vdata0, vcrc); + + vdata0 = (__vector unsigned long long) __builtin_crypto_vpmsumw + ((__vector unsigned int)vdata0, (__vector unsigned int)vconst1); + v0 = vec_xor(v0, vdata0); + + for (i = 16; i < len; i += 16) { + vconst1 = vec_ld(offset + i, vcrc_short_const); + vdata0 = vec_ld(i, (__vector unsigned long long*) p); + VEC_PERM(vdata0, vdata0, vconst1, vperm_const); + vdata0 = (__vector unsigned long long) __builtin_crypto_vpmsumw + ((__vector unsigned int)vdata0, (__vector unsigned int)vconst1); + v0 = vec_xor(v0, vdata0); + } + } else { + + /* Load initial values. */ + vdata0 = vec_ld(0, (__vector unsigned long long*) p); + vdata1 = vec_ld(16, (__vector unsigned long long*) p); + + VEC_PERM(vdata0, vdata0, vdata0, vperm_const); + VEC_PERM(vdata1, vdata1, vdata1, vperm_const); + + vdata2 = vec_ld(32, (__vector unsigned long long*) p); + vdata3 = vec_ld(48, (__vector unsigned long long*) p); + + VEC_PERM(vdata2, vdata2, vdata2, vperm_const); + VEC_PERM(vdata3, vdata3, vdata3, vperm_const); + + vdata4 = vec_ld(64, (__vector unsigned long long*) p); + vdata5 = vec_ld(80, (__vector unsigned long long*) p); + + VEC_PERM(vdata4, vdata4, vdata4, vperm_const); + VEC_PERM(vdata5, vdata5, vdata5, vperm_const); + + vdata6 = vec_ld(96, (__vector unsigned long long*) p); + vdata7 = vec_ld(112, (__vector unsigned long long*) p); + + VEC_PERM(vdata6, vdata6, vdata6, vperm_const); + VEC_PERM(vdata7, vdata7, vdata7, vperm_const); + + /* xor in initial value */ + vdata0 = vec_xor(vdata0, vcrc); + + p = (char *)p + 128; + + do { + /* Checksum in blocks of MAX_SIZE. */ + block_size = length; + if (block_size > MAX_SIZE) { + block_size = MAX_SIZE; + } + + length = length - block_size; + + /* + * Work out the offset into the constants table to start at. Each + * constant is 16 bytes, and it is used against 128 bytes of input + * data - 128 / 16 = 8 + */ + offset = (MAX_SIZE/8) - (block_size/8); + /* We reduce our final 128 bytes in a separate step */ + chunks = (block_size/128)-1; + + vconst1 = vec_ld(offset, vcrc_const); + + va0 = __builtin_crypto_vpmsumd ((__vector unsigned long long)vdata0, + (__vector unsigned long long)vconst1); + va1 = __builtin_crypto_vpmsumd ((__vector unsigned long long)vdata1, + (__vector unsigned long long)vconst1); + va2 = __builtin_crypto_vpmsumd ((__vector unsigned long long)vdata2, + (__vector unsigned long long)vconst1); + va3 = __builtin_crypto_vpmsumd ((__vector unsigned long long)vdata3, + (__vector unsigned long long)vconst1); + va4 = __builtin_crypto_vpmsumd ((__vector unsigned long long)vdata4, + (__vector unsigned long long)vconst1); + va5 = __builtin_crypto_vpmsumd ((__vector unsigned long long)vdata5, + (__vector unsigned long long)vconst1); + va6 = __builtin_crypto_vpmsumd ((__vector unsigned long long)vdata6, + (__vector unsigned long long)vconst1); + va7 = __builtin_crypto_vpmsumd ((__vector unsigned long long)vdata7, + (__vector unsigned long long)vconst1); + + if (chunks > 1) { + offset += 16; + vconst2 = vec_ld(offset, vcrc_const); + GROUP_ENDING_NOP; + + vdata0 = vec_ld(0, (__vector unsigned long long*) p); + VEC_PERM(vdata0, vdata0, vdata0, vperm_const); + + vdata1 = vec_ld(16, (__vector unsigned long long*) p); + VEC_PERM(vdata1, vdata1, vdata1, vperm_const); + + vdata2 = vec_ld(32, (__vector unsigned long long*) p); + VEC_PERM(vdata2, vdata2, vdata2, vperm_const); + + vdata3 = vec_ld(48, (__vector unsigned long long*) p); + VEC_PERM(vdata3, vdata3, vdata3, vperm_const); + + vdata4 = vec_ld(64, (__vector unsigned long long*) p); + VEC_PERM(vdata4, vdata4, vdata4, vperm_const); + + vdata5 = vec_ld(80, (__vector unsigned long long*) p); + VEC_PERM(vdata5, vdata5, vdata5, vperm_const); + + vdata6 = vec_ld(96, (__vector unsigned long long*) p); + VEC_PERM(vdata6, vdata6, vdata6, vperm_const); + + vdata7 = vec_ld(112, (__vector unsigned long long*) p); + VEC_PERM(vdata7, vdata7, vdata7, vperm_const); + + p = (char *)p + 128; + + /* + * main loop. We modulo schedule it such that it takes three + * iterations to complete - first iteration load, second + * iteration vpmsum, third iteration xor. + */ + for (i = 0; i < chunks-2; i++) { + vconst1 = vec_ld(offset, vcrc_const); + offset += 16; + GROUP_ENDING_NOP; + + v0 = vec_xor(v0, va0); + va0 = __builtin_crypto_vpmsumd ((__vector unsigned long + long)vdata0, (__vector unsigned long long)vconst2); + vdata0 = vec_ld(0, (__vector unsigned long long*) p); + VEC_PERM(vdata0, vdata0, vdata0, vperm_const); + GROUP_ENDING_NOP; + + v1 = vec_xor(v1, va1); + va1 = __builtin_crypto_vpmsumd ((__vector unsigned long + long)vdata1, (__vector unsigned long long)vconst2); + vdata1 = vec_ld(16, (__vector unsigned long long*) p); + VEC_PERM(vdata1, vdata1, vdata1, vperm_const); + GROUP_ENDING_NOP; + + v2 = vec_xor(v2, va2); + va2 = __builtin_crypto_vpmsumd ((__vector unsigned long + long)vdata2, (__vector unsigned long long)vconst2); + vdata2 = vec_ld(32, (__vector unsigned long long*) p); + VEC_PERM(vdata2, vdata2, vdata2, vperm_const); + GROUP_ENDING_NOP; + + v3 = vec_xor(v3, va3); + va3 = __builtin_crypto_vpmsumd ((__vector unsigned long + long)vdata3, (__vector unsigned long long)vconst2); + vdata3 = vec_ld(48, (__vector unsigned long long*) p); + VEC_PERM(vdata3, vdata3, vdata3, vperm_const); + + vconst2 = vec_ld(offset, vcrc_const); + GROUP_ENDING_NOP; + + v4 = vec_xor(v4, va4); + va4 = __builtin_crypto_vpmsumd ((__vector unsigned long + long)vdata4, (__vector unsigned long long)vconst1); + vdata4 = vec_ld(64, (__vector unsigned long long*) p); + VEC_PERM(vdata4, vdata4, vdata4, vperm_const); + GROUP_ENDING_NOP; + + v5 = vec_xor(v5, va5); + va5 = __builtin_crypto_vpmsumd ((__vector unsigned long + long)vdata5, (__vector unsigned long long)vconst1); + vdata5 = vec_ld(80, (__vector unsigned long long*) p); + VEC_PERM(vdata5, vdata5, vdata5, vperm_const); + GROUP_ENDING_NOP; + + v6 = vec_xor(v6, va6); + va6 = __builtin_crypto_vpmsumd ((__vector unsigned long + long)vdata6, (__vector unsigned long long)vconst1); + vdata6 = vec_ld(96, (__vector unsigned long long*) p); + VEC_PERM(vdata6, vdata6, vdata6, vperm_const); + GROUP_ENDING_NOP; + + v7 = vec_xor(v7, va7); + va7 = __builtin_crypto_vpmsumd ((__vector unsigned long + long)vdata7, (__vector unsigned long long)vconst1); + vdata7 = vec_ld(112, (__vector unsigned long long*) p); + VEC_PERM(vdata7, vdata7, vdata7, vperm_const); + + p = (char *)p + 128; + } + + /* First cool down*/ + vconst1 = vec_ld(offset, vcrc_const); + offset += 16; + + v0 = vec_xor(v0, va0); + va0 = __builtin_crypto_vpmsumd ((__vector unsigned long + long)vdata0, (__vector unsigned long long)vconst1); + GROUP_ENDING_NOP; + + v1 = vec_xor(v1, va1); + va1 = __builtin_crypto_vpmsumd ((__vector unsigned long + long)vdata1, (__vector unsigned long long)vconst1); + GROUP_ENDING_NOP; + + v2 = vec_xor(v2, va2); + va2 = __builtin_crypto_vpmsumd ((__vector unsigned long + long)vdata2, (__vector unsigned long long)vconst1); + GROUP_ENDING_NOP; + + v3 = vec_xor(v3, va3); + va3 = __builtin_crypto_vpmsumd ((__vector unsigned long + long)vdata3, (__vector unsigned long long)vconst1); + GROUP_ENDING_NOP; + + v4 = vec_xor(v4, va4); + va4 = __builtin_crypto_vpmsumd ((__vector unsigned long + long)vdata4, (__vector unsigned long long)vconst1); + GROUP_ENDING_NOP; + + v5 = vec_xor(v5, va5); + va5 = __builtin_crypto_vpmsumd ((__vector unsigned long + long)vdata5, (__vector unsigned long long)vconst1); + GROUP_ENDING_NOP; + + v6 = vec_xor(v6, va6); + va6 = __builtin_crypto_vpmsumd ((__vector unsigned long + long)vdata6, (__vector unsigned long long)vconst1); + GROUP_ENDING_NOP; + + v7 = vec_xor(v7, va7); + va7 = __builtin_crypto_vpmsumd ((__vector unsigned long + long)vdata7, (__vector unsigned long long)vconst1); + }/* else */ + + /* Second cool down. */ + v0 = vec_xor(v0, va0); + v1 = vec_xor(v1, va1); + v2 = vec_xor(v2, va2); + v3 = vec_xor(v3, va3); + v4 = vec_xor(v4, va4); + v5 = vec_xor(v5, va5); + v6 = vec_xor(v6, va6); + v7 = vec_xor(v7, va7); + +#ifdef REFLECT + /* + * vpmsumd produces a 96 bit result in the least significant bits + * of the register. Since we are bit reflected we have to shift it + * left 32 bits so it occupies the least significant bits in the + * bit reflected domain. + */ + v0 = (__vector unsigned long long)vec_sld((__vector unsigned char)v0, + (__vector unsigned char)vzero, 4); + v1 = (__vector unsigned long long)vec_sld((__vector unsigned char)v1, + (__vector unsigned char)vzero, 4); + v2 = (__vector unsigned long long)vec_sld((__vector unsigned char)v2, + (__vector unsigned char)vzero, 4); + v3 = (__vector unsigned long long)vec_sld((__vector unsigned char)v3, + (__vector unsigned char)vzero, 4); + v4 = (__vector unsigned long long)vec_sld((__vector unsigned char)v4, + (__vector unsigned char)vzero, 4); + v5 = (__vector unsigned long long)vec_sld((__vector unsigned char)v5, + (__vector unsigned char)vzero, 4); + v6 = (__vector unsigned long long)vec_sld((__vector unsigned char)v6, + (__vector unsigned char)vzero, 4); + v7 = (__vector unsigned long long)vec_sld((__vector unsigned char)v7, + (__vector unsigned char)vzero, 4); +#endif + + /* xor with the last 1024 bits. */ + va0 = vec_ld(0, (__vector unsigned long long*) p); + VEC_PERM(va0, va0, va0, vperm_const); + + va1 = vec_ld(16, (__vector unsigned long long*) p); + VEC_PERM(va1, va1, va1, vperm_const); + + va2 = vec_ld(32, (__vector unsigned long long*) p); + VEC_PERM(va2, va2, va2, vperm_const); + + va3 = vec_ld(48, (__vector unsigned long long*) p); + VEC_PERM(va3, va3, va3, vperm_const); + + va4 = vec_ld(64, (__vector unsigned long long*) p); + VEC_PERM(va4, va4, va4, vperm_const); + + va5 = vec_ld(80, (__vector unsigned long long*) p); + VEC_PERM(va5, va5, va5, vperm_const); + + va6 = vec_ld(96, (__vector unsigned long long*) p); + VEC_PERM(va6, va6, va6, vperm_const); + + va7 = vec_ld(112, (__vector unsigned long long*) p); + VEC_PERM(va7, va7, va7, vperm_const); + + p = (char *)p + 128; + + vdata0 = vec_xor(v0, va0); + vdata1 = vec_xor(v1, va1); + vdata2 = vec_xor(v2, va2); + vdata3 = vec_xor(v3, va3); + vdata4 = vec_xor(v4, va4); + vdata5 = vec_xor(v5, va5); + vdata6 = vec_xor(v6, va6); + vdata7 = vec_xor(v7, va7); + + /* Check if we have more blocks to process */ + next_block = 0; + if (length != 0) { + next_block = 1; + + /* zero v0-v7 */ + v0 = vec_xor(v0, v0); + v1 = vec_xor(v1, v1); + v2 = vec_xor(v2, v2); + v3 = vec_xor(v3, v3); + v4 = vec_xor(v4, v4); + v5 = vec_xor(v5, v5); + v6 = vec_xor(v6, v6); + v7 = vec_xor(v7, v7); + } + length = length + 128; + + } while (next_block); + + /* Calculate how many bytes we have left. */ + length = (len & 127); + + /* Calculate where in (short) constant table we need to start. */ + offset = 128 - length; + + v0 = vec_ld(offset, vcrc_short_const); + v1 = vec_ld(offset + 16, vcrc_short_const); + v2 = vec_ld(offset + 32, vcrc_short_const); + v3 = vec_ld(offset + 48, vcrc_short_const); + v4 = vec_ld(offset + 64, vcrc_short_const); + v5 = vec_ld(offset + 80, vcrc_short_const); + v6 = vec_ld(offset + 96, vcrc_short_const); + v7 = vec_ld(offset + 112, vcrc_short_const); + + offset += 128; + + v0 = (__vector unsigned long long)__builtin_crypto_vpmsumw ( + (__vector unsigned int)vdata0,(__vector unsigned int)v0); + v1 = (__vector unsigned long long)__builtin_crypto_vpmsumw ( + (__vector unsigned int)vdata1,(__vector unsigned int)v1); + v2 = (__vector unsigned long long)__builtin_crypto_vpmsumw ( + (__vector unsigned int)vdata2,(__vector unsigned int)v2); + v3 = (__vector unsigned long long)__builtin_crypto_vpmsumw ( + (__vector unsigned int)vdata3,(__vector unsigned int)v3); + v4 = (__vector unsigned long long)__builtin_crypto_vpmsumw ( + (__vector unsigned int)vdata4,(__vector unsigned int)v4); + v5 = (__vector unsigned long long)__builtin_crypto_vpmsumw ( + (__vector unsigned int)vdata5,(__vector unsigned int)v5); + v6 = (__vector unsigned long long)__builtin_crypto_vpmsumw ( + (__vector unsigned int)vdata6,(__vector unsigned int)v6); + v7 = (__vector unsigned long long)__builtin_crypto_vpmsumw ( + (__vector unsigned int)vdata7,(__vector unsigned int)v7); + + /* Now reduce the tail (0-112 bytes). */ + for (i = 0; i < length; i+=16) { + vdata0 = vec_ld(i,(__vector unsigned long long*)p); + VEC_PERM(vdata0, vdata0, vdata0, vperm_const); + va0 = vec_ld(offset + i,vcrc_short_const); + va0 = (__vector unsigned long long)__builtin_crypto_vpmsumw ( + (__vector unsigned int)vdata0,(__vector unsigned int)va0); + v0 = vec_xor(v0, va0); + } + + /* xor all parallel chunks together. */ + v0 = vec_xor(v0, v1); + v2 = vec_xor(v2, v3); + v4 = vec_xor(v4, v5); + v6 = vec_xor(v6, v7); + + v0 = vec_xor(v0, v2); + v4 = vec_xor(v4, v6); + + v0 = vec_xor(v0, v4); + } + + /* Barrett Reduction */ + vconst1 = vec_ld(0, v_Barrett_const); + vconst2 = vec_ld(16, v_Barrett_const); + + v1 = (__vector unsigned long long)vec_sld((__vector unsigned char)v0, + (__vector unsigned char)v0, 8); + v0 = vec_xor(v1,v0); + +#ifdef REFLECT + /* shift left one bit */ + vsht_splat = vec_splat_u8 (1); + v0 = (__vector unsigned long long)vec_sll ((__vector unsigned char)v0, + vsht_splat); +#endif + + v0 = vec_and(v0, vmask_64bit); + +#ifndef REFLECT + + /* + * Now for the actual algorithm. The idea is to calculate q, + * the multiple of our polynomial that we need to subtract. By + * doing the computation 2x bits higher (ie 64 bits) and shifting the + * result back down 2x bits, we round down to the nearest multiple. + */ + + /* ma */ + v1 = __builtin_crypto_vpmsumd ((__vector unsigned long long)v0, + (__vector unsigned long long)vconst1); + /* q = floor(ma/(2^64)) */ + v1 = (__vector unsigned long long)vec_sld ((__vector unsigned char)vzero, + (__vector unsigned char)v1, 8); + /* qn */ + v1 = __builtin_crypto_vpmsumd ((__vector unsigned long long)v1, + (__vector unsigned long long)vconst2); + /* a - qn, subtraction is xor in GF(2) */ + v0 = vec_xor (v0, v1); + /* + * Get the result into r3. We need to shift it left 8 bytes: + * V0 [ 0 1 2 X ] + * V0 [ 0 X 2 3 ] + */ + result = __builtin_unpack_vector_1 (v0); +#else + + /* + * The reflected version of Barrett reduction. Instead of bit + * reflecting our data (which is expensive to do), we bit reflect our + * constants and our algorithm, which means the intermediate data in + * our vector registers goes from 0-63 instead of 63-0. We can reflect + * the algorithm because we don't carry in mod 2 arithmetic. + */ + + /* bottom 32 bits of a */ + v1 = vec_and(v0, vmask_32bit); + + /* ma */ + v1 = __builtin_crypto_vpmsumd ((__vector unsigned long long)v1, + (__vector unsigned long long)vconst1); + + /* bottom 32bits of ma */ + v1 = vec_and(v1, vmask_32bit); + /* qn */ + v1 = __builtin_crypto_vpmsumd ((__vector unsigned long long)v1, + (__vector unsigned long long)vconst2); + /* a - qn, subtraction is xor in GF(2) */ + v0 = vec_xor (v0, v1); + + /* + * Since we are bit reflected, the result (ie the low 32 bits) is in + * the high 32 bits. We just need to shift it left 4 bytes + * V0 [ 0 1 X 3 ] + * V0 [ 0 X 2 3 ] + */ + + /* shift result into top 64 bits of */ + v0 = (__vector unsigned long long)vec_sld((__vector unsigned char)v0, + (__vector unsigned char)vzero, 4); + + result = __builtin_unpack_vector_0 (v0); +#endif + + return result; +} diff --git a/mysys/crc32/pcc_crc32_constants.h b/mysys/crc32/pcc_crc32_constants.h new file mode 100644 index 00000000000..2e07d2576ed --- /dev/null +++ b/mysys/crc32/pcc_crc32_constants.h @@ -0,0 +1,1206 @@ +/* +* +* THIS FILE IS GENERATED WITH +./crc32_constants -c -x -r 0x4c11db7 + +* This is from https://github.com/antonblanchard/crc32-vpmsum/ +* DO NOT MODIFY IT MANUALLY! +* +*/ + +#define CRC 0x4c11db7 +#define CRC_XOR +#define REFLECT +#define MAX_SIZE 32768 + +#ifndef __ASSEMBLER__ +#ifdef CRC_TABLE +static const unsigned int crc_table[] = { + 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, + 0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3, + 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, + 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, + 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, + 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, + 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, + 0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5, + 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, + 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, + 0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, + 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59, + 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, + 0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f, + 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, + 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, + 0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a, + 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433, + 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, + 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01, + 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, + 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, + 0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c, + 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, + 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, + 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb, + 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, + 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, + 0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086, + 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, + 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, + 0x59b33d17, 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad, + 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, + 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, + 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, + 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, + 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, + 0xf762575d, 0x806567cb, 0x196c3671, 0x6e6b06e7, + 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, + 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, + 0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, + 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b, + 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, + 0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79, + 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, + 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, + 0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, + 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, + 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, + 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713, + 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, + 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, + 0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e, + 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777, + 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, + 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45, + 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, + 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, + 0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0, + 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, + 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, + 0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf, + 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, + 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d,}; + +#endif /* CRC_TABLE */ +#ifdef POWER8_INTRINSICS + +/* Constants */ + +/* Reduce 262144 kbits to 1024 bits */ +static const __vector unsigned long long vcrc_const[255] + __attribute__((aligned (16))) = { +#ifdef __LITTLE_ENDIAN__ + /* x^261120 mod p(x)` << 1, x^261184 mod p(x)` << 1 */ + { 0x0000000099ea94a8, 0x00000001651797d2 }, + /* x^260096 mod p(x)` << 1, x^260160 mod p(x)` << 1 */ + { 0x00000000945a8420, 0x0000000021e0d56c }, + /* x^259072 mod p(x)` << 1, x^259136 mod p(x)` << 1 */ + { 0x0000000030762706, 0x000000000f95ecaa }, + /* x^258048 mod p(x)` << 1, x^258112 mod p(x)` << 1 */ + { 0x00000001a52fc582, 0x00000001ebd224ac }, + /* x^257024 mod p(x)` << 1, x^257088 mod p(x)` << 1 */ + { 0x00000001a4a7167a, 0x000000000ccb97ca }, + /* x^256000 mod p(x)` << 1, x^256064 mod p(x)` << 1 */ + { 0x000000000c18249a, 0x00000001006ec8a8 }, + /* x^254976 mod p(x)` << 1, x^255040 mod p(x)` << 1 */ + { 0x00000000a924ae7c, 0x000000014f58f196 }, + /* x^253952 mod p(x)` << 1, x^254016 mod p(x)` << 1 */ + { 0x00000001e12ccc12, 0x00000001a7192ca6 }, + /* x^252928 mod p(x)` << 1, x^252992 mod p(x)` << 1 */ + { 0x00000000a0b9d4ac, 0x000000019a64bab2 }, + /* x^251904 mod p(x)` << 1, x^251968 mod p(x)` << 1 */ + { 0x0000000095e8ddfe, 0x0000000014f4ed2e }, + /* x^250880 mod p(x)` << 1, x^250944 mod p(x)` << 1 */ + { 0x00000000233fddc4, 0x000000011092b6a2 }, + /* x^249856 mod p(x)` << 1, x^249920 mod p(x)` << 1 */ + { 0x00000001b4529b62, 0x00000000c8a1629c }, + /* x^248832 mod p(x)` << 1, x^248896 mod p(x)` << 1 */ + { 0x00000001a7fa0e64, 0x000000017bf32e8e }, + /* x^247808 mod p(x)` << 1, x^247872 mod p(x)` << 1 */ + { 0x00000001b5334592, 0x00000001f8cc6582 }, + /* x^246784 mod p(x)` << 1, x^246848 mod p(x)` << 1 */ + { 0x000000011f8ee1b4, 0x000000008631ddf0 }, + /* x^245760 mod p(x)` << 1, x^245824 mod p(x)` << 1 */ + { 0x000000006252e632, 0x000000007e5a76d0 }, + /* x^244736 mod p(x)` << 1, x^244800 mod p(x)` << 1 */ + { 0x00000000ab973e84, 0x000000002b09b31c }, + /* x^243712 mod p(x)` << 1, x^243776 mod p(x)` << 1 */ + { 0x000000007734f5ec, 0x00000001b2df1f84 }, + /* x^242688 mod p(x)` << 1, x^242752 mod p(x)` << 1 */ + { 0x000000007c547798, 0x00000001d6f56afc }, + /* x^241664 mod p(x)` << 1, x^241728 mod p(x)` << 1 */ + { 0x000000007ec40210, 0x00000001b9b5e70c }, + /* x^240640 mod p(x)` << 1, x^240704 mod p(x)` << 1 */ + { 0x00000001ab1695a8, 0x0000000034b626d2 }, + /* x^239616 mod p(x)` << 1, x^239680 mod p(x)` << 1 */ + { 0x0000000090494bba, 0x000000014c53479a }, + /* x^238592 mod p(x)` << 1, x^238656 mod p(x)` << 1 */ + { 0x00000001123fb816, 0x00000001a6d179a4 }, + /* x^237568 mod p(x)` << 1, x^237632 mod p(x)` << 1 */ + { 0x00000001e188c74c, 0x000000015abd16b4 }, + /* x^236544 mod p(x)` << 1, x^236608 mod p(x)` << 1 */ + { 0x00000001c2d3451c, 0x00000000018f9852 }, + /* x^235520 mod p(x)` << 1, x^235584 mod p(x)` << 1 */ + { 0x00000000f55cf1ca, 0x000000001fb3084a }, + /* x^234496 mod p(x)` << 1, x^234560 mod p(x)` << 1 */ + { 0x00000001a0531540, 0x00000000c53dfb04 }, + /* x^233472 mod p(x)` << 1, x^233536 mod p(x)` << 1 */ + { 0x0000000132cd7ebc, 0x00000000e10c9ad6 }, + /* x^232448 mod p(x)` << 1, x^232512 mod p(x)` << 1 */ + { 0x0000000073ab7f36, 0x0000000025aa994a }, + /* x^231424 mod p(x)` << 1, x^231488 mod p(x)` << 1 */ + { 0x0000000041aed1c2, 0x00000000fa3a74c4 }, + /* x^230400 mod p(x)` << 1, x^230464 mod p(x)` << 1 */ + { 0x0000000136c53800, 0x0000000033eb3f40 }, + /* x^229376 mod p(x)` << 1, x^229440 mod p(x)` << 1 */ + { 0x0000000126835a30, 0x000000017193f296 }, + /* x^228352 mod p(x)` << 1, x^228416 mod p(x)` << 1 */ + { 0x000000006241b502, 0x0000000043f6c86a }, + /* x^227328 mod p(x)` << 1, x^227392 mod p(x)` << 1 */ + { 0x00000000d5196ad4, 0x000000016b513ec6 }, + /* x^226304 mod p(x)` << 1, x^226368 mod p(x)` << 1 */ + { 0x000000009cfa769a, 0x00000000c8f25b4e }, + /* x^225280 mod p(x)` << 1, x^225344 mod p(x)` << 1 */ + { 0x00000000920e5df4, 0x00000001a45048ec }, + /* x^224256 mod p(x)` << 1, x^224320 mod p(x)` << 1 */ + { 0x0000000169dc310e, 0x000000000c441004 }, + /* x^223232 mod p(x)` << 1, x^223296 mod p(x)` << 1 */ + { 0x0000000009fc331c, 0x000000000e17cad6 }, + /* x^222208 mod p(x)` << 1, x^222272 mod p(x)` << 1 */ + { 0x000000010d94a81e, 0x00000001253ae964 }, + /* x^221184 mod p(x)` << 1, x^221248 mod p(x)` << 1 */ + { 0x0000000027a20ab2, 0x00000001d7c88ebc }, + /* x^220160 mod p(x)` << 1, x^220224 mod p(x)` << 1 */ + { 0x0000000114f87504, 0x00000001e7ca913a }, + /* x^219136 mod p(x)` << 1, x^219200 mod p(x)` << 1 */ + { 0x000000004b076d96, 0x0000000033ed078a }, + /* x^218112 mod p(x)` << 1, x^218176 mod p(x)` << 1 */ + { 0x00000000da4d1e74, 0x00000000e1839c78 }, + /* x^217088 mod p(x)` << 1, x^217152 mod p(x)` << 1 */ + { 0x000000001b81f672, 0x00000001322b267e }, + /* x^216064 mod p(x)` << 1, x^216128 mod p(x)` << 1 */ + { 0x000000009367c988, 0x00000000638231b6 }, + /* x^215040 mod p(x)` << 1, x^215104 mod p(x)` << 1 */ + { 0x00000001717214ca, 0x00000001ee7f16f4 }, + /* x^214016 mod p(x)` << 1, x^214080 mod p(x)` << 1 */ + { 0x000000009f47d820, 0x0000000117d9924a }, + /* x^212992 mod p(x)` << 1, x^213056 mod p(x)` << 1 */ + { 0x000000010d9a47d2, 0x00000000e1a9e0c4 }, + /* x^211968 mod p(x)` << 1, x^212032 mod p(x)` << 1 */ + { 0x00000000a696c58c, 0x00000001403731dc }, + /* x^210944 mod p(x)` << 1, x^211008 mod p(x)` << 1 */ + { 0x000000002aa28ec6, 0x00000001a5ea9682 }, + /* x^209920 mod p(x)` << 1, x^209984 mod p(x)` << 1 */ + { 0x00000001fe18fd9a, 0x0000000101c5c578 }, + /* x^208896 mod p(x)` << 1, x^208960 mod p(x)` << 1 */ + { 0x000000019d4fc1ae, 0x00000000dddf6494 }, + /* x^207872 mod p(x)` << 1, x^207936 mod p(x)` << 1 */ + { 0x00000001ba0e3dea, 0x00000000f1c3db28 }, + /* x^206848 mod p(x)` << 1, x^206912 mod p(x)` << 1 */ + { 0x0000000074b59a5e, 0x000000013112fb9c }, + /* x^205824 mod p(x)` << 1, x^205888 mod p(x)` << 1 */ + { 0x00000000f2b5ea98, 0x00000000b680b906 }, + /* x^204800 mod p(x)` << 1, x^204864 mod p(x)` << 1 */ + { 0x0000000187132676, 0x000000001a282932 }, + /* x^203776 mod p(x)` << 1, x^203840 mod p(x)` << 1 */ + { 0x000000010a8c6ad4, 0x0000000089406e7e }, + /* x^202752 mod p(x)` << 1, x^202816 mod p(x)` << 1 */ + { 0x00000001e21dfe70, 0x00000001def6be8c }, + /* x^201728 mod p(x)` << 1, x^201792 mod p(x)` << 1 */ + { 0x00000001da0050e4, 0x0000000075258728 }, + /* x^200704 mod p(x)` << 1, x^200768 mod p(x)` << 1 */ + { 0x00000000772172ae, 0x000000019536090a }, + /* x^199680 mod p(x)` << 1, x^199744 mod p(x)` << 1 */ + { 0x00000000e47724aa, 0x00000000f2455bfc }, + /* x^198656 mod p(x)` << 1, x^198720 mod p(x)` << 1 */ + { 0x000000003cd63ac4, 0x000000018c40baf4 }, + /* x^197632 mod p(x)` << 1, x^197696 mod p(x)` << 1 */ + { 0x00000001bf47d352, 0x000000004cd390d4 }, + /* x^196608 mod p(x)` << 1, x^196672 mod p(x)` << 1 */ + { 0x000000018dc1d708, 0x00000001e4ece95a }, + /* x^195584 mod p(x)` << 1, x^195648 mod p(x)` << 1 */ + { 0x000000002d4620a4, 0x000000001a3ee918 }, + /* x^194560 mod p(x)` << 1, x^194624 mod p(x)` << 1 */ + { 0x0000000058fd1740, 0x000000007c652fb8 }, + /* x^193536 mod p(x)` << 1, x^193600 mod p(x)` << 1 */ + { 0x00000000dadd9bfc, 0x000000011c67842c }, + /* x^192512 mod p(x)` << 1, x^192576 mod p(x)` << 1 */ + { 0x00000001ea2140be, 0x00000000254f759c }, + /* x^191488 mod p(x)` << 1, x^191552 mod p(x)` << 1 */ + { 0x000000009de128ba, 0x000000007ece94ca }, + /* x^190464 mod p(x)` << 1, x^190528 mod p(x)` << 1 */ + { 0x000000013ac3aa8e, 0x0000000038f258c2 }, + /* x^189440 mod p(x)` << 1, x^189504 mod p(x)` << 1 */ + { 0x0000000099980562, 0x00000001cdf17b00 }, + /* x^188416 mod p(x)` << 1, x^188480 mod p(x)` << 1 */ + { 0x00000001c1579c86, 0x000000011f882c16 }, + /* x^187392 mod p(x)` << 1, x^187456 mod p(x)` << 1 */ + { 0x0000000068dbbf94, 0x0000000100093fc8 }, + /* x^186368 mod p(x)` << 1, x^186432 mod p(x)` << 1 */ + { 0x000000004509fb04, 0x00000001cd684f16 }, + /* x^185344 mod p(x)` << 1, x^185408 mod p(x)` << 1 */ + { 0x00000001202f6398, 0x000000004bc6a70a }, + /* x^184320 mod p(x)` << 1, x^184384 mod p(x)` << 1 */ + { 0x000000013aea243e, 0x000000004fc7e8e4 }, + /* x^183296 mod p(x)` << 1, x^183360 mod p(x)` << 1 */ + { 0x00000001b4052ae6, 0x0000000130103f1c }, + /* x^182272 mod p(x)` << 1, x^182336 mod p(x)` << 1 */ + { 0x00000001cd2a0ae8, 0x0000000111b0024c }, + /* x^181248 mod p(x)` << 1, x^181312 mod p(x)` << 1 */ + { 0x00000001fe4aa8b4, 0x000000010b3079da }, + /* x^180224 mod p(x)` << 1, x^180288 mod p(x)` << 1 */ + { 0x00000001d1559a42, 0x000000010192bcc2 }, + /* x^179200 mod p(x)` << 1, x^179264 mod p(x)` << 1 */ + { 0x00000001f3e05ecc, 0x0000000074838d50 }, + /* x^178176 mod p(x)` << 1, x^178240 mod p(x)` << 1 */ + { 0x0000000104ddd2cc, 0x000000001b20f520 }, + /* x^177152 mod p(x)` << 1, x^177216 mod p(x)` << 1 */ + { 0x000000015393153c, 0x0000000050c3590a }, + /* x^176128 mod p(x)` << 1, x^176192 mod p(x)` << 1 */ + { 0x0000000057e942c6, 0x00000000b41cac8e }, + /* x^175104 mod p(x)` << 1, x^175168 mod p(x)` << 1 */ + { 0x000000012c633850, 0x000000000c72cc78 }, + /* x^174080 mod p(x)` << 1, x^174144 mod p(x)` << 1 */ + { 0x00000000ebcaae4c, 0x0000000030cdb032 }, + /* x^173056 mod p(x)` << 1, x^173120 mod p(x)` << 1 */ + { 0x000000013ee532a6, 0x000000013e09fc32 }, + /* x^172032 mod p(x)` << 1, x^172096 mod p(x)` << 1 */ + { 0x00000001bf0cbc7e, 0x000000001ed624d2 }, + /* x^171008 mod p(x)` << 1, x^171072 mod p(x)` << 1 */ + { 0x00000000d50b7a5a, 0x00000000781aee1a }, + /* x^169984 mod p(x)` << 1, x^170048 mod p(x)` << 1 */ + { 0x0000000002fca6e8, 0x00000001c4d8348c }, + /* x^168960 mod p(x)` << 1, x^169024 mod p(x)` << 1 */ + { 0x000000007af40044, 0x0000000057a40336 }, + /* x^167936 mod p(x)` << 1, x^168000 mod p(x)` << 1 */ + { 0x0000000016178744, 0x0000000085544940 }, + /* x^166912 mod p(x)` << 1, x^166976 mod p(x)` << 1 */ + { 0x000000014c177458, 0x000000019cd21e80 }, + /* x^165888 mod p(x)` << 1, x^165952 mod p(x)` << 1 */ + { 0x000000011b6ddf04, 0x000000013eb95bc0 }, + /* x^164864 mod p(x)` << 1, x^164928 mod p(x)` << 1 */ + { 0x00000001f3e29ccc, 0x00000001dfc9fdfc }, + /* x^163840 mod p(x)` << 1, x^163904 mod p(x)` << 1 */ + { 0x0000000135ae7562, 0x00000000cd028bc2 }, + /* x^162816 mod p(x)` << 1, x^162880 mod p(x)` << 1 */ + { 0x0000000190ef812c, 0x0000000090db8c44 }, + /* x^161792 mod p(x)` << 1, x^161856 mod p(x)` << 1 */ + { 0x0000000067a2c786, 0x000000010010a4ce }, + /* x^160768 mod p(x)` << 1, x^160832 mod p(x)` << 1 */ + { 0x0000000048b9496c, 0x00000001c8f4c72c }, + /* x^159744 mod p(x)` << 1, x^159808 mod p(x)` << 1 */ + { 0x000000015a422de6, 0x000000001c26170c }, + /* x^158720 mod p(x)` << 1, x^158784 mod p(x)` << 1 */ + { 0x00000001ef0e3640, 0x00000000e3fccf68 }, + /* x^157696 mod p(x)` << 1, x^157760 mod p(x)` << 1 */ + { 0x00000001006d2d26, 0x00000000d513ed24 }, + /* x^156672 mod p(x)` << 1, x^156736 mod p(x)` << 1 */ + { 0x00000001170d56d6, 0x00000000141beada }, + /* x^155648 mod p(x)` << 1, x^155712 mod p(x)` << 1 */ + { 0x00000000a5fb613c, 0x000000011071aea0 }, + /* x^154624 mod p(x)` << 1, x^154688 mod p(x)` << 1 */ + { 0x0000000040bbf7fc, 0x000000012e19080a }, + /* x^153600 mod p(x)` << 1, x^153664 mod p(x)` << 1 */ + { 0x000000016ac3a5b2, 0x0000000100ecf826 }, + /* x^152576 mod p(x)` << 1, x^152640 mod p(x)` << 1 */ + { 0x00000000abf16230, 0x0000000069b09412 }, + /* x^151552 mod p(x)` << 1, x^151616 mod p(x)` << 1 */ + { 0x00000001ebe23fac, 0x0000000122297bac }, + /* x^150528 mod p(x)` << 1, x^150592 mod p(x)` << 1 */ + { 0x000000008b6a0894, 0x00000000e9e4b068 }, + /* x^149504 mod p(x)` << 1, x^149568 mod p(x)` << 1 */ + { 0x00000001288ea478, 0x000000004b38651a }, + /* x^148480 mod p(x)` << 1, x^148544 mod p(x)` << 1 */ + { 0x000000016619c442, 0x00000001468360e2 }, + /* x^147456 mod p(x)` << 1, x^147520 mod p(x)` << 1 */ + { 0x0000000086230038, 0x00000000121c2408 }, + /* x^146432 mod p(x)` << 1, x^146496 mod p(x)` << 1 */ + { 0x000000017746a756, 0x00000000da7e7d08 }, + /* x^145408 mod p(x)` << 1, x^145472 mod p(x)` << 1 */ + { 0x0000000191b8f8f8, 0x00000001058d7652 }, + /* x^144384 mod p(x)` << 1, x^144448 mod p(x)` << 1 */ + { 0x000000008e167708, 0x000000014a098a90 }, + /* x^143360 mod p(x)` << 1, x^143424 mod p(x)` << 1 */ + { 0x0000000148b22d54, 0x0000000020dbe72e }, + /* x^142336 mod p(x)` << 1, x^142400 mod p(x)` << 1 */ + { 0x0000000044ba2c3c, 0x000000011e7323e8 }, + /* x^141312 mod p(x)` << 1, x^141376 mod p(x)` << 1 */ + { 0x00000000b54d2b52, 0x00000000d5d4bf94 }, + /* x^140288 mod p(x)` << 1, x^140352 mod p(x)` << 1 */ + { 0x0000000005a4fd8a, 0x0000000199d8746c }, + /* x^139264 mod p(x)` << 1, x^139328 mod p(x)` << 1 */ + { 0x0000000139f9fc46, 0x00000000ce9ca8a0 }, + /* x^138240 mod p(x)` << 1, x^138304 mod p(x)` << 1 */ + { 0x000000015a1fa824, 0x00000000136edece }, + /* x^137216 mod p(x)` << 1, x^137280 mod p(x)` << 1 */ + { 0x000000000a61ae4c, 0x000000019b92a068 }, + /* x^136192 mod p(x)` << 1, x^136256 mod p(x)` << 1 */ + { 0x0000000145e9113e, 0x0000000071d62206 }, + /* x^135168 mod p(x)` << 1, x^135232 mod p(x)` << 1 */ + { 0x000000006a348448, 0x00000000dfc50158 }, + /* x^134144 mod p(x)` << 1, x^134208 mod p(x)` << 1 */ + { 0x000000004d80a08c, 0x00000001517626bc }, + /* x^133120 mod p(x)` << 1, x^133184 mod p(x)` << 1 */ + { 0x000000014b6837a0, 0x0000000148d1e4fa }, + /* x^132096 mod p(x)` << 1, x^132160 mod p(x)` << 1 */ + { 0x000000016896a7fc, 0x0000000094d8266e }, + /* x^131072 mod p(x)` << 1, x^131136 mod p(x)` << 1 */ + { 0x000000014f187140, 0x00000000606c5e34 }, + /* x^130048 mod p(x)` << 1, x^130112 mod p(x)` << 1 */ + { 0x000000019581b9da, 0x000000019766beaa }, + /* x^129024 mod p(x)` << 1, x^129088 mod p(x)` << 1 */ + { 0x00000001091bc984, 0x00000001d80c506c }, + /* x^128000 mod p(x)` << 1, x^128064 mod p(x)` << 1 */ + { 0x000000001067223c, 0x000000001e73837c }, + /* x^126976 mod p(x)` << 1, x^127040 mod p(x)` << 1 */ + { 0x00000001ab16ea02, 0x0000000064d587de }, + /* x^125952 mod p(x)` << 1, x^126016 mod p(x)` << 1 */ + { 0x000000013c4598a8, 0x00000000f4a507b0 }, + /* x^124928 mod p(x)` << 1, x^124992 mod p(x)` << 1 */ + { 0x00000000b3735430, 0x0000000040e342fc }, + /* x^123904 mod p(x)` << 1, x^123968 mod p(x)` << 1 */ + { 0x00000001bb3fc0c0, 0x00000001d5ad9c3a }, + /* x^122880 mod p(x)` << 1, x^122944 mod p(x)` << 1 */ + { 0x00000001570ae19c, 0x0000000094a691a4 }, + /* x^121856 mod p(x)` << 1, x^121920 mod p(x)` << 1 */ + { 0x00000001ea910712, 0x00000001271ecdfa }, + /* x^120832 mod p(x)` << 1, x^120896 mod p(x)` << 1 */ + { 0x0000000167127128, 0x000000009e54475a }, + /* x^119808 mod p(x)` << 1, x^119872 mod p(x)` << 1 */ + { 0x0000000019e790a2, 0x00000000c9c099ee }, + /* x^118784 mod p(x)` << 1, x^118848 mod p(x)` << 1 */ + { 0x000000003788f710, 0x000000009a2f736c }, + /* x^117760 mod p(x)` << 1, x^117824 mod p(x)` << 1 */ + { 0x00000001682a160e, 0x00000000bb9f4996 }, + /* x^116736 mod p(x)` << 1, x^116800 mod p(x)` << 1 */ + { 0x000000007f0ebd2e, 0x00000001db688050 }, + /* x^115712 mod p(x)` << 1, x^115776 mod p(x)` << 1 */ + { 0x000000002b032080, 0x00000000e9b10af4 }, + /* x^114688 mod p(x)` << 1, x^114752 mod p(x)` << 1 */ + { 0x00000000cfd1664a, 0x000000012d4545e4 }, + /* x^113664 mod p(x)` << 1, x^113728 mod p(x)` << 1 */ + { 0x00000000aa1181c2, 0x000000000361139c }, + /* x^112640 mod p(x)` << 1, x^112704 mod p(x)` << 1 */ + { 0x00000000ddd08002, 0x00000001a5a1a3a8 }, + /* x^111616 mod p(x)` << 1, x^111680 mod p(x)` << 1 */ + { 0x00000000e8dd0446, 0x000000006844e0b0 }, + /* x^110592 mod p(x)` << 1, x^110656 mod p(x)` << 1 */ + { 0x00000001bbd94a00, 0x00000000c3762f28 }, + /* x^109568 mod p(x)` << 1, x^109632 mod p(x)` << 1 */ + { 0x00000000ab6cd180, 0x00000001d26287a2 }, + /* x^108544 mod p(x)` << 1, x^108608 mod p(x)` << 1 */ + { 0x0000000031803ce2, 0x00000001f6f0bba8 }, + /* x^107520 mod p(x)` << 1, x^107584 mod p(x)` << 1 */ + { 0x0000000024f40b0c, 0x000000002ffabd62 }, + /* x^106496 mod p(x)` << 1, x^106560 mod p(x)` << 1 */ + { 0x00000001ba1d9834, 0x00000000fb4516b8 }, + /* x^105472 mod p(x)` << 1, x^105536 mod p(x)` << 1 */ + { 0x0000000104de61aa, 0x000000018cfa961c }, + /* x^104448 mod p(x)` << 1, x^104512 mod p(x)` << 1 */ + { 0x0000000113e40d46, 0x000000019e588d52 }, + /* x^103424 mod p(x)` << 1, x^103488 mod p(x)` << 1 */ + { 0x00000001415598a0, 0x00000001180f0bbc }, + /* x^102400 mod p(x)` << 1, x^102464 mod p(x)` << 1 */ + { 0x00000000bf6c8c90, 0x00000000e1d9177a }, + /* x^101376 mod p(x)` << 1, x^101440 mod p(x)` << 1 */ + { 0x00000001788b0504, 0x0000000105abc27c }, + /* x^100352 mod p(x)` << 1, x^100416 mod p(x)` << 1 */ + { 0x0000000038385d02, 0x00000000972e4a58 }, + /* x^99328 mod p(x)` << 1, x^99392 mod p(x)` << 1 */ + { 0x00000001b6c83844, 0x0000000183499a5e }, + /* x^98304 mod p(x)` << 1, x^98368 mod p(x)` << 1 */ + { 0x0000000051061a8a, 0x00000001c96a8cca }, + /* x^97280 mod p(x)` << 1, x^97344 mod p(x)` << 1 */ + { 0x000000017351388a, 0x00000001a1a5b60c }, + /* x^96256 mod p(x)` << 1, x^96320 mod p(x)` << 1 */ + { 0x0000000132928f92, 0x00000000e4b6ac9c }, + /* x^95232 mod p(x)` << 1, x^95296 mod p(x)` << 1 */ + { 0x00000000e6b4f48a, 0x00000001807e7f5a }, + /* x^94208 mod p(x)` << 1, x^94272 mod p(x)` << 1 */ + { 0x0000000039d15e90, 0x000000017a7e3bc8 }, + /* x^93184 mod p(x)` << 1, x^93248 mod p(x)` << 1 */ + { 0x00000000312d6074, 0x00000000d73975da }, + /* x^92160 mod p(x)` << 1, x^92224 mod p(x)` << 1 */ + { 0x000000017bbb2cc4, 0x000000017375d038 }, + /* x^91136 mod p(x)` << 1, x^91200 mod p(x)` << 1 */ + { 0x000000016ded3e18, 0x00000000193680bc }, + /* x^90112 mod p(x)` << 1, x^90176 mod p(x)` << 1 */ + { 0x00000000f1638b16, 0x00000000999b06f6 }, + /* x^89088 mod p(x)` << 1, x^89152 mod p(x)` << 1 */ + { 0x00000001d38b9ecc, 0x00000001f685d2b8 }, + /* x^88064 mod p(x)` << 1, x^88128 mod p(x)` << 1 */ + { 0x000000018b8d09dc, 0x00000001f4ecbed2 }, + /* x^87040 mod p(x)` << 1, x^87104 mod p(x)` << 1 */ + { 0x00000000e7bc27d2, 0x00000000ba16f1a0 }, + /* x^86016 mod p(x)` << 1, x^86080 mod p(x)` << 1 */ + { 0x00000000275e1e96, 0x0000000115aceac4 }, + /* x^84992 mod p(x)` << 1, x^85056 mod p(x)` << 1 */ + { 0x00000000e2e3031e, 0x00000001aeff6292 }, + /* x^83968 mod p(x)` << 1, x^84032 mod p(x)` << 1 */ + { 0x00000001041c84d8, 0x000000009640124c }, + /* x^82944 mod p(x)` << 1, x^83008 mod p(x)` << 1 */ + { 0x00000000706ce672, 0x0000000114f41f02 }, + /* x^81920 mod p(x)` << 1, x^81984 mod p(x)` << 1 */ + { 0x000000015d5070da, 0x000000009c5f3586 }, + /* x^80896 mod p(x)` << 1, x^80960 mod p(x)` << 1 */ + { 0x0000000038f9493a, 0x00000001878275fa }, + /* x^79872 mod p(x)` << 1, x^79936 mod p(x)` << 1 */ + { 0x00000000a3348a76, 0x00000000ddc42ce8 }, + /* x^78848 mod p(x)` << 1, x^78912 mod p(x)` << 1 */ + { 0x00000001ad0aab92, 0x0000000181d2c73a }, + /* x^77824 mod p(x)` << 1, x^77888 mod p(x)` << 1 */ + { 0x000000019e85f712, 0x0000000141c9320a }, + /* x^76800 mod p(x)` << 1, x^76864 mod p(x)` << 1 */ + { 0x000000005a871e76, 0x000000015235719a }, + /* x^75776 mod p(x)` << 1, x^75840 mod p(x)` << 1 */ + { 0x000000017249c662, 0x00000000be27d804 }, + /* x^74752 mod p(x)` << 1, x^74816 mod p(x)` << 1 */ + { 0x000000003a084712, 0x000000006242d45a }, + /* x^73728 mod p(x)` << 1, x^73792 mod p(x)` << 1 */ + { 0x00000000ed438478, 0x000000009a53638e }, + /* x^72704 mod p(x)` << 1, x^72768 mod p(x)` << 1 */ + { 0x00000000abac34cc, 0x00000001001ecfb6 }, + /* x^71680 mod p(x)` << 1, x^71744 mod p(x)` << 1 */ + { 0x000000005f35ef3e, 0x000000016d7c2d64 }, + /* x^70656 mod p(x)` << 1, x^70720 mod p(x)` << 1 */ + { 0x0000000047d6608c, 0x00000001d0ce46c0 }, + /* x^69632 mod p(x)` << 1, x^69696 mod p(x)` << 1 */ + { 0x000000002d01470e, 0x0000000124c907b4 }, + /* x^68608 mod p(x)` << 1, x^68672 mod p(x)` << 1 */ + { 0x0000000158bbc7b0, 0x0000000018a555ca }, + /* x^67584 mod p(x)` << 1, x^67648 mod p(x)` << 1 */ + { 0x00000000c0a23e8e, 0x000000006b0980bc }, + /* x^66560 mod p(x)` << 1, x^66624 mod p(x)` << 1 */ + { 0x00000001ebd85c88, 0x000000008bbba964 }, + /* x^65536 mod p(x)` << 1, x^65600 mod p(x)` << 1 */ + { 0x000000019ee20bb2, 0x00000001070a5a1e }, + /* x^64512 mod p(x)` << 1, x^64576 mod p(x)` << 1 */ + { 0x00000001acabf2d6, 0x000000002204322a }, + /* x^63488 mod p(x)` << 1, x^63552 mod p(x)` << 1 */ + { 0x00000001b7963d56, 0x00000000a27524d0 }, + /* x^62464 mod p(x)` << 1, x^62528 mod p(x)` << 1 */ + { 0x000000017bffa1fe, 0x0000000020b1e4ba }, + /* x^61440 mod p(x)` << 1, x^61504 mod p(x)` << 1 */ + { 0x000000001f15333e, 0x0000000032cc27fc }, + /* x^60416 mod p(x)` << 1, x^60480 mod p(x)` << 1 */ + { 0x000000018593129e, 0x0000000044dd22b8 }, + /* x^59392 mod p(x)` << 1, x^59456 mod p(x)` << 1 */ + { 0x000000019cb32602, 0x00000000dffc9e0a }, + /* x^58368 mod p(x)` << 1, x^58432 mod p(x)` << 1 */ + { 0x0000000142b05cc8, 0x00000001b7a0ed14 }, + /* x^57344 mod p(x)` << 1, x^57408 mod p(x)` << 1 */ + { 0x00000001be49e7a4, 0x00000000c7842488 }, + /* x^56320 mod p(x)` << 1, x^56384 mod p(x)` << 1 */ + { 0x0000000108f69d6c, 0x00000001c02a4fee }, + /* x^55296 mod p(x)` << 1, x^55360 mod p(x)` << 1 */ + { 0x000000006c0971f0, 0x000000003c273778 }, + /* x^54272 mod p(x)` << 1, x^54336 mod p(x)` << 1 */ + { 0x000000005b16467a, 0x00000001d63f8894 }, + /* x^53248 mod p(x)` << 1, x^53312 mod p(x)` << 1 */ + { 0x00000001551a628e, 0x000000006be557d6 }, + /* x^52224 mod p(x)` << 1, x^52288 mod p(x)` << 1 */ + { 0x000000019e42ea92, 0x000000006a7806ea }, + /* x^51200 mod p(x)` << 1, x^51264 mod p(x)` << 1 */ + { 0x000000012fa83ff2, 0x000000016155aa0c }, + /* x^50176 mod p(x)` << 1, x^50240 mod p(x)` << 1 */ + { 0x000000011ca9cde0, 0x00000000908650ac }, + /* x^49152 mod p(x)` << 1, x^49216 mod p(x)` << 1 */ + { 0x00000000c8e5cd74, 0x00000000aa5a8084 }, + /* x^48128 mod p(x)` << 1, x^48192 mod p(x)` << 1 */ + { 0x0000000096c27f0c, 0x0000000191bb500a }, + /* x^47104 mod p(x)` << 1, x^47168 mod p(x)` << 1 */ + { 0x000000002baed926, 0x0000000064e9bed0 }, + /* x^46080 mod p(x)` << 1, x^46144 mod p(x)` << 1 */ + { 0x000000017c8de8d2, 0x000000009444f302 }, + /* x^45056 mod p(x)` << 1, x^45120 mod p(x)` << 1 */ + { 0x00000000d43d6068, 0x000000019db07d3c }, + /* x^44032 mod p(x)` << 1, x^44096 mod p(x)` << 1 */ + { 0x00000000cb2c4b26, 0x00000001359e3e6e }, + /* x^43008 mod p(x)` << 1, x^43072 mod p(x)` << 1 */ + { 0x0000000145b8da26, 0x00000001e4f10dd2 }, + /* x^41984 mod p(x)` << 1, x^42048 mod p(x)` << 1 */ + { 0x000000018fff4b08, 0x0000000124f5735e }, + /* x^40960 mod p(x)` << 1, x^41024 mod p(x)` << 1 */ + { 0x0000000150b58ed0, 0x0000000124760a4c }, + /* x^39936 mod p(x)` << 1, x^40000 mod p(x)` << 1 */ + { 0x00000001549f39bc, 0x000000000f1fc186 }, + /* x^38912 mod p(x)` << 1, x^38976 mod p(x)` << 1 */ + { 0x00000000ef4d2f42, 0x00000000150e4cc4 }, + /* x^37888 mod p(x)` << 1, x^37952 mod p(x)` << 1 */ + { 0x00000001b1468572, 0x000000002a6204e8 }, + /* x^36864 mod p(x)` << 1, x^36928 mod p(x)` << 1 */ + { 0x000000013d7403b2, 0x00000000beb1d432 }, + /* x^35840 mod p(x)` << 1, x^35904 mod p(x)` << 1 */ + { 0x00000001a4681842, 0x0000000135f3f1f0 }, + /* x^34816 mod p(x)` << 1, x^34880 mod p(x)` << 1 */ + { 0x0000000167714492, 0x0000000074fe2232 }, + /* x^33792 mod p(x)` << 1, x^33856 mod p(x)` << 1 */ + { 0x00000001e599099a, 0x000000001ac6e2ba }, + /* x^32768 mod p(x)` << 1, x^32832 mod p(x)` << 1 */ + { 0x00000000fe128194, 0x0000000013fca91e }, + /* x^31744 mod p(x)` << 1, x^31808 mod p(x)` << 1 */ + { 0x0000000077e8b990, 0x0000000183f4931e }, + /* x^30720 mod p(x)` << 1, x^30784 mod p(x)` << 1 */ + { 0x00000001a267f63a, 0x00000000b6d9b4e4 }, + /* x^29696 mod p(x)` << 1, x^29760 mod p(x)` << 1 */ + { 0x00000001945c245a, 0x00000000b5188656 }, + /* x^28672 mod p(x)` << 1, x^28736 mod p(x)` << 1 */ + { 0x0000000149002e76, 0x0000000027a81a84 }, + /* x^27648 mod p(x)` << 1, x^27712 mod p(x)` << 1 */ + { 0x00000001bb8310a4, 0x0000000125699258 }, + /* x^26624 mod p(x)` << 1, x^26688 mod p(x)` << 1 */ + { 0x000000019ec60bcc, 0x00000001b23de796 }, + /* x^25600 mod p(x)` << 1, x^25664 mod p(x)` << 1 */ + { 0x000000012d8590ae, 0x00000000fe4365dc }, + /* x^24576 mod p(x)` << 1, x^24640 mod p(x)` << 1 */ + { 0x0000000065b00684, 0x00000000c68f497a }, + /* x^23552 mod p(x)` << 1, x^23616 mod p(x)` << 1 */ + { 0x000000015e5aeadc, 0x00000000fbf521ee }, + /* x^22528 mod p(x)` << 1, x^22592 mod p(x)` << 1 */ + { 0x00000000b77ff2b0, 0x000000015eac3378 }, + /* x^21504 mod p(x)` << 1, x^21568 mod p(x)` << 1 */ + { 0x0000000188da2ff6, 0x0000000134914b90 }, + /* x^20480 mod p(x)` << 1, x^20544 mod p(x)` << 1 */ + { 0x0000000063da929a, 0x0000000016335cfe }, + /* x^19456 mod p(x)` << 1, x^19520 mod p(x)` << 1 */ + { 0x00000001389caa80, 0x000000010372d10c }, + /* x^18432 mod p(x)` << 1, x^18496 mod p(x)` << 1 */ + { 0x000000013db599d2, 0x000000015097b908 }, + /* x^17408 mod p(x)` << 1, x^17472 mod p(x)` << 1 */ + { 0x0000000122505a86, 0x00000001227a7572 }, + /* x^16384 mod p(x)` << 1, x^16448 mod p(x)` << 1 */ + { 0x000000016bd72746, 0x000000009a8f75c0 }, + /* x^15360 mod p(x)` << 1, x^15424 mod p(x)` << 1 */ + { 0x00000001c3faf1d4, 0x00000000682c77a2 }, + /* x^14336 mod p(x)` << 1, x^14400 mod p(x)` << 1 */ + { 0x00000001111c826c, 0x00000000231f091c }, + /* x^13312 mod p(x)` << 1, x^13376 mod p(x)` << 1 */ + { 0x00000000153e9fb2, 0x000000007d4439f2 }, + /* x^12288 mod p(x)` << 1, x^12352 mod p(x)` << 1 */ + { 0x000000002b1f7b60, 0x000000017e221efc }, + /* x^11264 mod p(x)` << 1, x^11328 mod p(x)` << 1 */ + { 0x00000000b1dba570, 0x0000000167457c38 }, + /* x^10240 mod p(x)` << 1, x^10304 mod p(x)` << 1 */ + { 0x00000001f6397b76, 0x00000000bdf081c4 }, + /* x^9216 mod p(x)` << 1, x^9280 mod p(x)` << 1 */ + { 0x0000000156335214, 0x000000016286d6b0 }, + /* x^8192 mod p(x)` << 1, x^8256 mod p(x)` << 1 */ + { 0x00000001d70e3986, 0x00000000c84f001c }, + /* x^7168 mod p(x)` << 1, x^7232 mod p(x)` << 1 */ + { 0x000000003701a774, 0x0000000064efe7c0 }, + /* x^6144 mod p(x)` << 1, x^6208 mod p(x)` << 1 */ + { 0x00000000ac81ef72, 0x000000000ac2d904 }, + /* x^5120 mod p(x)` << 1, x^5184 mod p(x)` << 1 */ + { 0x0000000133212464, 0x00000000fd226d14 }, + /* x^4096 mod p(x)` << 1, x^4160 mod p(x)` << 1 */ + { 0x00000000e4e45610, 0x000000011cfd42e0 }, + /* x^3072 mod p(x)` << 1, x^3136 mod p(x)` << 1 */ + { 0x000000000c1bd370, 0x000000016e5a5678 }, + /* x^2048 mod p(x)` << 1, x^2112 mod p(x)` << 1 */ + { 0x00000001a7b9e7a6, 0x00000001d888fe22 }, + /* x^1024 mod p(x)` << 1, x^1088 mod p(x)` << 1 */ + { 0x000000007d657a10, 0x00000001af77fcd4 } +#else /* __LITTLE_ENDIAN__ */ + /* x^261120 mod p(x)` << 1, x^261184 mod p(x)` << 1 */ + { 0x00000001651797d2, 0x0000000099ea94a8 }, + /* x^260096 mod p(x)` << 1, x^260160 mod p(x)` << 1 */ + { 0x0000000021e0d56c, 0x00000000945a8420 }, + /* x^259072 mod p(x)` << 1, x^259136 mod p(x)` << 1 */ + { 0x000000000f95ecaa, 0x0000000030762706 }, + /* x^258048 mod p(x)` << 1, x^258112 mod p(x)` << 1 */ + { 0x00000001ebd224ac, 0x00000001a52fc582 }, + /* x^257024 mod p(x)` << 1, x^257088 mod p(x)` << 1 */ + { 0x000000000ccb97ca, 0x00000001a4a7167a }, + /* x^256000 mod p(x)` << 1, x^256064 mod p(x)` << 1 */ + { 0x00000001006ec8a8, 0x000000000c18249a }, + /* x^254976 mod p(x)` << 1, x^255040 mod p(x)` << 1 */ + { 0x000000014f58f196, 0x00000000a924ae7c }, + /* x^253952 mod p(x)` << 1, x^254016 mod p(x)` << 1 */ + { 0x00000001a7192ca6, 0x00000001e12ccc12 }, + /* x^252928 mod p(x)` << 1, x^252992 mod p(x)` << 1 */ + { 0x000000019a64bab2, 0x00000000a0b9d4ac }, + /* x^251904 mod p(x)` << 1, x^251968 mod p(x)` << 1 */ + { 0x0000000014f4ed2e, 0x0000000095e8ddfe }, + /* x^250880 mod p(x)` << 1, x^250944 mod p(x)` << 1 */ + { 0x000000011092b6a2, 0x00000000233fddc4 }, + /* x^249856 mod p(x)` << 1, x^249920 mod p(x)` << 1 */ + { 0x00000000c8a1629c, 0x00000001b4529b62 }, + /* x^248832 mod p(x)` << 1, x^248896 mod p(x)` << 1 */ + { 0x000000017bf32e8e, 0x00000001a7fa0e64 }, + /* x^247808 mod p(x)` << 1, x^247872 mod p(x)` << 1 */ + { 0x00000001f8cc6582, 0x00000001b5334592 }, + /* x^246784 mod p(x)` << 1, x^246848 mod p(x)` << 1 */ + { 0x000000008631ddf0, 0x000000011f8ee1b4 }, + /* x^245760 mod p(x)` << 1, x^245824 mod p(x)` << 1 */ + { 0x000000007e5a76d0, 0x000000006252e632 }, + /* x^244736 mod p(x)` << 1, x^244800 mod p(x)` << 1 */ + { 0x000000002b09b31c, 0x00000000ab973e84 }, + /* x^243712 mod p(x)` << 1, x^243776 mod p(x)` << 1 */ + { 0x00000001b2df1f84, 0x000000007734f5ec }, + /* x^242688 mod p(x)` << 1, x^242752 mod p(x)` << 1 */ + { 0x00000001d6f56afc, 0x000000007c547798 }, + /* x^241664 mod p(x)` << 1, x^241728 mod p(x)` << 1 */ + { 0x00000001b9b5e70c, 0x000000007ec40210 }, + /* x^240640 mod p(x)` << 1, x^240704 mod p(x)` << 1 */ + { 0x0000000034b626d2, 0x00000001ab1695a8 }, + /* x^239616 mod p(x)` << 1, x^239680 mod p(x)` << 1 */ + { 0x000000014c53479a, 0x0000000090494bba }, + /* x^238592 mod p(x)` << 1, x^238656 mod p(x)` << 1 */ + { 0x00000001a6d179a4, 0x00000001123fb816 }, + /* x^237568 mod p(x)` << 1, x^237632 mod p(x)` << 1 */ + { 0x000000015abd16b4, 0x00000001e188c74c }, + /* x^236544 mod p(x)` << 1, x^236608 mod p(x)` << 1 */ + { 0x00000000018f9852, 0x00000001c2d3451c }, + /* x^235520 mod p(x)` << 1, x^235584 mod p(x)` << 1 */ + { 0x000000001fb3084a, 0x00000000f55cf1ca }, + /* x^234496 mod p(x)` << 1, x^234560 mod p(x)` << 1 */ + { 0x00000000c53dfb04, 0x00000001a0531540 }, + /* x^233472 mod p(x)` << 1, x^233536 mod p(x)` << 1 */ + { 0x00000000e10c9ad6, 0x0000000132cd7ebc }, + /* x^232448 mod p(x)` << 1, x^232512 mod p(x)` << 1 */ + { 0x0000000025aa994a, 0x0000000073ab7f36 }, + /* x^231424 mod p(x)` << 1, x^231488 mod p(x)` << 1 */ + { 0x00000000fa3a74c4, 0x0000000041aed1c2 }, + /* x^230400 mod p(x)` << 1, x^230464 mod p(x)` << 1 */ + { 0x0000000033eb3f40, 0x0000000136c53800 }, + /* x^229376 mod p(x)` << 1, x^229440 mod p(x)` << 1 */ + { 0x000000017193f296, 0x0000000126835a30 }, + /* x^228352 mod p(x)` << 1, x^228416 mod p(x)` << 1 */ + { 0x0000000043f6c86a, 0x000000006241b502 }, + /* x^227328 mod p(x)` << 1, x^227392 mod p(x)` << 1 */ + { 0x000000016b513ec6, 0x00000000d5196ad4 }, + /* x^226304 mod p(x)` << 1, x^226368 mod p(x)` << 1 */ + { 0x00000000c8f25b4e, 0x000000009cfa769a }, + /* x^225280 mod p(x)` << 1, x^225344 mod p(x)` << 1 */ + { 0x00000001a45048ec, 0x00000000920e5df4 }, + /* x^224256 mod p(x)` << 1, x^224320 mod p(x)` << 1 */ + { 0x000000000c441004, 0x0000000169dc310e }, + /* x^223232 mod p(x)` << 1, x^223296 mod p(x)` << 1 */ + { 0x000000000e17cad6, 0x0000000009fc331c }, + /* x^222208 mod p(x)` << 1, x^222272 mod p(x)` << 1 */ + { 0x00000001253ae964, 0x000000010d94a81e }, + /* x^221184 mod p(x)` << 1, x^221248 mod p(x)` << 1 */ + { 0x00000001d7c88ebc, 0x0000000027a20ab2 }, + /* x^220160 mod p(x)` << 1, x^220224 mod p(x)` << 1 */ + { 0x00000001e7ca913a, 0x0000000114f87504 }, + /* x^219136 mod p(x)` << 1, x^219200 mod p(x)` << 1 */ + { 0x0000000033ed078a, 0x000000004b076d96 }, + /* x^218112 mod p(x)` << 1, x^218176 mod p(x)` << 1 */ + { 0x00000000e1839c78, 0x00000000da4d1e74 }, + /* x^217088 mod p(x)` << 1, x^217152 mod p(x)` << 1 */ + { 0x00000001322b267e, 0x000000001b81f672 }, + /* x^216064 mod p(x)` << 1, x^216128 mod p(x)` << 1 */ + { 0x00000000638231b6, 0x000000009367c988 }, + /* x^215040 mod p(x)` << 1, x^215104 mod p(x)` << 1 */ + { 0x00000001ee7f16f4, 0x00000001717214ca }, + /* x^214016 mod p(x)` << 1, x^214080 mod p(x)` << 1 */ + { 0x0000000117d9924a, 0x000000009f47d820 }, + /* x^212992 mod p(x)` << 1, x^213056 mod p(x)` << 1 */ + { 0x00000000e1a9e0c4, 0x000000010d9a47d2 }, + /* x^211968 mod p(x)` << 1, x^212032 mod p(x)` << 1 */ + { 0x00000001403731dc, 0x00000000a696c58c }, + /* x^210944 mod p(x)` << 1, x^211008 mod p(x)` << 1 */ + { 0x00000001a5ea9682, 0x000000002aa28ec6 }, + /* x^209920 mod p(x)` << 1, x^209984 mod p(x)` << 1 */ + { 0x0000000101c5c578, 0x00000001fe18fd9a }, + /* x^208896 mod p(x)` << 1, x^208960 mod p(x)` << 1 */ + { 0x00000000dddf6494, 0x000000019d4fc1ae }, + /* x^207872 mod p(x)` << 1, x^207936 mod p(x)` << 1 */ + { 0x00000000f1c3db28, 0x00000001ba0e3dea }, + /* x^206848 mod p(x)` << 1, x^206912 mod p(x)` << 1 */ + { 0x000000013112fb9c, 0x0000000074b59a5e }, + /* x^205824 mod p(x)` << 1, x^205888 mod p(x)` << 1 */ + { 0x00000000b680b906, 0x00000000f2b5ea98 }, + /* x^204800 mod p(x)` << 1, x^204864 mod p(x)` << 1 */ + { 0x000000001a282932, 0x0000000187132676 }, + /* x^203776 mod p(x)` << 1, x^203840 mod p(x)` << 1 */ + { 0x0000000089406e7e, 0x000000010a8c6ad4 }, + /* x^202752 mod p(x)` << 1, x^202816 mod p(x)` << 1 */ + { 0x00000001def6be8c, 0x00000001e21dfe70 }, + /* x^201728 mod p(x)` << 1, x^201792 mod p(x)` << 1 */ + { 0x0000000075258728, 0x00000001da0050e4 }, + /* x^200704 mod p(x)` << 1, x^200768 mod p(x)` << 1 */ + { 0x000000019536090a, 0x00000000772172ae }, + /* x^199680 mod p(x)` << 1, x^199744 mod p(x)` << 1 */ + { 0x00000000f2455bfc, 0x00000000e47724aa }, + /* x^198656 mod p(x)` << 1, x^198720 mod p(x)` << 1 */ + { 0x000000018c40baf4, 0x000000003cd63ac4 }, + /* x^197632 mod p(x)` << 1, x^197696 mod p(x)` << 1 */ + { 0x000000004cd390d4, 0x00000001bf47d352 }, + /* x^196608 mod p(x)` << 1, x^196672 mod p(x)` << 1 */ + { 0x00000001e4ece95a, 0x000000018dc1d708 }, + /* x^195584 mod p(x)` << 1, x^195648 mod p(x)` << 1 */ + { 0x000000001a3ee918, 0x000000002d4620a4 }, + /* x^194560 mod p(x)` << 1, x^194624 mod p(x)` << 1 */ + { 0x000000007c652fb8, 0x0000000058fd1740 }, + /* x^193536 mod p(x)` << 1, x^193600 mod p(x)` << 1 */ + { 0x000000011c67842c, 0x00000000dadd9bfc }, + /* x^192512 mod p(x)` << 1, x^192576 mod p(x)` << 1 */ + { 0x00000000254f759c, 0x00000001ea2140be }, + /* x^191488 mod p(x)` << 1, x^191552 mod p(x)` << 1 */ + { 0x000000007ece94ca, 0x000000009de128ba }, + /* x^190464 mod p(x)` << 1, x^190528 mod p(x)` << 1 */ + { 0x0000000038f258c2, 0x000000013ac3aa8e }, + /* x^189440 mod p(x)` << 1, x^189504 mod p(x)` << 1 */ + { 0x00000001cdf17b00, 0x0000000099980562 }, + /* x^188416 mod p(x)` << 1, x^188480 mod p(x)` << 1 */ + { 0x000000011f882c16, 0x00000001c1579c86 }, + /* x^187392 mod p(x)` << 1, x^187456 mod p(x)` << 1 */ + { 0x0000000100093fc8, 0x0000000068dbbf94 }, + /* x^186368 mod p(x)` << 1, x^186432 mod p(x)` << 1 */ + { 0x00000001cd684f16, 0x000000004509fb04 }, + /* x^185344 mod p(x)` << 1, x^185408 mod p(x)` << 1 */ + { 0x000000004bc6a70a, 0x00000001202f6398 }, + /* x^184320 mod p(x)` << 1, x^184384 mod p(x)` << 1 */ + { 0x000000004fc7e8e4, 0x000000013aea243e }, + /* x^183296 mod p(x)` << 1, x^183360 mod p(x)` << 1 */ + { 0x0000000130103f1c, 0x00000001b4052ae6 }, + /* x^182272 mod p(x)` << 1, x^182336 mod p(x)` << 1 */ + { 0x0000000111b0024c, 0x00000001cd2a0ae8 }, + /* x^181248 mod p(x)` << 1, x^181312 mod p(x)` << 1 */ + { 0x000000010b3079da, 0x00000001fe4aa8b4 }, + /* x^180224 mod p(x)` << 1, x^180288 mod p(x)` << 1 */ + { 0x000000010192bcc2, 0x00000001d1559a42 }, + /* x^179200 mod p(x)` << 1, x^179264 mod p(x)` << 1 */ + { 0x0000000074838d50, 0x00000001f3e05ecc }, + /* x^178176 mod p(x)` << 1, x^178240 mod p(x)` << 1 */ + { 0x000000001b20f520, 0x0000000104ddd2cc }, + /* x^177152 mod p(x)` << 1, x^177216 mod p(x)` << 1 */ + { 0x0000000050c3590a, 0x000000015393153c }, + /* x^176128 mod p(x)` << 1, x^176192 mod p(x)` << 1 */ + { 0x00000000b41cac8e, 0x0000000057e942c6 }, + /* x^175104 mod p(x)` << 1, x^175168 mod p(x)` << 1 */ + { 0x000000000c72cc78, 0x000000012c633850 }, + /* x^174080 mod p(x)` << 1, x^174144 mod p(x)` << 1 */ + { 0x0000000030cdb032, 0x00000000ebcaae4c }, + /* x^173056 mod p(x)` << 1, x^173120 mod p(x)` << 1 */ + { 0x000000013e09fc32, 0x000000013ee532a6 }, + /* x^172032 mod p(x)` << 1, x^172096 mod p(x)` << 1 */ + { 0x000000001ed624d2, 0x00000001bf0cbc7e }, + /* x^171008 mod p(x)` << 1, x^171072 mod p(x)` << 1 */ + { 0x00000000781aee1a, 0x00000000d50b7a5a }, + /* x^169984 mod p(x)` << 1, x^170048 mod p(x)` << 1 */ + { 0x00000001c4d8348c, 0x0000000002fca6e8 }, + /* x^168960 mod p(x)` << 1, x^169024 mod p(x)` << 1 */ + { 0x0000000057a40336, 0x000000007af40044 }, + /* x^167936 mod p(x)` << 1, x^168000 mod p(x)` << 1 */ + { 0x0000000085544940, 0x0000000016178744 }, + /* x^166912 mod p(x)` << 1, x^166976 mod p(x)` << 1 */ + { 0x000000019cd21e80, 0x000000014c177458 }, + /* x^165888 mod p(x)` << 1, x^165952 mod p(x)` << 1 */ + { 0x000000013eb95bc0, 0x000000011b6ddf04 }, + /* x^164864 mod p(x)` << 1, x^164928 mod p(x)` << 1 */ + { 0x00000001dfc9fdfc, 0x00000001f3e29ccc }, + /* x^163840 mod p(x)` << 1, x^163904 mod p(x)` << 1 */ + { 0x00000000cd028bc2, 0x0000000135ae7562 }, + /* x^162816 mod p(x)` << 1, x^162880 mod p(x)` << 1 */ + { 0x0000000090db8c44, 0x0000000190ef812c }, + /* x^161792 mod p(x)` << 1, x^161856 mod p(x)` << 1 */ + { 0x000000010010a4ce, 0x0000000067a2c786 }, + /* x^160768 mod p(x)` << 1, x^160832 mod p(x)` << 1 */ + { 0x00000001c8f4c72c, 0x0000000048b9496c }, + /* x^159744 mod p(x)` << 1, x^159808 mod p(x)` << 1 */ + { 0x000000001c26170c, 0x000000015a422de6 }, + /* x^158720 mod p(x)` << 1, x^158784 mod p(x)` << 1 */ + { 0x00000000e3fccf68, 0x00000001ef0e3640 }, + /* x^157696 mod p(x)` << 1, x^157760 mod p(x)` << 1 */ + { 0x00000000d513ed24, 0x00000001006d2d26 }, + /* x^156672 mod p(x)` << 1, x^156736 mod p(x)` << 1 */ + { 0x00000000141beada, 0x00000001170d56d6 }, + /* x^155648 mod p(x)` << 1, x^155712 mod p(x)` << 1 */ + { 0x000000011071aea0, 0x00000000a5fb613c }, + /* x^154624 mod p(x)` << 1, x^154688 mod p(x)` << 1 */ + { 0x000000012e19080a, 0x0000000040bbf7fc }, + /* x^153600 mod p(x)` << 1, x^153664 mod p(x)` << 1 */ + { 0x0000000100ecf826, 0x000000016ac3a5b2 }, + /* x^152576 mod p(x)` << 1, x^152640 mod p(x)` << 1 */ + { 0x0000000069b09412, 0x00000000abf16230 }, + /* x^151552 mod p(x)` << 1, x^151616 mod p(x)` << 1 */ + { 0x0000000122297bac, 0x00000001ebe23fac }, + /* x^150528 mod p(x)` << 1, x^150592 mod p(x)` << 1 */ + { 0x00000000e9e4b068, 0x000000008b6a0894 }, + /* x^149504 mod p(x)` << 1, x^149568 mod p(x)` << 1 */ + { 0x000000004b38651a, 0x00000001288ea478 }, + /* x^148480 mod p(x)` << 1, x^148544 mod p(x)` << 1 */ + { 0x00000001468360e2, 0x000000016619c442 }, + /* x^147456 mod p(x)` << 1, x^147520 mod p(x)` << 1 */ + { 0x00000000121c2408, 0x0000000086230038 }, + /* x^146432 mod p(x)` << 1, x^146496 mod p(x)` << 1 */ + { 0x00000000da7e7d08, 0x000000017746a756 }, + /* x^145408 mod p(x)` << 1, x^145472 mod p(x)` << 1 */ + { 0x00000001058d7652, 0x0000000191b8f8f8 }, + /* x^144384 mod p(x)` << 1, x^144448 mod p(x)` << 1 */ + { 0x000000014a098a90, 0x000000008e167708 }, + /* x^143360 mod p(x)` << 1, x^143424 mod p(x)` << 1 */ + { 0x0000000020dbe72e, 0x0000000148b22d54 }, + /* x^142336 mod p(x)` << 1, x^142400 mod p(x)` << 1 */ + { 0x000000011e7323e8, 0x0000000044ba2c3c }, + /* x^141312 mod p(x)` << 1, x^141376 mod p(x)` << 1 */ + { 0x00000000d5d4bf94, 0x00000000b54d2b52 }, + /* x^140288 mod p(x)` << 1, x^140352 mod p(x)` << 1 */ + { 0x0000000199d8746c, 0x0000000005a4fd8a }, + /* x^139264 mod p(x)` << 1, x^139328 mod p(x)` << 1 */ + { 0x00000000ce9ca8a0, 0x0000000139f9fc46 }, + /* x^138240 mod p(x)` << 1, x^138304 mod p(x)` << 1 */ + { 0x00000000136edece, 0x000000015a1fa824 }, + /* x^137216 mod p(x)` << 1, x^137280 mod p(x)` << 1 */ + { 0x000000019b92a068, 0x000000000a61ae4c }, + /* x^136192 mod p(x)` << 1, x^136256 mod p(x)` << 1 */ + { 0x0000000071d62206, 0x0000000145e9113e }, + /* x^135168 mod p(x)` << 1, x^135232 mod p(x)` << 1 */ + { 0x00000000dfc50158, 0x000000006a348448 }, + /* x^134144 mod p(x)` << 1, x^134208 mod p(x)` << 1 */ + { 0x00000001517626bc, 0x000000004d80a08c }, + /* x^133120 mod p(x)` << 1, x^133184 mod p(x)` << 1 */ + { 0x0000000148d1e4fa, 0x000000014b6837a0 }, + /* x^132096 mod p(x)` << 1, x^132160 mod p(x)` << 1 */ + { 0x0000000094d8266e, 0x000000016896a7fc }, + /* x^131072 mod p(x)` << 1, x^131136 mod p(x)` << 1 */ + { 0x00000000606c5e34, 0x000000014f187140 }, + /* x^130048 mod p(x)` << 1, x^130112 mod p(x)` << 1 */ + { 0x000000019766beaa, 0x000000019581b9da }, + /* x^129024 mod p(x)` << 1, x^129088 mod p(x)` << 1 */ + { 0x00000001d80c506c, 0x00000001091bc984 }, + /* x^128000 mod p(x)` << 1, x^128064 mod p(x)` << 1 */ + { 0x000000001e73837c, 0x000000001067223c }, + /* x^126976 mod p(x)` << 1, x^127040 mod p(x)` << 1 */ + { 0x0000000064d587de, 0x00000001ab16ea02 }, + /* x^125952 mod p(x)` << 1, x^126016 mod p(x)` << 1 */ + { 0x00000000f4a507b0, 0x000000013c4598a8 }, + /* x^124928 mod p(x)` << 1, x^124992 mod p(x)` << 1 */ + { 0x0000000040e342fc, 0x00000000b3735430 }, + /* x^123904 mod p(x)` << 1, x^123968 mod p(x)` << 1 */ + { 0x00000001d5ad9c3a, 0x00000001bb3fc0c0 }, + /* x^122880 mod p(x)` << 1, x^122944 mod p(x)` << 1 */ + { 0x0000000094a691a4, 0x00000001570ae19c }, + /* x^121856 mod p(x)` << 1, x^121920 mod p(x)` << 1 */ + { 0x00000001271ecdfa, 0x00000001ea910712 }, + /* x^120832 mod p(x)` << 1, x^120896 mod p(x)` << 1 */ + { 0x000000009e54475a, 0x0000000167127128 }, + /* x^119808 mod p(x)` << 1, x^119872 mod p(x)` << 1 */ + { 0x00000000c9c099ee, 0x0000000019e790a2 }, + /* x^118784 mod p(x)` << 1, x^118848 mod p(x)` << 1 */ + { 0x000000009a2f736c, 0x000000003788f710 }, + /* x^117760 mod p(x)` << 1, x^117824 mod p(x)` << 1 */ + { 0x00000000bb9f4996, 0x00000001682a160e }, + /* x^116736 mod p(x)` << 1, x^116800 mod p(x)` << 1 */ + { 0x00000001db688050, 0x000000007f0ebd2e }, + /* x^115712 mod p(x)` << 1, x^115776 mod p(x)` << 1 */ + { 0x00000000e9b10af4, 0x000000002b032080 }, + /* x^114688 mod p(x)` << 1, x^114752 mod p(x)` << 1 */ + { 0x000000012d4545e4, 0x00000000cfd1664a }, + /* x^113664 mod p(x)` << 1, x^113728 mod p(x)` << 1 */ + { 0x000000000361139c, 0x00000000aa1181c2 }, + /* x^112640 mod p(x)` << 1, x^112704 mod p(x)` << 1 */ + { 0x00000001a5a1a3a8, 0x00000000ddd08002 }, + /* x^111616 mod p(x)` << 1, x^111680 mod p(x)` << 1 */ + { 0x000000006844e0b0, 0x00000000e8dd0446 }, + /* x^110592 mod p(x)` << 1, x^110656 mod p(x)` << 1 */ + { 0x00000000c3762f28, 0x00000001bbd94a00 }, + /* x^109568 mod p(x)` << 1, x^109632 mod p(x)` << 1 */ + { 0x00000001d26287a2, 0x00000000ab6cd180 }, + /* x^108544 mod p(x)` << 1, x^108608 mod p(x)` << 1 */ + { 0x00000001f6f0bba8, 0x0000000031803ce2 }, + /* x^107520 mod p(x)` << 1, x^107584 mod p(x)` << 1 */ + { 0x000000002ffabd62, 0x0000000024f40b0c }, + /* x^106496 mod p(x)` << 1, x^106560 mod p(x)` << 1 */ + { 0x00000000fb4516b8, 0x00000001ba1d9834 }, + /* x^105472 mod p(x)` << 1, x^105536 mod p(x)` << 1 */ + { 0x000000018cfa961c, 0x0000000104de61aa }, + /* x^104448 mod p(x)` << 1, x^104512 mod p(x)` << 1 */ + { 0x000000019e588d52, 0x0000000113e40d46 }, + /* x^103424 mod p(x)` << 1, x^103488 mod p(x)` << 1 */ + { 0x00000001180f0bbc, 0x00000001415598a0 }, + /* x^102400 mod p(x)` << 1, x^102464 mod p(x)` << 1 */ + { 0x00000000e1d9177a, 0x00000000bf6c8c90 }, + /* x^101376 mod p(x)` << 1, x^101440 mod p(x)` << 1 */ + { 0x0000000105abc27c, 0x00000001788b0504 }, + /* x^100352 mod p(x)` << 1, x^100416 mod p(x)` << 1 */ + { 0x00000000972e4a58, 0x0000000038385d02 }, + /* x^99328 mod p(x)` << 1, x^99392 mod p(x)` << 1 */ + { 0x0000000183499a5e, 0x00000001b6c83844 }, + /* x^98304 mod p(x)` << 1, x^98368 mod p(x)` << 1 */ + { 0x00000001c96a8cca, 0x0000000051061a8a }, + /* x^97280 mod p(x)` << 1, x^97344 mod p(x)` << 1 */ + { 0x00000001a1a5b60c, 0x000000017351388a }, + /* x^96256 mod p(x)` << 1, x^96320 mod p(x)` << 1 */ + { 0x00000000e4b6ac9c, 0x0000000132928f92 }, + /* x^95232 mod p(x)` << 1, x^95296 mod p(x)` << 1 */ + { 0x00000001807e7f5a, 0x00000000e6b4f48a }, + /* x^94208 mod p(x)` << 1, x^94272 mod p(x)` << 1 */ + { 0x000000017a7e3bc8, 0x0000000039d15e90 }, + /* x^93184 mod p(x)` << 1, x^93248 mod p(x)` << 1 */ + { 0x00000000d73975da, 0x00000000312d6074 }, + /* x^92160 mod p(x)` << 1, x^92224 mod p(x)` << 1 */ + { 0x000000017375d038, 0x000000017bbb2cc4 }, + /* x^91136 mod p(x)` << 1, x^91200 mod p(x)` << 1 */ + { 0x00000000193680bc, 0x000000016ded3e18 }, + /* x^90112 mod p(x)` << 1, x^90176 mod p(x)` << 1 */ + { 0x00000000999b06f6, 0x00000000f1638b16 }, + /* x^89088 mod p(x)` << 1, x^89152 mod p(x)` << 1 */ + { 0x00000001f685d2b8, 0x00000001d38b9ecc }, + /* x^88064 mod p(x)` << 1, x^88128 mod p(x)` << 1 */ + { 0x00000001f4ecbed2, 0x000000018b8d09dc }, + /* x^87040 mod p(x)` << 1, x^87104 mod p(x)` << 1 */ + { 0x00000000ba16f1a0, 0x00000000e7bc27d2 }, + /* x^86016 mod p(x)` << 1, x^86080 mod p(x)` << 1 */ + { 0x0000000115aceac4, 0x00000000275e1e96 }, + /* x^84992 mod p(x)` << 1, x^85056 mod p(x)` << 1 */ + { 0x00000001aeff6292, 0x00000000e2e3031e }, + /* x^83968 mod p(x)` << 1, x^84032 mod p(x)` << 1 */ + { 0x000000009640124c, 0x00000001041c84d8 }, + /* x^82944 mod p(x)` << 1, x^83008 mod p(x)` << 1 */ + { 0x0000000114f41f02, 0x00000000706ce672 }, + /* x^81920 mod p(x)` << 1, x^81984 mod p(x)` << 1 */ + { 0x000000009c5f3586, 0x000000015d5070da }, + /* x^80896 mod p(x)` << 1, x^80960 mod p(x)` << 1 */ + { 0x00000001878275fa, 0x0000000038f9493a }, + /* x^79872 mod p(x)` << 1, x^79936 mod p(x)` << 1 */ + { 0x00000000ddc42ce8, 0x00000000a3348a76 }, + /* x^78848 mod p(x)` << 1, x^78912 mod p(x)` << 1 */ + { 0x0000000181d2c73a, 0x00000001ad0aab92 }, + /* x^77824 mod p(x)` << 1, x^77888 mod p(x)` << 1 */ + { 0x0000000141c9320a, 0x000000019e85f712 }, + /* x^76800 mod p(x)` << 1, x^76864 mod p(x)` << 1 */ + { 0x000000015235719a, 0x000000005a871e76 }, + /* x^75776 mod p(x)` << 1, x^75840 mod p(x)` << 1 */ + { 0x00000000be27d804, 0x000000017249c662 }, + /* x^74752 mod p(x)` << 1, x^74816 mod p(x)` << 1 */ + { 0x000000006242d45a, 0x000000003a084712 }, + /* x^73728 mod p(x)` << 1, x^73792 mod p(x)` << 1 */ + { 0x000000009a53638e, 0x00000000ed438478 }, + /* x^72704 mod p(x)` << 1, x^72768 mod p(x)` << 1 */ + { 0x00000001001ecfb6, 0x00000000abac34cc }, + /* x^71680 mod p(x)` << 1, x^71744 mod p(x)` << 1 */ + { 0x000000016d7c2d64, 0x000000005f35ef3e }, + /* x^70656 mod p(x)` << 1, x^70720 mod p(x)` << 1 */ + { 0x00000001d0ce46c0, 0x0000000047d6608c }, + /* x^69632 mod p(x)` << 1, x^69696 mod p(x)` << 1 */ + { 0x0000000124c907b4, 0x000000002d01470e }, + /* x^68608 mod p(x)` << 1, x^68672 mod p(x)` << 1 */ + { 0x0000000018a555ca, 0x0000000158bbc7b0 }, + /* x^67584 mod p(x)` << 1, x^67648 mod p(x)` << 1 */ + { 0x000000006b0980bc, 0x00000000c0a23e8e }, + /* x^66560 mod p(x)` << 1, x^66624 mod p(x)` << 1 */ + { 0x000000008bbba964, 0x00000001ebd85c88 }, + /* x^65536 mod p(x)` << 1, x^65600 mod p(x)` << 1 */ + { 0x00000001070a5a1e, 0x000000019ee20bb2 }, + /* x^64512 mod p(x)` << 1, x^64576 mod p(x)` << 1 */ + { 0x000000002204322a, 0x00000001acabf2d6 }, + /* x^63488 mod p(x)` << 1, x^63552 mod p(x)` << 1 */ + { 0x00000000a27524d0, 0x00000001b7963d56 }, + /* x^62464 mod p(x)` << 1, x^62528 mod p(x)` << 1 */ + { 0x0000000020b1e4ba, 0x000000017bffa1fe }, + /* x^61440 mod p(x)` << 1, x^61504 mod p(x)` << 1 */ + { 0x0000000032cc27fc, 0x000000001f15333e }, + /* x^60416 mod p(x)` << 1, x^60480 mod p(x)` << 1 */ + { 0x0000000044dd22b8, 0x000000018593129e }, + /* x^59392 mod p(x)` << 1, x^59456 mod p(x)` << 1 */ + { 0x00000000dffc9e0a, 0x000000019cb32602 }, + /* x^58368 mod p(x)` << 1, x^58432 mod p(x)` << 1 */ + { 0x00000001b7a0ed14, 0x0000000142b05cc8 }, + /* x^57344 mod p(x)` << 1, x^57408 mod p(x)` << 1 */ + { 0x00000000c7842488, 0x00000001be49e7a4 }, + /* x^56320 mod p(x)` << 1, x^56384 mod p(x)` << 1 */ + { 0x00000001c02a4fee, 0x0000000108f69d6c }, + /* x^55296 mod p(x)` << 1, x^55360 mod p(x)` << 1 */ + { 0x000000003c273778, 0x000000006c0971f0 }, + /* x^54272 mod p(x)` << 1, x^54336 mod p(x)` << 1 */ + { 0x00000001d63f8894, 0x000000005b16467a }, + /* x^53248 mod p(x)` << 1, x^53312 mod p(x)` << 1 */ + { 0x000000006be557d6, 0x00000001551a628e }, + /* x^52224 mod p(x)` << 1, x^52288 mod p(x)` << 1 */ + { 0x000000006a7806ea, 0x000000019e42ea92 }, + /* x^51200 mod p(x)` << 1, x^51264 mod p(x)` << 1 */ + { 0x000000016155aa0c, 0x000000012fa83ff2 }, + /* x^50176 mod p(x)` << 1, x^50240 mod p(x)` << 1 */ + { 0x00000000908650ac, 0x000000011ca9cde0 }, + /* x^49152 mod p(x)` << 1, x^49216 mod p(x)` << 1 */ + { 0x00000000aa5a8084, 0x00000000c8e5cd74 }, + /* x^48128 mod p(x)` << 1, x^48192 mod p(x)` << 1 */ + { 0x0000000191bb500a, 0x0000000096c27f0c }, + /* x^47104 mod p(x)` << 1, x^47168 mod p(x)` << 1 */ + { 0x0000000064e9bed0, 0x000000002baed926 }, + /* x^46080 mod p(x)` << 1, x^46144 mod p(x)` << 1 */ + { 0x000000009444f302, 0x000000017c8de8d2 }, + /* x^45056 mod p(x)` << 1, x^45120 mod p(x)` << 1 */ + { 0x000000019db07d3c, 0x00000000d43d6068 }, + /* x^44032 mod p(x)` << 1, x^44096 mod p(x)` << 1 */ + { 0x00000001359e3e6e, 0x00000000cb2c4b26 }, + /* x^43008 mod p(x)` << 1, x^43072 mod p(x)` << 1 */ + { 0x00000001e4f10dd2, 0x0000000145b8da26 }, + /* x^41984 mod p(x)` << 1, x^42048 mod p(x)` << 1 */ + { 0x0000000124f5735e, 0x000000018fff4b08 }, + /* x^40960 mod p(x)` << 1, x^41024 mod p(x)` << 1 */ + { 0x0000000124760a4c, 0x0000000150b58ed0 }, + /* x^39936 mod p(x)` << 1, x^40000 mod p(x)` << 1 */ + { 0x000000000f1fc186, 0x00000001549f39bc }, + /* x^38912 mod p(x)` << 1, x^38976 mod p(x)` << 1 */ + { 0x00000000150e4cc4, 0x00000000ef4d2f42 }, + /* x^37888 mod p(x)` << 1, x^37952 mod p(x)` << 1 */ + { 0x000000002a6204e8, 0x00000001b1468572 }, + /* x^36864 mod p(x)` << 1, x^36928 mod p(x)` << 1 */ + { 0x00000000beb1d432, 0x000000013d7403b2 }, + /* x^35840 mod p(x)` << 1, x^35904 mod p(x)` << 1 */ + { 0x0000000135f3f1f0, 0x00000001a4681842 }, + /* x^34816 mod p(x)` << 1, x^34880 mod p(x)` << 1 */ + { 0x0000000074fe2232, 0x0000000167714492 }, + /* x^33792 mod p(x)` << 1, x^33856 mod p(x)` << 1 */ + { 0x000000001ac6e2ba, 0x00000001e599099a }, + /* x^32768 mod p(x)` << 1, x^32832 mod p(x)` << 1 */ + { 0x0000000013fca91e, 0x00000000fe128194 }, + /* x^31744 mod p(x)` << 1, x^31808 mod p(x)` << 1 */ + { 0x0000000183f4931e, 0x0000000077e8b990 }, + /* x^30720 mod p(x)` << 1, x^30784 mod p(x)` << 1 */ + { 0x00000000b6d9b4e4, 0x00000001a267f63a }, + /* x^29696 mod p(x)` << 1, x^29760 mod p(x)` << 1 */ + { 0x00000000b5188656, 0x00000001945c245a }, + /* x^28672 mod p(x)` << 1, x^28736 mod p(x)` << 1 */ + { 0x0000000027a81a84, 0x0000000149002e76 }, + /* x^27648 mod p(x)` << 1, x^27712 mod p(x)` << 1 */ + { 0x0000000125699258, 0x00000001bb8310a4 }, + /* x^26624 mod p(x)` << 1, x^26688 mod p(x)` << 1 */ + { 0x00000001b23de796, 0x000000019ec60bcc }, + /* x^25600 mod p(x)` << 1, x^25664 mod p(x)` << 1 */ + { 0x00000000fe4365dc, 0x000000012d8590ae }, + /* x^24576 mod p(x)` << 1, x^24640 mod p(x)` << 1 */ + { 0x00000000c68f497a, 0x0000000065b00684 }, + /* x^23552 mod p(x)` << 1, x^23616 mod p(x)` << 1 */ + { 0x00000000fbf521ee, 0x000000015e5aeadc }, + /* x^22528 mod p(x)` << 1, x^22592 mod p(x)` << 1 */ + { 0x000000015eac3378, 0x00000000b77ff2b0 }, + /* x^21504 mod p(x)` << 1, x^21568 mod p(x)` << 1 */ + { 0x0000000134914b90, 0x0000000188da2ff6 }, + /* x^20480 mod p(x)` << 1, x^20544 mod p(x)` << 1 */ + { 0x0000000016335cfe, 0x0000000063da929a }, + /* x^19456 mod p(x)` << 1, x^19520 mod p(x)` << 1 */ + { 0x000000010372d10c, 0x00000001389caa80 }, + /* x^18432 mod p(x)` << 1, x^18496 mod p(x)` << 1 */ + { 0x000000015097b908, 0x000000013db599d2 }, + /* x^17408 mod p(x)` << 1, x^17472 mod p(x)` << 1 */ + { 0x00000001227a7572, 0x0000000122505a86 }, + /* x^16384 mod p(x)` << 1, x^16448 mod p(x)` << 1 */ + { 0x000000009a8f75c0, 0x000000016bd72746 }, + /* x^15360 mod p(x)` << 1, x^15424 mod p(x)` << 1 */ + { 0x00000000682c77a2, 0x00000001c3faf1d4 }, + /* x^14336 mod p(x)` << 1, x^14400 mod p(x)` << 1 */ + { 0x00000000231f091c, 0x00000001111c826c }, + /* x^13312 mod p(x)` << 1, x^13376 mod p(x)` << 1 */ + { 0x000000007d4439f2, 0x00000000153e9fb2 }, + /* x^12288 mod p(x)` << 1, x^12352 mod p(x)` << 1 */ + { 0x000000017e221efc, 0x000000002b1f7b60 }, + /* x^11264 mod p(x)` << 1, x^11328 mod p(x)` << 1 */ + { 0x0000000167457c38, 0x00000000b1dba570 }, + /* x^10240 mod p(x)` << 1, x^10304 mod p(x)` << 1 */ + { 0x00000000bdf081c4, 0x00000001f6397b76 }, + /* x^9216 mod p(x)` << 1, x^9280 mod p(x)` << 1 */ + { 0x000000016286d6b0, 0x0000000156335214 }, + /* x^8192 mod p(x)` << 1, x^8256 mod p(x)` << 1 */ + { 0x00000000c84f001c, 0x00000001d70e3986 }, + /* x^7168 mod p(x)` << 1, x^7232 mod p(x)` << 1 */ + { 0x0000000064efe7c0, 0x000000003701a774 }, + /* x^6144 mod p(x)` << 1, x^6208 mod p(x)` << 1 */ + { 0x000000000ac2d904, 0x00000000ac81ef72 }, + /* x^5120 mod p(x)` << 1, x^5184 mod p(x)` << 1 */ + { 0x00000000fd226d14, 0x0000000133212464 }, + /* x^4096 mod p(x)` << 1, x^4160 mod p(x)` << 1 */ + { 0x000000011cfd42e0, 0x00000000e4e45610 }, + /* x^3072 mod p(x)` << 1, x^3136 mod p(x)` << 1 */ + { 0x000000016e5a5678, 0x000000000c1bd370 }, + /* x^2048 mod p(x)` << 1, x^2112 mod p(x)` << 1 */ + { 0x00000001d888fe22, 0x00000001a7b9e7a6 }, + /* x^1024 mod p(x)` << 1, x^1088 mod p(x)` << 1 */ + { 0x00000001af77fcd4, 0x000000007d657a10 } +#endif /* __LITTLE_ENDIAN__ */ + }; + +/* Reduce final 1024-2048 bits to 64 bits, shifting 32 bits to include the trailing 32 bits of zeros */ + +static const __vector unsigned long long vcrc_short_const[16] + __attribute__((aligned (16))) = { +#ifdef __LITTLE_ENDIAN__ + /* x^1952 mod p(x) , x^1984 mod p(x) , x^2016 mod p(x) , x^2048 mod p(x) */ + { 0x99168a18ec447f11, 0xed837b2613e8221e }, + /* x^1824 mod p(x) , x^1856 mod p(x) , x^1888 mod p(x) , x^1920 mod p(x) */ + { 0xe23e954e8fd2cd3c, 0xc8acdd8147b9ce5a }, + /* x^1696 mod p(x) , x^1728 mod p(x) , x^1760 mod p(x) , x^1792 mod p(x) */ + { 0x92f8befe6b1d2b53, 0xd9ad6d87d4277e25 }, + /* x^1568 mod p(x) , x^1600 mod p(x) , x^1632 mod p(x) , x^1664 mod p(x) */ + { 0xf38a3556291ea462, 0xc10ec5e033fbca3b }, + /* x^1440 mod p(x) , x^1472 mod p(x) , x^1504 mod p(x) , x^1536 mod p(x) */ + { 0x974ac56262b6ca4b, 0xc0b55b0e82e02e2f }, + /* x^1312 mod p(x) , x^1344 mod p(x) , x^1376 mod p(x) , x^1408 mod p(x) */ + { 0x855712b3784d2a56, 0x71aa1df0e172334d }, + /* x^1184 mod p(x) , x^1216 mod p(x) , x^1248 mod p(x) , x^1280 mod p(x) */ + { 0xa5abe9f80eaee722, 0xfee3053e3969324d }, + /* x^1056 mod p(x) , x^1088 mod p(x) , x^1120 mod p(x) , x^1152 mod p(x) */ + { 0x1fa0943ddb54814c, 0xf44779b93eb2bd08 }, + /* x^928 mod p(x) , x^960 mod p(x) , x^992 mod p(x) , x^1024 mod p(x) */ + { 0xa53ff440d7bbfe6a, 0xf5449b3f00cc3374 }, + /* x^800 mod p(x) , x^832 mod p(x) , x^864 mod p(x) , x^896 mod p(x) */ + { 0xebe7e3566325605c, 0x6f8346e1d777606e }, + /* x^672 mod p(x) , x^704 mod p(x) , x^736 mod p(x) , x^768 mod p(x) */ + { 0xc65a272ce5b592b8, 0xe3ab4f2ac0b95347 }, + /* x^544 mod p(x) , x^576 mod p(x) , x^608 mod p(x) , x^640 mod p(x) */ + { 0x5705a9ca4721589f, 0xaa2215ea329ecc11 }, + /* x^416 mod p(x) , x^448 mod p(x) , x^480 mod p(x) , x^512 mod p(x) */ + { 0xe3720acb88d14467, 0x1ed8f66ed95efd26 }, + /* x^288 mod p(x) , x^320 mod p(x) , x^352 mod p(x) , x^384 mod p(x) */ + { 0xba1aca0315141c31, 0x78ed02d5a700e96a }, + /* x^160 mod p(x) , x^192 mod p(x) , x^224 mod p(x) , x^256 mod p(x) */ + { 0xad2a31b3ed627dae, 0xba8ccbe832b39da3 }, + /* x^32 mod p(x) , x^64 mod p(x) , x^96 mod p(x) , x^128 mod p(x) */ + { 0x6655004fa06a2517, 0xedb88320b1e6b092 } +#else /* __LITTLE_ENDIAN__ */ + /* x^1952 mod p(x) , x^1984 mod p(x) , x^2016 mod p(x) , x^2048 mod p(x) */ + { 0xed837b2613e8221e, 0x99168a18ec447f11 }, + /* x^1824 mod p(x) , x^1856 mod p(x) , x^1888 mod p(x) , x^1920 mod p(x) */ + { 0xc8acdd8147b9ce5a, 0xe23e954e8fd2cd3c }, + /* x^1696 mod p(x) , x^1728 mod p(x) , x^1760 mod p(x) , x^1792 mod p(x) */ + { 0xd9ad6d87d4277e25, 0x92f8befe6b1d2b53 }, + /* x^1568 mod p(x) , x^1600 mod p(x) , x^1632 mod p(x) , x^1664 mod p(x) */ + { 0xc10ec5e033fbca3b, 0xf38a3556291ea462 }, + /* x^1440 mod p(x) , x^1472 mod p(x) , x^1504 mod p(x) , x^1536 mod p(x) */ + { 0xc0b55b0e82e02e2f, 0x974ac56262b6ca4b }, + /* x^1312 mod p(x) , x^1344 mod p(x) , x^1376 mod p(x) , x^1408 mod p(x) */ + { 0x71aa1df0e172334d, 0x855712b3784d2a56 }, + /* x^1184 mod p(x) , x^1216 mod p(x) , x^1248 mod p(x) , x^1280 mod p(x) */ + { 0xfee3053e3969324d, 0xa5abe9f80eaee722 }, + /* x^1056 mod p(x) , x^1088 mod p(x) , x^1120 mod p(x) , x^1152 mod p(x) */ + { 0xf44779b93eb2bd08, 0x1fa0943ddb54814c }, + /* x^928 mod p(x) , x^960 mod p(x) , x^992 mod p(x) , x^1024 mod p(x) */ + { 0xf5449b3f00cc3374, 0xa53ff440d7bbfe6a }, + /* x^800 mod p(x) , x^832 mod p(x) , x^864 mod p(x) , x^896 mod p(x) */ + { 0x6f8346e1d777606e, 0xebe7e3566325605c }, + /* x^672 mod p(x) , x^704 mod p(x) , x^736 mod p(x) , x^768 mod p(x) */ + { 0xe3ab4f2ac0b95347, 0xc65a272ce5b592b8 }, + /* x^544 mod p(x) , x^576 mod p(x) , x^608 mod p(x) , x^640 mod p(x) */ + { 0xaa2215ea329ecc11, 0x5705a9ca4721589f }, + /* x^416 mod p(x) , x^448 mod p(x) , x^480 mod p(x) , x^512 mod p(x) */ + { 0x1ed8f66ed95efd26, 0xe3720acb88d14467 }, + /* x^288 mod p(x) , x^320 mod p(x) , x^352 mod p(x) , x^384 mod p(x) */ + { 0x78ed02d5a700e96a, 0xba1aca0315141c31 }, + /* x^160 mod p(x) , x^192 mod p(x) , x^224 mod p(x) , x^256 mod p(x) */ + { 0xba8ccbe832b39da3, 0xad2a31b3ed627dae }, + /* x^32 mod p(x) , x^64 mod p(x) , x^96 mod p(x) , x^128 mod p(x) */ + { 0xedb88320b1e6b092, 0x6655004fa06a2517 } +#endif /* __LITTLE_ENDIAN__ */ + }; + +/* Barrett constants */ +/* 33 bit reflected Barrett constant m - (4^32)/n */ + +static const __vector unsigned long long v_Barrett_const[2] + __attribute__((aligned (16))) = { + /* x^64 div p(x) */ +#ifdef __LITTLE_ENDIAN__ + { 0x00000001f7011641, 0x0000000000000000 }, + { 0x00000001db710641, 0x0000000000000000 } +#else /* __LITTLE_ENDIAN__ */ + { 0x0000000000000000, 0x00000001f7011641 }, + { 0x0000000000000000, 0x00000001db710641 } +#endif /* __LITTLE_ENDIAN__ */ + }; +#endif /* POWER8_INTRINSICS */ + +#endif /* __ASSEMBLER__ */ diff --git a/mysys/crc32/pcc_crc32c_constants.h b/mysys/crc32/pcc_crc32c_constants.h new file mode 100644 index 00000000000..40b216b6057 --- /dev/null +++ b/mysys/crc32/pcc_crc32c_constants.h @@ -0,0 +1,1206 @@ +/* +* +* THIS FILE IS GENERATED WITH +./crc32_constants -c -x -r 0x1edc6f41 + +* This is from https://github.com/antonblanchard/crc32-vpmsum/ +* DO NOT MODIFY IT MANUALLY! +* +*/ + +#define CRC 0x1edc6f41 +#define CRC_XOR +#define REFLECT +#define MAX_SIZE 32768 + +#ifndef __ASSEMBLER__ +#ifdef CRC_TABLE +static const unsigned int crc_table[] = { + 0x00000000, 0xf26b8303, 0xe13b70f7, 0x1350f3f4, + 0xc79a971f, 0x35f1141c, 0x26a1e7e8, 0xd4ca64eb, + 0x8ad958cf, 0x78b2dbcc, 0x6be22838, 0x9989ab3b, + 0x4d43cfd0, 0xbf284cd3, 0xac78bf27, 0x5e133c24, + 0x105ec76f, 0xe235446c, 0xf165b798, 0x030e349b, + 0xd7c45070, 0x25afd373, 0x36ff2087, 0xc494a384, + 0x9a879fa0, 0x68ec1ca3, 0x7bbcef57, 0x89d76c54, + 0x5d1d08bf, 0xaf768bbc, 0xbc267848, 0x4e4dfb4b, + 0x20bd8ede, 0xd2d60ddd, 0xc186fe29, 0x33ed7d2a, + 0xe72719c1, 0x154c9ac2, 0x061c6936, 0xf477ea35, + 0xaa64d611, 0x580f5512, 0x4b5fa6e6, 0xb93425e5, + 0x6dfe410e, 0x9f95c20d, 0x8cc531f9, 0x7eaeb2fa, + 0x30e349b1, 0xc288cab2, 0xd1d83946, 0x23b3ba45, + 0xf779deae, 0x05125dad, 0x1642ae59, 0xe4292d5a, + 0xba3a117e, 0x4851927d, 0x5b016189, 0xa96ae28a, + 0x7da08661, 0x8fcb0562, 0x9c9bf696, 0x6ef07595, + 0x417b1dbc, 0xb3109ebf, 0xa0406d4b, 0x522bee48, + 0x86e18aa3, 0x748a09a0, 0x67dafa54, 0x95b17957, + 0xcba24573, 0x39c9c670, 0x2a993584, 0xd8f2b687, + 0x0c38d26c, 0xfe53516f, 0xed03a29b, 0x1f682198, + 0x5125dad3, 0xa34e59d0, 0xb01eaa24, 0x42752927, + 0x96bf4dcc, 0x64d4cecf, 0x77843d3b, 0x85efbe38, + 0xdbfc821c, 0x2997011f, 0x3ac7f2eb, 0xc8ac71e8, + 0x1c661503, 0xee0d9600, 0xfd5d65f4, 0x0f36e6f7, + 0x61c69362, 0x93ad1061, 0x80fde395, 0x72966096, + 0xa65c047d, 0x5437877e, 0x4767748a, 0xb50cf789, + 0xeb1fcbad, 0x197448ae, 0x0a24bb5a, 0xf84f3859, + 0x2c855cb2, 0xdeeedfb1, 0xcdbe2c45, 0x3fd5af46, + 0x7198540d, 0x83f3d70e, 0x90a324fa, 0x62c8a7f9, + 0xb602c312, 0x44694011, 0x5739b3e5, 0xa55230e6, + 0xfb410cc2, 0x092a8fc1, 0x1a7a7c35, 0xe811ff36, + 0x3cdb9bdd, 0xceb018de, 0xdde0eb2a, 0x2f8b6829, + 0x82f63b78, 0x709db87b, 0x63cd4b8f, 0x91a6c88c, + 0x456cac67, 0xb7072f64, 0xa457dc90, 0x563c5f93, + 0x082f63b7, 0xfa44e0b4, 0xe9141340, 0x1b7f9043, + 0xcfb5f4a8, 0x3dde77ab, 0x2e8e845f, 0xdce5075c, + 0x92a8fc17, 0x60c37f14, 0x73938ce0, 0x81f80fe3, + 0x55326b08, 0xa759e80b, 0xb4091bff, 0x466298fc, + 0x1871a4d8, 0xea1a27db, 0xf94ad42f, 0x0b21572c, + 0xdfeb33c7, 0x2d80b0c4, 0x3ed04330, 0xccbbc033, + 0xa24bb5a6, 0x502036a5, 0x4370c551, 0xb11b4652, + 0x65d122b9, 0x97baa1ba, 0x84ea524e, 0x7681d14d, + 0x2892ed69, 0xdaf96e6a, 0xc9a99d9e, 0x3bc21e9d, + 0xef087a76, 0x1d63f975, 0x0e330a81, 0xfc588982, + 0xb21572c9, 0x407ef1ca, 0x532e023e, 0xa145813d, + 0x758fe5d6, 0x87e466d5, 0x94b49521, 0x66df1622, + 0x38cc2a06, 0xcaa7a905, 0xd9f75af1, 0x2b9cd9f2, + 0xff56bd19, 0x0d3d3e1a, 0x1e6dcdee, 0xec064eed, + 0xc38d26c4, 0x31e6a5c7, 0x22b65633, 0xd0ddd530, + 0x0417b1db, 0xf67c32d8, 0xe52cc12c, 0x1747422f, + 0x49547e0b, 0xbb3ffd08, 0xa86f0efc, 0x5a048dff, + 0x8ecee914, 0x7ca56a17, 0x6ff599e3, 0x9d9e1ae0, + 0xd3d3e1ab, 0x21b862a8, 0x32e8915c, 0xc083125f, + 0x144976b4, 0xe622f5b7, 0xf5720643, 0x07198540, + 0x590ab964, 0xab613a67, 0xb831c993, 0x4a5a4a90, + 0x9e902e7b, 0x6cfbad78, 0x7fab5e8c, 0x8dc0dd8f, + 0xe330a81a, 0x115b2b19, 0x020bd8ed, 0xf0605bee, + 0x24aa3f05, 0xd6c1bc06, 0xc5914ff2, 0x37faccf1, + 0x69e9f0d5, 0x9b8273d6, 0x88d28022, 0x7ab90321, + 0xae7367ca, 0x5c18e4c9, 0x4f48173d, 0xbd23943e, + 0xf36e6f75, 0x0105ec76, 0x12551f82, 0xe03e9c81, + 0x34f4f86a, 0xc69f7b69, 0xd5cf889d, 0x27a40b9e, + 0x79b737ba, 0x8bdcb4b9, 0x988c474d, 0x6ae7c44e, + 0xbe2da0a5, 0x4c4623a6, 0x5f16d052, 0xad7d5351,}; + +#endif /* CRC_TABLE */ +#ifdef POWER8_INTRINSICS + +/* Constants */ + +/* Reduce 262144 kbits to 1024 bits */ +static const __vector unsigned long long vcrc_const[255] + __attribute__((aligned (16))) = { +#ifdef __LITTLE_ENDIAN__ + /* x^261120 mod p(x)` << 1, x^261184 mod p(x)` << 1 */ + { 0x000000009c37c408, 0x00000000b6ca9e20 }, + /* x^260096 mod p(x)` << 1, x^260160 mod p(x)` << 1 */ + { 0x00000001b51df26c, 0x00000000350249a8 }, + /* x^259072 mod p(x)` << 1, x^259136 mod p(x)` << 1 */ + { 0x000000000724b9d0, 0x00000001862dac54 }, + /* x^258048 mod p(x)` << 1, x^258112 mod p(x)` << 1 */ + { 0x00000001c00532fe, 0x00000001d87fb48c }, + /* x^257024 mod p(x)` << 1, x^257088 mod p(x)` << 1 */ + { 0x00000000f05a9362, 0x00000001f39b699e }, + /* x^256000 mod p(x)` << 1, x^256064 mod p(x)` << 1 */ + { 0x00000001e1007970, 0x0000000101da11b4 }, + /* x^254976 mod p(x)` << 1, x^255040 mod p(x)` << 1 */ + { 0x00000000a57366ee, 0x00000001cab571e0 }, + /* x^253952 mod p(x)` << 1, x^254016 mod p(x)` << 1 */ + { 0x0000000192011284, 0x00000000c7020cfe }, + /* x^252928 mod p(x)` << 1, x^252992 mod p(x)` << 1 */ + { 0x0000000162716d9a, 0x00000000cdaed1ae }, + /* x^251904 mod p(x)` << 1, x^251968 mod p(x)` << 1 */ + { 0x00000000cd97ecde, 0x00000001e804effc }, + /* x^250880 mod p(x)` << 1, x^250944 mod p(x)` << 1 */ + { 0x0000000058812bc0, 0x0000000077c3ea3a }, + /* x^249856 mod p(x)` << 1, x^249920 mod p(x)` << 1 */ + { 0x0000000088b8c12e, 0x0000000068df31b4 }, + /* x^248832 mod p(x)` << 1, x^248896 mod p(x)` << 1 */ + { 0x00000001230b234c, 0x00000000b059b6c2 }, + /* x^247808 mod p(x)` << 1, x^247872 mod p(x)` << 1 */ + { 0x00000001120b416e, 0x0000000145fb8ed8 }, + /* x^246784 mod p(x)` << 1, x^246848 mod p(x)` << 1 */ + { 0x00000001974aecb0, 0x00000000cbc09168 }, + /* x^245760 mod p(x)` << 1, x^245824 mod p(x)` << 1 */ + { 0x000000008ee3f226, 0x000000005ceeedc2 }, + /* x^244736 mod p(x)` << 1, x^244800 mod p(x)` << 1 */ + { 0x00000001089aba9a, 0x0000000047d74e86 }, + /* x^243712 mod p(x)` << 1, x^243776 mod p(x)` << 1 */ + { 0x0000000065113872, 0x00000001407e9e22 }, + /* x^242688 mod p(x)` << 1, x^242752 mod p(x)` << 1 */ + { 0x000000005c07ec10, 0x00000001da967bda }, + /* x^241664 mod p(x)` << 1, x^241728 mod p(x)` << 1 */ + { 0x0000000187590924, 0x000000006c898368 }, + /* x^240640 mod p(x)` << 1, x^240704 mod p(x)` << 1 */ + { 0x00000000e35da7c6, 0x00000000f2d14c98 }, + /* x^239616 mod p(x)` << 1, x^239680 mod p(x)` << 1 */ + { 0x000000000415855a, 0x00000001993c6ad4 }, + /* x^238592 mod p(x)` << 1, x^238656 mod p(x)` << 1 */ + { 0x0000000073617758, 0x000000014683d1ac }, + /* x^237568 mod p(x)` << 1, x^237632 mod p(x)` << 1 */ + { 0x0000000176021d28, 0x00000001a7c93e6c }, + /* x^236544 mod p(x)` << 1, x^236608 mod p(x)` << 1 */ + { 0x00000001c358fd0a, 0x000000010211e90a }, + /* x^235520 mod p(x)` << 1, x^235584 mod p(x)` << 1 */ + { 0x00000001ff7a2c18, 0x000000001119403e }, + /* x^234496 mod p(x)` << 1, x^234560 mod p(x)` << 1 */ + { 0x00000000f2d9f7e4, 0x000000001c3261aa }, + /* x^233472 mod p(x)` << 1, x^233536 mod p(x)` << 1 */ + { 0x000000016cf1f9c8, 0x000000014e37a634 }, + /* x^232448 mod p(x)` << 1, x^232512 mod p(x)` << 1 */ + { 0x000000010af9279a, 0x0000000073786c0c }, + /* x^231424 mod p(x)` << 1, x^231488 mod p(x)` << 1 */ + { 0x0000000004f101e8, 0x000000011dc037f8 }, + /* x^230400 mod p(x)` << 1, x^230464 mod p(x)` << 1 */ + { 0x0000000070bcf184, 0x0000000031433dfc }, + /* x^229376 mod p(x)` << 1, x^229440 mod p(x)` << 1 */ + { 0x000000000a8de642, 0x000000009cde8348 }, + /* x^228352 mod p(x)` << 1, x^228416 mod p(x)` << 1 */ + { 0x0000000062ea130c, 0x0000000038d3c2a6 }, + /* x^227328 mod p(x)` << 1, x^227392 mod p(x)` << 1 */ + { 0x00000001eb31cbb2, 0x000000011b25f260 }, + /* x^226304 mod p(x)` << 1, x^226368 mod p(x)` << 1 */ + { 0x0000000170783448, 0x000000001629e6f0 }, + /* x^225280 mod p(x)` << 1, x^225344 mod p(x)` << 1 */ + { 0x00000001a684b4c6, 0x0000000160838b4c }, + /* x^224256 mod p(x)` << 1, x^224320 mod p(x)` << 1 */ + { 0x00000000253ca5b4, 0x000000007a44011c }, + /* x^223232 mod p(x)` << 1, x^223296 mod p(x)` << 1 */ + { 0x0000000057b4b1e2, 0x00000000226f417a }, + /* x^222208 mod p(x)` << 1, x^222272 mod p(x)` << 1 */ + { 0x00000000b6bd084c, 0x0000000045eb2eb4 }, + /* x^221184 mod p(x)` << 1, x^221248 mod p(x)` << 1 */ + { 0x0000000123c2d592, 0x000000014459d70c }, + /* x^220160 mod p(x)` << 1, x^220224 mod p(x)` << 1 */ + { 0x00000000159dafce, 0x00000001d406ed82 }, + /* x^219136 mod p(x)` << 1, x^219200 mod p(x)` << 1 */ + { 0x0000000127e1a64e, 0x0000000160c8e1a8 }, + /* x^218112 mod p(x)` << 1, x^218176 mod p(x)` << 1 */ + { 0x0000000056860754, 0x0000000027ba8098 }, + /* x^217088 mod p(x)` << 1, x^217152 mod p(x)` << 1 */ + { 0x00000001e661aae8, 0x000000006d92d018 }, + /* x^216064 mod p(x)` << 1, x^216128 mod p(x)` << 1 */ + { 0x00000000f82c6166, 0x000000012ed7e3f2 }, + /* x^215040 mod p(x)` << 1, x^215104 mod p(x)` << 1 */ + { 0x00000000c4f9c7ae, 0x000000002dc87788 }, + /* x^214016 mod p(x)` << 1, x^214080 mod p(x)` << 1 */ + { 0x0000000074203d20, 0x0000000018240bb8 }, + /* x^212992 mod p(x)` << 1, x^213056 mod p(x)` << 1 */ + { 0x0000000198173052, 0x000000001ad38158 }, + /* x^211968 mod p(x)` << 1, x^212032 mod p(x)` << 1 */ + { 0x00000001ce8aba54, 0x00000001396b78f2 }, + /* x^210944 mod p(x)` << 1, x^211008 mod p(x)` << 1 */ + { 0x00000001850d5d94, 0x000000011a681334 }, + /* x^209920 mod p(x)` << 1, x^209984 mod p(x)` << 1 */ + { 0x00000001d609239c, 0x000000012104732e }, + /* x^208896 mod p(x)` << 1, x^208960 mod p(x)` << 1 */ + { 0x000000001595f048, 0x00000000a140d90c }, + /* x^207872 mod p(x)` << 1, x^207936 mod p(x)` << 1 */ + { 0x0000000042ccee08, 0x00000001b7215eda }, + /* x^206848 mod p(x)` << 1, x^206912 mod p(x)` << 1 */ + { 0x000000010a389d74, 0x00000001aaf1df3c }, + /* x^205824 mod p(x)` << 1, x^205888 mod p(x)` << 1 */ + { 0x000000012a840da6, 0x0000000029d15b8a }, + /* x^204800 mod p(x)` << 1, x^204864 mod p(x)` << 1 */ + { 0x000000001d181c0c, 0x00000000f1a96922 }, + /* x^203776 mod p(x)` << 1, x^203840 mod p(x)` << 1 */ + { 0x0000000068b7d1f6, 0x00000001ac80d03c }, + /* x^202752 mod p(x)` << 1, x^202816 mod p(x)` << 1 */ + { 0x000000005b0f14fc, 0x000000000f11d56a }, + /* x^201728 mod p(x)` << 1, x^201792 mod p(x)` << 1 */ + { 0x0000000179e9e730, 0x00000001f1c022a2 }, + /* x^200704 mod p(x)` << 1, x^200768 mod p(x)` << 1 */ + { 0x00000001ce1368d6, 0x0000000173d00ae2 }, + /* x^199680 mod p(x)` << 1, x^199744 mod p(x)` << 1 */ + { 0x0000000112c3a84c, 0x00000001d4ffe4ac }, + /* x^198656 mod p(x)` << 1, x^198720 mod p(x)` << 1 */ + { 0x00000000de940fee, 0x000000016edc5ae4 }, + /* x^197632 mod p(x)` << 1, x^197696 mod p(x)` << 1 */ + { 0x00000000fe896b7e, 0x00000001f1a02140 }, + /* x^196608 mod p(x)` << 1, x^196672 mod p(x)` << 1 */ + { 0x00000001f797431c, 0x00000000ca0b28a0 }, + /* x^195584 mod p(x)` << 1, x^195648 mod p(x)` << 1 */ + { 0x0000000053e989ba, 0x00000001928e30a2 }, + /* x^194560 mod p(x)` << 1, x^194624 mod p(x)` << 1 */ + { 0x000000003920cd16, 0x0000000097b1b002 }, + /* x^193536 mod p(x)` << 1, x^193600 mod p(x)` << 1 */ + { 0x00000001e6f579b8, 0x00000000b15bf906 }, + /* x^192512 mod p(x)` << 1, x^192576 mod p(x)` << 1 */ + { 0x000000007493cb0a, 0x00000000411c5d52 }, + /* x^191488 mod p(x)` << 1, x^191552 mod p(x)` << 1 */ + { 0x00000001bdd376d8, 0x00000001c36f3300 }, + /* x^190464 mod p(x)` << 1, x^190528 mod p(x)` << 1 */ + { 0x000000016badfee6, 0x00000001119227e0 }, + /* x^189440 mod p(x)` << 1, x^189504 mod p(x)` << 1 */ + { 0x0000000071de5c58, 0x00000000114d4702 }, + /* x^188416 mod p(x)` << 1, x^188480 mod p(x)` << 1 */ + { 0x00000000453f317c, 0x00000000458b5b98 }, + /* x^187392 mod p(x)` << 1, x^187456 mod p(x)` << 1 */ + { 0x0000000121675cce, 0x000000012e31fb8e }, + /* x^186368 mod p(x)` << 1, x^186432 mod p(x)` << 1 */ + { 0x00000001f409ee92, 0x000000005cf619d8 }, + /* x^185344 mod p(x)` << 1, x^185408 mod p(x)` << 1 */ + { 0x00000000f36b9c88, 0x0000000063f4d8b2 }, + /* x^184320 mod p(x)` << 1, x^184384 mod p(x)` << 1 */ + { 0x0000000036b398f4, 0x000000004138dc8a }, + /* x^183296 mod p(x)` << 1, x^183360 mod p(x)` << 1 */ + { 0x00000001748f9adc, 0x00000001d29ee8e0 }, + /* x^182272 mod p(x)` << 1, x^182336 mod p(x)` << 1 */ + { 0x00000001be94ec00, 0x000000006a08ace8 }, + /* x^181248 mod p(x)` << 1, x^181312 mod p(x)` << 1 */ + { 0x00000000b74370d6, 0x0000000127d42010 }, + /* x^180224 mod p(x)` << 1, x^180288 mod p(x)` << 1 */ + { 0x00000001174d0b98, 0x0000000019d76b62 }, + /* x^179200 mod p(x)` << 1, x^179264 mod p(x)` << 1 */ + { 0x00000000befc06a4, 0x00000001b1471f6e }, + /* x^178176 mod p(x)` << 1, x^178240 mod p(x)` << 1 */ + { 0x00000001ae125288, 0x00000001f64c19cc }, + /* x^177152 mod p(x)` << 1, x^177216 mod p(x)` << 1 */ + { 0x0000000095c19b34, 0x00000000003c0ea0 }, + /* x^176128 mod p(x)` << 1, x^176192 mod p(x)` << 1 */ + { 0x00000001a78496f2, 0x000000014d73abf6 }, + /* x^175104 mod p(x)` << 1, x^175168 mod p(x)` << 1 */ + { 0x00000001ac5390a0, 0x00000001620eb844 }, + /* x^174080 mod p(x)` << 1, x^174144 mod p(x)` << 1 */ + { 0x000000002a80ed6e, 0x0000000147655048 }, + /* x^173056 mod p(x)` << 1, x^173120 mod p(x)` << 1 */ + { 0x00000001fa9b0128, 0x0000000067b5077e }, + /* x^172032 mod p(x)` << 1, x^172096 mod p(x)` << 1 */ + { 0x00000001ea94929e, 0x0000000010ffe206 }, + /* x^171008 mod p(x)` << 1, x^171072 mod p(x)` << 1 */ + { 0x0000000125f4305c, 0x000000000fee8f1e }, + /* x^169984 mod p(x)` << 1, x^170048 mod p(x)` << 1 */ + { 0x00000001471e2002, 0x00000001da26fbae }, + /* x^168960 mod p(x)` << 1, x^169024 mod p(x)` << 1 */ + { 0x0000000132d2253a, 0x00000001b3a8bd88 }, + /* x^167936 mod p(x)` << 1, x^168000 mod p(x)` << 1 */ + { 0x00000000f26b3592, 0x00000000e8f3898e }, + /* x^166912 mod p(x)` << 1, x^166976 mod p(x)` << 1 */ + { 0x00000000bc8b67b0, 0x00000000b0d0d28c }, + /* x^165888 mod p(x)` << 1, x^165952 mod p(x)` << 1 */ + { 0x000000013a826ef2, 0x0000000030f2a798 }, + /* x^164864 mod p(x)` << 1, x^164928 mod p(x)` << 1 */ + { 0x0000000081482c84, 0x000000000fba1002 }, + /* x^163840 mod p(x)` << 1, x^163904 mod p(x)` << 1 */ + { 0x00000000e77307c2, 0x00000000bdb9bd72 }, + /* x^162816 mod p(x)` << 1, x^162880 mod p(x)` << 1 */ + { 0x00000000d4a07ec8, 0x0000000075d3bf5a }, + /* x^161792 mod p(x)` << 1, x^161856 mod p(x)` << 1 */ + { 0x0000000017102100, 0x00000000ef1f98a0 }, + /* x^160768 mod p(x)` << 1, x^160832 mod p(x)` << 1 */ + { 0x00000000db406486, 0x00000000689c7602 }, + /* x^159744 mod p(x)` << 1, x^159808 mod p(x)` << 1 */ + { 0x0000000192db7f88, 0x000000016d5fa5fe }, + /* x^158720 mod p(x)` << 1, x^158784 mod p(x)` << 1 */ + { 0x000000018bf67b1e, 0x00000001d0d2b9ca }, + /* x^157696 mod p(x)` << 1, x^157760 mod p(x)` << 1 */ + { 0x000000007c09163e, 0x0000000041e7b470 }, + /* x^156672 mod p(x)` << 1, x^156736 mod p(x)` << 1 */ + { 0x000000000adac060, 0x00000001cbb6495e }, + /* x^155648 mod p(x)` << 1, x^155712 mod p(x)` << 1 */ + { 0x00000000bd8316ae, 0x000000010052a0b0 }, + /* x^154624 mod p(x)` << 1, x^154688 mod p(x)` << 1 */ + { 0x000000019f09ab54, 0x00000001d8effb5c }, + /* x^153600 mod p(x)` << 1, x^153664 mod p(x)` << 1 */ + { 0x0000000125155542, 0x00000001d969853c }, + /* x^152576 mod p(x)` << 1, x^152640 mod p(x)` << 1 */ + { 0x000000018fdb5882, 0x00000000523ccce2 }, + /* x^151552 mod p(x)` << 1, x^151616 mod p(x)` << 1 */ + { 0x00000000e794b3f4, 0x000000001e2436bc }, + /* x^150528 mod p(x)` << 1, x^150592 mod p(x)` << 1 */ + { 0x000000016f9bb022, 0x00000000ddd1c3a2 }, + /* x^149504 mod p(x)` << 1, x^149568 mod p(x)` << 1 */ + { 0x00000000290c9978, 0x0000000019fcfe38 }, + /* x^148480 mod p(x)` << 1, x^148544 mod p(x)` << 1 */ + { 0x0000000083c0f350, 0x00000001ce95db64 }, + /* x^147456 mod p(x)` << 1, x^147520 mod p(x)` << 1 */ + { 0x0000000173ea6628, 0x00000000af582806 }, + /* x^146432 mod p(x)` << 1, x^146496 mod p(x)` << 1 */ + { 0x00000001c8b4e00a, 0x00000001006388f6 }, + /* x^145408 mod p(x)` << 1, x^145472 mod p(x)` << 1 */ + { 0x00000000de95d6aa, 0x0000000179eca00a }, + /* x^144384 mod p(x)` << 1, x^144448 mod p(x)` << 1 */ + { 0x000000010b7f7248, 0x0000000122410a6a }, + /* x^143360 mod p(x)` << 1, x^143424 mod p(x)` << 1 */ + { 0x00000001326e3a06, 0x000000004288e87c }, + /* x^142336 mod p(x)` << 1, x^142400 mod p(x)` << 1 */ + { 0x00000000bb62c2e6, 0x000000016c5490da }, + /* x^141312 mod p(x)` << 1, x^141376 mod p(x)` << 1 */ + { 0x0000000156a4b2c2, 0x00000000d1c71f6e }, + /* x^140288 mod p(x)` << 1, x^140352 mod p(x)` << 1 */ + { 0x000000011dfe763a, 0x00000001b4ce08a6 }, + /* x^139264 mod p(x)` << 1, x^139328 mod p(x)` << 1 */ + { 0x000000007bcca8e2, 0x00000001466ba60c }, + /* x^138240 mod p(x)` << 1, x^138304 mod p(x)` << 1 */ + { 0x0000000186118faa, 0x00000001f6c488a4 }, + /* x^137216 mod p(x)` << 1, x^137280 mod p(x)` << 1 */ + { 0x0000000111a65a88, 0x000000013bfb0682 }, + /* x^136192 mod p(x)` << 1, x^136256 mod p(x)` << 1 */ + { 0x000000003565e1c4, 0x00000000690e9e54 }, + /* x^135168 mod p(x)` << 1, x^135232 mod p(x)` << 1 */ + { 0x000000012ed02a82, 0x00000000281346b6 }, + /* x^134144 mod p(x)` << 1, x^134208 mod p(x)` << 1 */ + { 0x00000000c486ecfc, 0x0000000156464024 }, + /* x^133120 mod p(x)` << 1, x^133184 mod p(x)` << 1 */ + { 0x0000000001b951b2, 0x000000016063a8dc }, + /* x^132096 mod p(x)` << 1, x^132160 mod p(x)` << 1 */ + { 0x0000000048143916, 0x0000000116a66362 }, + /* x^131072 mod p(x)` << 1, x^131136 mod p(x)` << 1 */ + { 0x00000001dc2ae124, 0x000000017e8aa4d2 }, + /* x^130048 mod p(x)` << 1, x^130112 mod p(x)` << 1 */ + { 0x00000001416c58d6, 0x00000001728eb10c }, + /* x^129024 mod p(x)` << 1, x^129088 mod p(x)` << 1 */ + { 0x00000000a479744a, 0x00000001b08fd7fa }, + /* x^128000 mod p(x)` << 1, x^128064 mod p(x)` << 1 */ + { 0x0000000096ca3a26, 0x00000001092a16e8 }, + /* x^126976 mod p(x)` << 1, x^127040 mod p(x)` << 1 */ + { 0x00000000ff223d4e, 0x00000000a505637c }, + /* x^125952 mod p(x)` << 1, x^126016 mod p(x)` << 1 */ + { 0x000000010e84da42, 0x00000000d94869b2 }, + /* x^124928 mod p(x)` << 1, x^124992 mod p(x)` << 1 */ + { 0x00000001b61ba3d0, 0x00000001c8b203ae }, + /* x^123904 mod p(x)` << 1, x^123968 mod p(x)` << 1 */ + { 0x00000000680f2de8, 0x000000005704aea0 }, + /* x^122880 mod p(x)` << 1, x^122944 mod p(x)` << 1 */ + { 0x000000008772a9a8, 0x000000012e295fa2 }, + /* x^121856 mod p(x)` << 1, x^121920 mod p(x)` << 1 */ + { 0x0000000155f295bc, 0x000000011d0908bc }, + /* x^120832 mod p(x)` << 1, x^120896 mod p(x)` << 1 */ + { 0x00000000595f9282, 0x0000000193ed97ea }, + /* x^119808 mod p(x)` << 1, x^119872 mod p(x)` << 1 */ + { 0x0000000164b1c25a, 0x000000013a0f1c52 }, + /* x^118784 mod p(x)` << 1, x^118848 mod p(x)` << 1 */ + { 0x00000000fbd67c50, 0x000000010c2c40c0 }, + /* x^117760 mod p(x)` << 1, x^117824 mod p(x)` << 1 */ + { 0x0000000096076268, 0x00000000ff6fac3e }, + /* x^116736 mod p(x)` << 1, x^116800 mod p(x)` << 1 */ + { 0x00000001d288e4cc, 0x000000017b3609c0 }, + /* x^115712 mod p(x)` << 1, x^115776 mod p(x)` << 1 */ + { 0x00000001eaac1bdc, 0x0000000088c8c922 }, + /* x^114688 mod p(x)` << 1, x^114752 mod p(x)` << 1 */ + { 0x00000001f1ea39e2, 0x00000001751baae6 }, + /* x^113664 mod p(x)` << 1, x^113728 mod p(x)` << 1 */ + { 0x00000001eb6506fc, 0x0000000107952972 }, + /* x^112640 mod p(x)` << 1, x^112704 mod p(x)` << 1 */ + { 0x000000010f806ffe, 0x0000000162b00abe }, + /* x^111616 mod p(x)` << 1, x^111680 mod p(x)` << 1 */ + { 0x000000010408481e, 0x000000000d7b404c }, + /* x^110592 mod p(x)` << 1, x^110656 mod p(x)` << 1 */ + { 0x0000000188260534, 0x00000000763b13d4 }, + /* x^109568 mod p(x)` << 1, x^109632 mod p(x)` << 1 */ + { 0x0000000058fc73e0, 0x00000000f6dc22d8 }, + /* x^108544 mod p(x)` << 1, x^108608 mod p(x)` << 1 */ + { 0x00000000391c59b8, 0x000000007daae060 }, + /* x^107520 mod p(x)` << 1, x^107584 mod p(x)` << 1 */ + { 0x000000018b638400, 0x000000013359ab7c }, + /* x^106496 mod p(x)` << 1, x^106560 mod p(x)` << 1 */ + { 0x000000011738f5c4, 0x000000008add438a }, + /* x^105472 mod p(x)` << 1, x^105536 mod p(x)` << 1 */ + { 0x000000008cf7c6da, 0x00000001edbefdea }, + /* x^104448 mod p(x)` << 1, x^104512 mod p(x)` << 1 */ + { 0x00000001ef97fb16, 0x000000004104e0f8 }, + /* x^103424 mod p(x)` << 1, x^103488 mod p(x)` << 1 */ + { 0x0000000102130e20, 0x00000000b48a8222 }, + /* x^102400 mod p(x)` << 1, x^102464 mod p(x)` << 1 */ + { 0x00000000db968898, 0x00000001bcb46844 }, + /* x^101376 mod p(x)` << 1, x^101440 mod p(x)` << 1 */ + { 0x00000000b5047b5e, 0x000000013293ce0a }, + /* x^100352 mod p(x)` << 1, x^100416 mod p(x)` << 1 */ + { 0x000000010b90fdb2, 0x00000001710d0844 }, + /* x^99328 mod p(x)` << 1, x^99392 mod p(x)` << 1 */ + { 0x000000004834a32e, 0x0000000117907f6e }, + /* x^98304 mod p(x)` << 1, x^98368 mod p(x)` << 1 */ + { 0x0000000059c8f2b0, 0x0000000087ddf93e }, + /* x^97280 mod p(x)` << 1, x^97344 mod p(x)` << 1 */ + { 0x0000000122cec508, 0x000000005970e9b0 }, + /* x^96256 mod p(x)` << 1, x^96320 mod p(x)` << 1 */ + { 0x000000000a330cda, 0x0000000185b2b7d0 }, + /* x^95232 mod p(x)` << 1, x^95296 mod p(x)` << 1 */ + { 0x000000014a47148c, 0x00000001dcee0efc }, + /* x^94208 mod p(x)` << 1, x^94272 mod p(x)` << 1 */ + { 0x0000000042c61cb8, 0x0000000030da2722 }, + /* x^93184 mod p(x)` << 1, x^93248 mod p(x)` << 1 */ + { 0x0000000012fe6960, 0x000000012f925a18 }, + /* x^92160 mod p(x)` << 1, x^92224 mod p(x)` << 1 */ + { 0x00000000dbda2c20, 0x00000000dd2e357c }, + /* x^91136 mod p(x)` << 1, x^91200 mod p(x)` << 1 */ + { 0x000000011122410c, 0x00000000071c80de }, + /* x^90112 mod p(x)` << 1, x^90176 mod p(x)` << 1 */ + { 0x00000000977b2070, 0x000000011513140a }, + /* x^89088 mod p(x)` << 1, x^89152 mod p(x)` << 1 */ + { 0x000000014050438e, 0x00000001df876e8e }, + /* x^88064 mod p(x)` << 1, x^88128 mod p(x)` << 1 */ + { 0x0000000147c840e8, 0x000000015f81d6ce }, + /* x^87040 mod p(x)` << 1, x^87104 mod p(x)` << 1 */ + { 0x00000001cc7c88ce, 0x000000019dd94dbe }, + /* x^86016 mod p(x)` << 1, x^86080 mod p(x)` << 1 */ + { 0x00000001476b35a4, 0x00000001373d206e }, + /* x^84992 mod p(x)` << 1, x^85056 mod p(x)` << 1 */ + { 0x000000013d52d508, 0x00000000668ccade }, + /* x^83968 mod p(x)` << 1, x^84032 mod p(x)` << 1 */ + { 0x000000008e4be32e, 0x00000001b192d268 }, + /* x^82944 mod p(x)` << 1, x^83008 mod p(x)` << 1 */ + { 0x00000000024120fe, 0x00000000e30f3a78 }, + /* x^81920 mod p(x)` << 1, x^81984 mod p(x)` << 1 */ + { 0x00000000ddecddb4, 0x000000010ef1f7bc }, + /* x^80896 mod p(x)` << 1, x^80960 mod p(x)` << 1 */ + { 0x00000000d4d403bc, 0x00000001f5ac7380 }, + /* x^79872 mod p(x)` << 1, x^79936 mod p(x)` << 1 */ + { 0x00000001734b89aa, 0x000000011822ea70 }, + /* x^78848 mod p(x)` << 1, x^78912 mod p(x)` << 1 */ + { 0x000000010e7a58d6, 0x00000000c3a33848 }, + /* x^77824 mod p(x)` << 1, x^77888 mod p(x)` << 1 */ + { 0x00000001f9f04e9c, 0x00000001bd151c24 }, + /* x^76800 mod p(x)` << 1, x^76864 mod p(x)` << 1 */ + { 0x00000000b692225e, 0x0000000056002d76 }, + /* x^75776 mod p(x)` << 1, x^75840 mod p(x)` << 1 */ + { 0x000000019b8d3f3e, 0x000000014657c4f4 }, + /* x^74752 mod p(x)` << 1, x^74816 mod p(x)` << 1 */ + { 0x00000001a874f11e, 0x0000000113742d7c }, + /* x^73728 mod p(x)` << 1, x^73792 mod p(x)` << 1 */ + { 0x000000010d5a4254, 0x000000019c5920ba }, + /* x^72704 mod p(x)` << 1, x^72768 mod p(x)` << 1 */ + { 0x00000000bbb2f5d6, 0x000000005216d2d6 }, + /* x^71680 mod p(x)` << 1, x^71744 mod p(x)` << 1 */ + { 0x0000000179cc0e36, 0x0000000136f5ad8a }, + /* x^70656 mod p(x)` << 1, x^70720 mod p(x)` << 1 */ + { 0x00000001dca1da4a, 0x000000018b07beb6 }, + /* x^69632 mod p(x)` << 1, x^69696 mod p(x)` << 1 */ + { 0x00000000feb1a192, 0x00000000db1e93b0 }, + /* x^68608 mod p(x)` << 1, x^68672 mod p(x)` << 1 */ + { 0x00000000d1eeedd6, 0x000000000b96fa3a }, + /* x^67584 mod p(x)` << 1, x^67648 mod p(x)` << 1 */ + { 0x000000008fad9bb4, 0x00000001d9968af0 }, + /* x^66560 mod p(x)` << 1, x^66624 mod p(x)` << 1 */ + { 0x00000001884938e4, 0x000000000e4a77a2 }, + /* x^65536 mod p(x)` << 1, x^65600 mod p(x)` << 1 */ + { 0x00000001bc2e9bc0, 0x00000000508c2ac8 }, + /* x^64512 mod p(x)` << 1, x^64576 mod p(x)` << 1 */ + { 0x00000001f9658a68, 0x0000000021572a80 }, + /* x^63488 mod p(x)` << 1, x^63552 mod p(x)` << 1 */ + { 0x000000001b9224fc, 0x00000001b859daf2 }, + /* x^62464 mod p(x)` << 1, x^62528 mod p(x)` << 1 */ + { 0x0000000055b2fb84, 0x000000016f788474 }, + /* x^61440 mod p(x)` << 1, x^61504 mod p(x)` << 1 */ + { 0x000000018b090348, 0x00000001b438810e }, + /* x^60416 mod p(x)` << 1, x^60480 mod p(x)` << 1 */ + { 0x000000011ccbd5ea, 0x0000000095ddc6f2 }, + /* x^59392 mod p(x)` << 1, x^59456 mod p(x)` << 1 */ + { 0x0000000007ae47f8, 0x00000001d977c20c }, + /* x^58368 mod p(x)` << 1, x^58432 mod p(x)` << 1 */ + { 0x0000000172acbec0, 0x00000000ebedb99a }, + /* x^57344 mod p(x)` << 1, x^57408 mod p(x)` << 1 */ + { 0x00000001c6e3ff20, 0x00000001df9e9e92 }, + /* x^56320 mod p(x)` << 1, x^56384 mod p(x)` << 1 */ + { 0x00000000e1b38744, 0x00000001a4a3f952 }, + /* x^55296 mod p(x)` << 1, x^55360 mod p(x)` << 1 */ + { 0x00000000791585b2, 0x00000000e2f51220 }, + /* x^54272 mod p(x)` << 1, x^54336 mod p(x)` << 1 */ + { 0x00000000ac53b894, 0x000000004aa01f3e }, + /* x^53248 mod p(x)` << 1, x^53312 mod p(x)` << 1 */ + { 0x00000001ed5f2cf4, 0x00000000b3e90a58 }, + /* x^52224 mod p(x)` << 1, x^52288 mod p(x)` << 1 */ + { 0x00000001df48b2e0, 0x000000000c9ca2aa }, + /* x^51200 mod p(x)` << 1, x^51264 mod p(x)` << 1 */ + { 0x00000000049c1c62, 0x0000000151682316 }, + /* x^50176 mod p(x)` << 1, x^50240 mod p(x)` << 1 */ + { 0x000000017c460c12, 0x0000000036fce78c }, + /* x^49152 mod p(x)` << 1, x^49216 mod p(x)` << 1 */ + { 0x000000015be4da7e, 0x000000009037dc10 }, + /* x^48128 mod p(x)` << 1, x^48192 mod p(x)` << 1 */ + { 0x000000010f38f668, 0x00000000d3298582 }, + /* x^47104 mod p(x)` << 1, x^47168 mod p(x)` << 1 */ + { 0x0000000039f40a00, 0x00000001b42e8ad6 }, + /* x^46080 mod p(x)` << 1, x^46144 mod p(x)` << 1 */ + { 0x00000000bd4c10c4, 0x00000000142a9838 }, + /* x^45056 mod p(x)` << 1, x^45120 mod p(x)` << 1 */ + { 0x0000000042db1d98, 0x0000000109c7f190 }, + /* x^44032 mod p(x)` << 1, x^44096 mod p(x)` << 1 */ + { 0x00000001c905bae6, 0x0000000056ff9310 }, + /* x^43008 mod p(x)` << 1, x^43072 mod p(x)` << 1 */ + { 0x00000000069d40ea, 0x00000001594513aa }, + /* x^41984 mod p(x)` << 1, x^42048 mod p(x)` << 1 */ + { 0x000000008e4fbad0, 0x00000001e3b5b1e8 }, + /* x^40960 mod p(x)` << 1, x^41024 mod p(x)` << 1 */ + { 0x0000000047bedd46, 0x000000011dd5fc08 }, + /* x^39936 mod p(x)` << 1, x^40000 mod p(x)` << 1 */ + { 0x0000000026396bf8, 0x00000001675f0cc2 }, + /* x^38912 mod p(x)` << 1, x^38976 mod p(x)` << 1 */ + { 0x00000000379beb92, 0x00000000d1c8dd44 }, + /* x^37888 mod p(x)` << 1, x^37952 mod p(x)` << 1 */ + { 0x000000000abae54a, 0x0000000115ebd3d8 }, + /* x^36864 mod p(x)` << 1, x^36928 mod p(x)` << 1 */ + { 0x0000000007e6a128, 0x00000001ecbd0dac }, + /* x^35840 mod p(x)` << 1, x^35904 mod p(x)` << 1 */ + { 0x000000000ade29d2, 0x00000000cdf67af2 }, + /* x^34816 mod p(x)` << 1, x^34880 mod p(x)` << 1 */ + { 0x00000000f974c45c, 0x000000004c01ff4c }, + /* x^33792 mod p(x)` << 1, x^33856 mod p(x)` << 1 */ + { 0x00000000e77ac60a, 0x00000000f2d8657e }, + /* x^32768 mod p(x)` << 1, x^32832 mod p(x)` << 1 */ + { 0x0000000145895816, 0x000000006bae74c4 }, + /* x^31744 mod p(x)` << 1, x^31808 mod p(x)` << 1 */ + { 0x0000000038e362be, 0x0000000152af8aa0 }, + /* x^30720 mod p(x)` << 1, x^30784 mod p(x)` << 1 */ + { 0x000000007f991a64, 0x0000000004663802 }, + /* x^29696 mod p(x)` << 1, x^29760 mod p(x)` << 1 */ + { 0x00000000fa366d3a, 0x00000001ab2f5afc }, + /* x^28672 mod p(x)` << 1, x^28736 mod p(x)` << 1 */ + { 0x00000001a2bb34f0, 0x0000000074a4ebd4 }, + /* x^27648 mod p(x)` << 1, x^27712 mod p(x)` << 1 */ + { 0x0000000028a9981e, 0x00000001d7ab3a4c }, + /* x^26624 mod p(x)` << 1, x^26688 mod p(x)` << 1 */ + { 0x00000001dbc672be, 0x00000001a8da60c6 }, + /* x^25600 mod p(x)` << 1, x^25664 mod p(x)` << 1 */ + { 0x00000000b04d77f6, 0x000000013cf63820 }, + /* x^24576 mod p(x)` << 1, x^24640 mod p(x)` << 1 */ + { 0x0000000124400d96, 0x00000000bec12e1e }, + /* x^23552 mod p(x)` << 1, x^23616 mod p(x)` << 1 */ + { 0x000000014ca4b414, 0x00000001c6368010 }, + /* x^22528 mod p(x)` << 1, x^22592 mod p(x)` << 1 */ + { 0x000000012fe2c938, 0x00000001e6e78758 }, + /* x^21504 mod p(x)` << 1, x^21568 mod p(x)` << 1 */ + { 0x00000001faed01e6, 0x000000008d7f2b3c }, + /* x^20480 mod p(x)` << 1, x^20544 mod p(x)` << 1 */ + { 0x000000007e80ecfe, 0x000000016b4a156e }, + /* x^19456 mod p(x)` << 1, x^19520 mod p(x)` << 1 */ + { 0x0000000098daee94, 0x00000001c63cfeb6 }, + /* x^18432 mod p(x)` << 1, x^18496 mod p(x)` << 1 */ + { 0x000000010a04edea, 0x000000015f902670 }, + /* x^17408 mod p(x)` << 1, x^17472 mod p(x)` << 1 */ + { 0x00000001c00b4524, 0x00000001cd5de11e }, + /* x^16384 mod p(x)` << 1, x^16448 mod p(x)` << 1 */ + { 0x0000000170296550, 0x000000001acaec54 }, + /* x^15360 mod p(x)` << 1, x^15424 mod p(x)` << 1 */ + { 0x0000000181afaa48, 0x000000002bd0ca78 }, + /* x^14336 mod p(x)` << 1, x^14400 mod p(x)` << 1 */ + { 0x0000000185a31ffa, 0x0000000032d63d5c }, + /* x^13312 mod p(x)` << 1, x^13376 mod p(x)` << 1 */ + { 0x000000002469f608, 0x000000001c6d4e4c }, + /* x^12288 mod p(x)` << 1, x^12352 mod p(x)` << 1 */ + { 0x000000006980102a, 0x0000000106a60b92 }, + /* x^11264 mod p(x)` << 1, x^11328 mod p(x)` << 1 */ + { 0x0000000111ea9ca8, 0x00000000d3855e12 }, + /* x^10240 mod p(x)` << 1, x^10304 mod p(x)` << 1 */ + { 0x00000001bd1d29ce, 0x00000000e3125636 }, + /* x^9216 mod p(x)` << 1, x^9280 mod p(x)` << 1 */ + { 0x00000001b34b9580, 0x000000009e8f7ea4 }, + /* x^8192 mod p(x)` << 1, x^8256 mod p(x)` << 1 */ + { 0x000000003076054e, 0x00000001c82e562c }, + /* x^7168 mod p(x)` << 1, x^7232 mod p(x)` << 1 */ + { 0x000000012a608ea4, 0x00000000ca9f09ce }, + /* x^6144 mod p(x)` << 1, x^6208 mod p(x)` << 1 */ + { 0x00000000784d05fe, 0x00000000c63764e6 }, + /* x^5120 mod p(x)` << 1, x^5184 mod p(x)` << 1 */ + { 0x000000016ef0d82a, 0x0000000168d2e49e }, + /* x^4096 mod p(x)` << 1, x^4160 mod p(x)` << 1 */ + { 0x0000000075bda454, 0x00000000e986c148 }, + /* x^3072 mod p(x)` << 1, x^3136 mod p(x)` << 1 */ + { 0x000000003dc0a1c4, 0x00000000cfb65894 }, + /* x^2048 mod p(x)` << 1, x^2112 mod p(x)` << 1 */ + { 0x00000000e9a5d8be, 0x0000000111cadee4 }, + /* x^1024 mod p(x)` << 1, x^1088 mod p(x)` << 1 */ + { 0x00000001609bc4b4, 0x0000000171fb63ce } +#else /* __LITTLE_ENDIAN__ */ + /* x^261120 mod p(x)` << 1, x^261184 mod p(x)` << 1 */ + { 0x00000000b6ca9e20, 0x000000009c37c408 }, + /* x^260096 mod p(x)` << 1, x^260160 mod p(x)` << 1 */ + { 0x00000000350249a8, 0x00000001b51df26c }, + /* x^259072 mod p(x)` << 1, x^259136 mod p(x)` << 1 */ + { 0x00000001862dac54, 0x000000000724b9d0 }, + /* x^258048 mod p(x)` << 1, x^258112 mod p(x)` << 1 */ + { 0x00000001d87fb48c, 0x00000001c00532fe }, + /* x^257024 mod p(x)` << 1, x^257088 mod p(x)` << 1 */ + { 0x00000001f39b699e, 0x00000000f05a9362 }, + /* x^256000 mod p(x)` << 1, x^256064 mod p(x)` << 1 */ + { 0x0000000101da11b4, 0x00000001e1007970 }, + /* x^254976 mod p(x)` << 1, x^255040 mod p(x)` << 1 */ + { 0x00000001cab571e0, 0x00000000a57366ee }, + /* x^253952 mod p(x)` << 1, x^254016 mod p(x)` << 1 */ + { 0x00000000c7020cfe, 0x0000000192011284 }, + /* x^252928 mod p(x)` << 1, x^252992 mod p(x)` << 1 */ + { 0x00000000cdaed1ae, 0x0000000162716d9a }, + /* x^251904 mod p(x)` << 1, x^251968 mod p(x)` << 1 */ + { 0x00000001e804effc, 0x00000000cd97ecde }, + /* x^250880 mod p(x)` << 1, x^250944 mod p(x)` << 1 */ + { 0x0000000077c3ea3a, 0x0000000058812bc0 }, + /* x^249856 mod p(x)` << 1, x^249920 mod p(x)` << 1 */ + { 0x0000000068df31b4, 0x0000000088b8c12e }, + /* x^248832 mod p(x)` << 1, x^248896 mod p(x)` << 1 */ + { 0x00000000b059b6c2, 0x00000001230b234c }, + /* x^247808 mod p(x)` << 1, x^247872 mod p(x)` << 1 */ + { 0x0000000145fb8ed8, 0x00000001120b416e }, + /* x^246784 mod p(x)` << 1, x^246848 mod p(x)` << 1 */ + { 0x00000000cbc09168, 0x00000001974aecb0 }, + /* x^245760 mod p(x)` << 1, x^245824 mod p(x)` << 1 */ + { 0x000000005ceeedc2, 0x000000008ee3f226 }, + /* x^244736 mod p(x)` << 1, x^244800 mod p(x)` << 1 */ + { 0x0000000047d74e86, 0x00000001089aba9a }, + /* x^243712 mod p(x)` << 1, x^243776 mod p(x)` << 1 */ + { 0x00000001407e9e22, 0x0000000065113872 }, + /* x^242688 mod p(x)` << 1, x^242752 mod p(x)` << 1 */ + { 0x00000001da967bda, 0x000000005c07ec10 }, + /* x^241664 mod p(x)` << 1, x^241728 mod p(x)` << 1 */ + { 0x000000006c898368, 0x0000000187590924 }, + /* x^240640 mod p(x)` << 1, x^240704 mod p(x)` << 1 */ + { 0x00000000f2d14c98, 0x00000000e35da7c6 }, + /* x^239616 mod p(x)` << 1, x^239680 mod p(x)` << 1 */ + { 0x00000001993c6ad4, 0x000000000415855a }, + /* x^238592 mod p(x)` << 1, x^238656 mod p(x)` << 1 */ + { 0x000000014683d1ac, 0x0000000073617758 }, + /* x^237568 mod p(x)` << 1, x^237632 mod p(x)` << 1 */ + { 0x00000001a7c93e6c, 0x0000000176021d28 }, + /* x^236544 mod p(x)` << 1, x^236608 mod p(x)` << 1 */ + { 0x000000010211e90a, 0x00000001c358fd0a }, + /* x^235520 mod p(x)` << 1, x^235584 mod p(x)` << 1 */ + { 0x000000001119403e, 0x00000001ff7a2c18 }, + /* x^234496 mod p(x)` << 1, x^234560 mod p(x)` << 1 */ + { 0x000000001c3261aa, 0x00000000f2d9f7e4 }, + /* x^233472 mod p(x)` << 1, x^233536 mod p(x)` << 1 */ + { 0x000000014e37a634, 0x000000016cf1f9c8 }, + /* x^232448 mod p(x)` << 1, x^232512 mod p(x)` << 1 */ + { 0x0000000073786c0c, 0x000000010af9279a }, + /* x^231424 mod p(x)` << 1, x^231488 mod p(x)` << 1 */ + { 0x000000011dc037f8, 0x0000000004f101e8 }, + /* x^230400 mod p(x)` << 1, x^230464 mod p(x)` << 1 */ + { 0x0000000031433dfc, 0x0000000070bcf184 }, + /* x^229376 mod p(x)` << 1, x^229440 mod p(x)` << 1 */ + { 0x000000009cde8348, 0x000000000a8de642 }, + /* x^228352 mod p(x)` << 1, x^228416 mod p(x)` << 1 */ + { 0x0000000038d3c2a6, 0x0000000062ea130c }, + /* x^227328 mod p(x)` << 1, x^227392 mod p(x)` << 1 */ + { 0x000000011b25f260, 0x00000001eb31cbb2 }, + /* x^226304 mod p(x)` << 1, x^226368 mod p(x)` << 1 */ + { 0x000000001629e6f0, 0x0000000170783448 }, + /* x^225280 mod p(x)` << 1, x^225344 mod p(x)` << 1 */ + { 0x0000000160838b4c, 0x00000001a684b4c6 }, + /* x^224256 mod p(x)` << 1, x^224320 mod p(x)` << 1 */ + { 0x000000007a44011c, 0x00000000253ca5b4 }, + /* x^223232 mod p(x)` << 1, x^223296 mod p(x)` << 1 */ + { 0x00000000226f417a, 0x0000000057b4b1e2 }, + /* x^222208 mod p(x)` << 1, x^222272 mod p(x)` << 1 */ + { 0x0000000045eb2eb4, 0x00000000b6bd084c }, + /* x^221184 mod p(x)` << 1, x^221248 mod p(x)` << 1 */ + { 0x000000014459d70c, 0x0000000123c2d592 }, + /* x^220160 mod p(x)` << 1, x^220224 mod p(x)` << 1 */ + { 0x00000001d406ed82, 0x00000000159dafce }, + /* x^219136 mod p(x)` << 1, x^219200 mod p(x)` << 1 */ + { 0x0000000160c8e1a8, 0x0000000127e1a64e }, + /* x^218112 mod p(x)` << 1, x^218176 mod p(x)` << 1 */ + { 0x0000000027ba8098, 0x0000000056860754 }, + /* x^217088 mod p(x)` << 1, x^217152 mod p(x)` << 1 */ + { 0x000000006d92d018, 0x00000001e661aae8 }, + /* x^216064 mod p(x)` << 1, x^216128 mod p(x)` << 1 */ + { 0x000000012ed7e3f2, 0x00000000f82c6166 }, + /* x^215040 mod p(x)` << 1, x^215104 mod p(x)` << 1 */ + { 0x000000002dc87788, 0x00000000c4f9c7ae }, + /* x^214016 mod p(x)` << 1, x^214080 mod p(x)` << 1 */ + { 0x0000000018240bb8, 0x0000000074203d20 }, + /* x^212992 mod p(x)` << 1, x^213056 mod p(x)` << 1 */ + { 0x000000001ad38158, 0x0000000198173052 }, + /* x^211968 mod p(x)` << 1, x^212032 mod p(x)` << 1 */ + { 0x00000001396b78f2, 0x00000001ce8aba54 }, + /* x^210944 mod p(x)` << 1, x^211008 mod p(x)` << 1 */ + { 0x000000011a681334, 0x00000001850d5d94 }, + /* x^209920 mod p(x)` << 1, x^209984 mod p(x)` << 1 */ + { 0x000000012104732e, 0x00000001d609239c }, + /* x^208896 mod p(x)` << 1, x^208960 mod p(x)` << 1 */ + { 0x00000000a140d90c, 0x000000001595f048 }, + /* x^207872 mod p(x)` << 1, x^207936 mod p(x)` << 1 */ + { 0x00000001b7215eda, 0x0000000042ccee08 }, + /* x^206848 mod p(x)` << 1, x^206912 mod p(x)` << 1 */ + { 0x00000001aaf1df3c, 0x000000010a389d74 }, + /* x^205824 mod p(x)` << 1, x^205888 mod p(x)` << 1 */ + { 0x0000000029d15b8a, 0x000000012a840da6 }, + /* x^204800 mod p(x)` << 1, x^204864 mod p(x)` << 1 */ + { 0x00000000f1a96922, 0x000000001d181c0c }, + /* x^203776 mod p(x)` << 1, x^203840 mod p(x)` << 1 */ + { 0x00000001ac80d03c, 0x0000000068b7d1f6 }, + /* x^202752 mod p(x)` << 1, x^202816 mod p(x)` << 1 */ + { 0x000000000f11d56a, 0x000000005b0f14fc }, + /* x^201728 mod p(x)` << 1, x^201792 mod p(x)` << 1 */ + { 0x00000001f1c022a2, 0x0000000179e9e730 }, + /* x^200704 mod p(x)` << 1, x^200768 mod p(x)` << 1 */ + { 0x0000000173d00ae2, 0x00000001ce1368d6 }, + /* x^199680 mod p(x)` << 1, x^199744 mod p(x)` << 1 */ + { 0x00000001d4ffe4ac, 0x0000000112c3a84c }, + /* x^198656 mod p(x)` << 1, x^198720 mod p(x)` << 1 */ + { 0x000000016edc5ae4, 0x00000000de940fee }, + /* x^197632 mod p(x)` << 1, x^197696 mod p(x)` << 1 */ + { 0x00000001f1a02140, 0x00000000fe896b7e }, + /* x^196608 mod p(x)` << 1, x^196672 mod p(x)` << 1 */ + { 0x00000000ca0b28a0, 0x00000001f797431c }, + /* x^195584 mod p(x)` << 1, x^195648 mod p(x)` << 1 */ + { 0x00000001928e30a2, 0x0000000053e989ba }, + /* x^194560 mod p(x)` << 1, x^194624 mod p(x)` << 1 */ + { 0x0000000097b1b002, 0x000000003920cd16 }, + /* x^193536 mod p(x)` << 1, x^193600 mod p(x)` << 1 */ + { 0x00000000b15bf906, 0x00000001e6f579b8 }, + /* x^192512 mod p(x)` << 1, x^192576 mod p(x)` << 1 */ + { 0x00000000411c5d52, 0x000000007493cb0a }, + /* x^191488 mod p(x)` << 1, x^191552 mod p(x)` << 1 */ + { 0x00000001c36f3300, 0x00000001bdd376d8 }, + /* x^190464 mod p(x)` << 1, x^190528 mod p(x)` << 1 */ + { 0x00000001119227e0, 0x000000016badfee6 }, + /* x^189440 mod p(x)` << 1, x^189504 mod p(x)` << 1 */ + { 0x00000000114d4702, 0x0000000071de5c58 }, + /* x^188416 mod p(x)` << 1, x^188480 mod p(x)` << 1 */ + { 0x00000000458b5b98, 0x00000000453f317c }, + /* x^187392 mod p(x)` << 1, x^187456 mod p(x)` << 1 */ + { 0x000000012e31fb8e, 0x0000000121675cce }, + /* x^186368 mod p(x)` << 1, x^186432 mod p(x)` << 1 */ + { 0x000000005cf619d8, 0x00000001f409ee92 }, + /* x^185344 mod p(x)` << 1, x^185408 mod p(x)` << 1 */ + { 0x0000000063f4d8b2, 0x00000000f36b9c88 }, + /* x^184320 mod p(x)` << 1, x^184384 mod p(x)` << 1 */ + { 0x000000004138dc8a, 0x0000000036b398f4 }, + /* x^183296 mod p(x)` << 1, x^183360 mod p(x)` << 1 */ + { 0x00000001d29ee8e0, 0x00000001748f9adc }, + /* x^182272 mod p(x)` << 1, x^182336 mod p(x)` << 1 */ + { 0x000000006a08ace8, 0x00000001be94ec00 }, + /* x^181248 mod p(x)` << 1, x^181312 mod p(x)` << 1 */ + { 0x0000000127d42010, 0x00000000b74370d6 }, + /* x^180224 mod p(x)` << 1, x^180288 mod p(x)` << 1 */ + { 0x0000000019d76b62, 0x00000001174d0b98 }, + /* x^179200 mod p(x)` << 1, x^179264 mod p(x)` << 1 */ + { 0x00000001b1471f6e, 0x00000000befc06a4 }, + /* x^178176 mod p(x)` << 1, x^178240 mod p(x)` << 1 */ + { 0x00000001f64c19cc, 0x00000001ae125288 }, + /* x^177152 mod p(x)` << 1, x^177216 mod p(x)` << 1 */ + { 0x00000000003c0ea0, 0x0000000095c19b34 }, + /* x^176128 mod p(x)` << 1, x^176192 mod p(x)` << 1 */ + { 0x000000014d73abf6, 0x00000001a78496f2 }, + /* x^175104 mod p(x)` << 1, x^175168 mod p(x)` << 1 */ + { 0x00000001620eb844, 0x00000001ac5390a0 }, + /* x^174080 mod p(x)` << 1, x^174144 mod p(x)` << 1 */ + { 0x0000000147655048, 0x000000002a80ed6e }, + /* x^173056 mod p(x)` << 1, x^173120 mod p(x)` << 1 */ + { 0x0000000067b5077e, 0x00000001fa9b0128 }, + /* x^172032 mod p(x)` << 1, x^172096 mod p(x)` << 1 */ + { 0x0000000010ffe206, 0x00000001ea94929e }, + /* x^171008 mod p(x)` << 1, x^171072 mod p(x)` << 1 */ + { 0x000000000fee8f1e, 0x0000000125f4305c }, + /* x^169984 mod p(x)` << 1, x^170048 mod p(x)` << 1 */ + { 0x00000001da26fbae, 0x00000001471e2002 }, + /* x^168960 mod p(x)` << 1, x^169024 mod p(x)` << 1 */ + { 0x00000001b3a8bd88, 0x0000000132d2253a }, + /* x^167936 mod p(x)` << 1, x^168000 mod p(x)` << 1 */ + { 0x00000000e8f3898e, 0x00000000f26b3592 }, + /* x^166912 mod p(x)` << 1, x^166976 mod p(x)` << 1 */ + { 0x00000000b0d0d28c, 0x00000000bc8b67b0 }, + /* x^165888 mod p(x)` << 1, x^165952 mod p(x)` << 1 */ + { 0x0000000030f2a798, 0x000000013a826ef2 }, + /* x^164864 mod p(x)` << 1, x^164928 mod p(x)` << 1 */ + { 0x000000000fba1002, 0x0000000081482c84 }, + /* x^163840 mod p(x)` << 1, x^163904 mod p(x)` << 1 */ + { 0x00000000bdb9bd72, 0x00000000e77307c2 }, + /* x^162816 mod p(x)` << 1, x^162880 mod p(x)` << 1 */ + { 0x0000000075d3bf5a, 0x00000000d4a07ec8 }, + /* x^161792 mod p(x)` << 1, x^161856 mod p(x)` << 1 */ + { 0x00000000ef1f98a0, 0x0000000017102100 }, + /* x^160768 mod p(x)` << 1, x^160832 mod p(x)` << 1 */ + { 0x00000000689c7602, 0x00000000db406486 }, + /* x^159744 mod p(x)` << 1, x^159808 mod p(x)` << 1 */ + { 0x000000016d5fa5fe, 0x0000000192db7f88 }, + /* x^158720 mod p(x)` << 1, x^158784 mod p(x)` << 1 */ + { 0x00000001d0d2b9ca, 0x000000018bf67b1e }, + /* x^157696 mod p(x)` << 1, x^157760 mod p(x)` << 1 */ + { 0x0000000041e7b470, 0x000000007c09163e }, + /* x^156672 mod p(x)` << 1, x^156736 mod p(x)` << 1 */ + { 0x00000001cbb6495e, 0x000000000adac060 }, + /* x^155648 mod p(x)` << 1, x^155712 mod p(x)` << 1 */ + { 0x000000010052a0b0, 0x00000000bd8316ae }, + /* x^154624 mod p(x)` << 1, x^154688 mod p(x)` << 1 */ + { 0x00000001d8effb5c, 0x000000019f09ab54 }, + /* x^153600 mod p(x)` << 1, x^153664 mod p(x)` << 1 */ + { 0x00000001d969853c, 0x0000000125155542 }, + /* x^152576 mod p(x)` << 1, x^152640 mod p(x)` << 1 */ + { 0x00000000523ccce2, 0x000000018fdb5882 }, + /* x^151552 mod p(x)` << 1, x^151616 mod p(x)` << 1 */ + { 0x000000001e2436bc, 0x00000000e794b3f4 }, + /* x^150528 mod p(x)` << 1, x^150592 mod p(x)` << 1 */ + { 0x00000000ddd1c3a2, 0x000000016f9bb022 }, + /* x^149504 mod p(x)` << 1, x^149568 mod p(x)` << 1 */ + { 0x0000000019fcfe38, 0x00000000290c9978 }, + /* x^148480 mod p(x)` << 1, x^148544 mod p(x)` << 1 */ + { 0x00000001ce95db64, 0x0000000083c0f350 }, + /* x^147456 mod p(x)` << 1, x^147520 mod p(x)` << 1 */ + { 0x00000000af582806, 0x0000000173ea6628 }, + /* x^146432 mod p(x)` << 1, x^146496 mod p(x)` << 1 */ + { 0x00000001006388f6, 0x00000001c8b4e00a }, + /* x^145408 mod p(x)` << 1, x^145472 mod p(x)` << 1 */ + { 0x0000000179eca00a, 0x00000000de95d6aa }, + /* x^144384 mod p(x)` << 1, x^144448 mod p(x)` << 1 */ + { 0x0000000122410a6a, 0x000000010b7f7248 }, + /* x^143360 mod p(x)` << 1, x^143424 mod p(x)` << 1 */ + { 0x000000004288e87c, 0x00000001326e3a06 }, + /* x^142336 mod p(x)` << 1, x^142400 mod p(x)` << 1 */ + { 0x000000016c5490da, 0x00000000bb62c2e6 }, + /* x^141312 mod p(x)` << 1, x^141376 mod p(x)` << 1 */ + { 0x00000000d1c71f6e, 0x0000000156a4b2c2 }, + /* x^140288 mod p(x)` << 1, x^140352 mod p(x)` << 1 */ + { 0x00000001b4ce08a6, 0x000000011dfe763a }, + /* x^139264 mod p(x)` << 1, x^139328 mod p(x)` << 1 */ + { 0x00000001466ba60c, 0x000000007bcca8e2 }, + /* x^138240 mod p(x)` << 1, x^138304 mod p(x)` << 1 */ + { 0x00000001f6c488a4, 0x0000000186118faa }, + /* x^137216 mod p(x)` << 1, x^137280 mod p(x)` << 1 */ + { 0x000000013bfb0682, 0x0000000111a65a88 }, + /* x^136192 mod p(x)` << 1, x^136256 mod p(x)` << 1 */ + { 0x00000000690e9e54, 0x000000003565e1c4 }, + /* x^135168 mod p(x)` << 1, x^135232 mod p(x)` << 1 */ + { 0x00000000281346b6, 0x000000012ed02a82 }, + /* x^134144 mod p(x)` << 1, x^134208 mod p(x)` << 1 */ + { 0x0000000156464024, 0x00000000c486ecfc }, + /* x^133120 mod p(x)` << 1, x^133184 mod p(x)` << 1 */ + { 0x000000016063a8dc, 0x0000000001b951b2 }, + /* x^132096 mod p(x)` << 1, x^132160 mod p(x)` << 1 */ + { 0x0000000116a66362, 0x0000000048143916 }, + /* x^131072 mod p(x)` << 1, x^131136 mod p(x)` << 1 */ + { 0x000000017e8aa4d2, 0x00000001dc2ae124 }, + /* x^130048 mod p(x)` << 1, x^130112 mod p(x)` << 1 */ + { 0x00000001728eb10c, 0x00000001416c58d6 }, + /* x^129024 mod p(x)` << 1, x^129088 mod p(x)` << 1 */ + { 0x00000001b08fd7fa, 0x00000000a479744a }, + /* x^128000 mod p(x)` << 1, x^128064 mod p(x)` << 1 */ + { 0x00000001092a16e8, 0x0000000096ca3a26 }, + /* x^126976 mod p(x)` << 1, x^127040 mod p(x)` << 1 */ + { 0x00000000a505637c, 0x00000000ff223d4e }, + /* x^125952 mod p(x)` << 1, x^126016 mod p(x)` << 1 */ + { 0x00000000d94869b2, 0x000000010e84da42 }, + /* x^124928 mod p(x)` << 1, x^124992 mod p(x)` << 1 */ + { 0x00000001c8b203ae, 0x00000001b61ba3d0 }, + /* x^123904 mod p(x)` << 1, x^123968 mod p(x)` << 1 */ + { 0x000000005704aea0, 0x00000000680f2de8 }, + /* x^122880 mod p(x)` << 1, x^122944 mod p(x)` << 1 */ + { 0x000000012e295fa2, 0x000000008772a9a8 }, + /* x^121856 mod p(x)` << 1, x^121920 mod p(x)` << 1 */ + { 0x000000011d0908bc, 0x0000000155f295bc }, + /* x^120832 mod p(x)` << 1, x^120896 mod p(x)` << 1 */ + { 0x0000000193ed97ea, 0x00000000595f9282 }, + /* x^119808 mod p(x)` << 1, x^119872 mod p(x)` << 1 */ + { 0x000000013a0f1c52, 0x0000000164b1c25a }, + /* x^118784 mod p(x)` << 1, x^118848 mod p(x)` << 1 */ + { 0x000000010c2c40c0, 0x00000000fbd67c50 }, + /* x^117760 mod p(x)` << 1, x^117824 mod p(x)` << 1 */ + { 0x00000000ff6fac3e, 0x0000000096076268 }, + /* x^116736 mod p(x)` << 1, x^116800 mod p(x)` << 1 */ + { 0x000000017b3609c0, 0x00000001d288e4cc }, + /* x^115712 mod p(x)` << 1, x^115776 mod p(x)` << 1 */ + { 0x0000000088c8c922, 0x00000001eaac1bdc }, + /* x^114688 mod p(x)` << 1, x^114752 mod p(x)` << 1 */ + { 0x00000001751baae6, 0x00000001f1ea39e2 }, + /* x^113664 mod p(x)` << 1, x^113728 mod p(x)` << 1 */ + { 0x0000000107952972, 0x00000001eb6506fc }, + /* x^112640 mod p(x)` << 1, x^112704 mod p(x)` << 1 */ + { 0x0000000162b00abe, 0x000000010f806ffe }, + /* x^111616 mod p(x)` << 1, x^111680 mod p(x)` << 1 */ + { 0x000000000d7b404c, 0x000000010408481e }, + /* x^110592 mod p(x)` << 1, x^110656 mod p(x)` << 1 */ + { 0x00000000763b13d4, 0x0000000188260534 }, + /* x^109568 mod p(x)` << 1, x^109632 mod p(x)` << 1 */ + { 0x00000000f6dc22d8, 0x0000000058fc73e0 }, + /* x^108544 mod p(x)` << 1, x^108608 mod p(x)` << 1 */ + { 0x000000007daae060, 0x00000000391c59b8 }, + /* x^107520 mod p(x)` << 1, x^107584 mod p(x)` << 1 */ + { 0x000000013359ab7c, 0x000000018b638400 }, + /* x^106496 mod p(x)` << 1, x^106560 mod p(x)` << 1 */ + { 0x000000008add438a, 0x000000011738f5c4 }, + /* x^105472 mod p(x)` << 1, x^105536 mod p(x)` << 1 */ + { 0x00000001edbefdea, 0x000000008cf7c6da }, + /* x^104448 mod p(x)` << 1, x^104512 mod p(x)` << 1 */ + { 0x000000004104e0f8, 0x00000001ef97fb16 }, + /* x^103424 mod p(x)` << 1, x^103488 mod p(x)` << 1 */ + { 0x00000000b48a8222, 0x0000000102130e20 }, + /* x^102400 mod p(x)` << 1, x^102464 mod p(x)` << 1 */ + { 0x00000001bcb46844, 0x00000000db968898 }, + /* x^101376 mod p(x)` << 1, x^101440 mod p(x)` << 1 */ + { 0x000000013293ce0a, 0x00000000b5047b5e }, + /* x^100352 mod p(x)` << 1, x^100416 mod p(x)` << 1 */ + { 0x00000001710d0844, 0x000000010b90fdb2 }, + /* x^99328 mod p(x)` << 1, x^99392 mod p(x)` << 1 */ + { 0x0000000117907f6e, 0x000000004834a32e }, + /* x^98304 mod p(x)` << 1, x^98368 mod p(x)` << 1 */ + { 0x0000000087ddf93e, 0x0000000059c8f2b0 }, + /* x^97280 mod p(x)` << 1, x^97344 mod p(x)` << 1 */ + { 0x000000005970e9b0, 0x0000000122cec508 }, + /* x^96256 mod p(x)` << 1, x^96320 mod p(x)` << 1 */ + { 0x0000000185b2b7d0, 0x000000000a330cda }, + /* x^95232 mod p(x)` << 1, x^95296 mod p(x)` << 1 */ + { 0x00000001dcee0efc, 0x000000014a47148c }, + /* x^94208 mod p(x)` << 1, x^94272 mod p(x)` << 1 */ + { 0x0000000030da2722, 0x0000000042c61cb8 }, + /* x^93184 mod p(x)` << 1, x^93248 mod p(x)` << 1 */ + { 0x000000012f925a18, 0x0000000012fe6960 }, + /* x^92160 mod p(x)` << 1, x^92224 mod p(x)` << 1 */ + { 0x00000000dd2e357c, 0x00000000dbda2c20 }, + /* x^91136 mod p(x)` << 1, x^91200 mod p(x)` << 1 */ + { 0x00000000071c80de, 0x000000011122410c }, + /* x^90112 mod p(x)` << 1, x^90176 mod p(x)` << 1 */ + { 0x000000011513140a, 0x00000000977b2070 }, + /* x^89088 mod p(x)` << 1, x^89152 mod p(x)` << 1 */ + { 0x00000001df876e8e, 0x000000014050438e }, + /* x^88064 mod p(x)` << 1, x^88128 mod p(x)` << 1 */ + { 0x000000015f81d6ce, 0x0000000147c840e8 }, + /* x^87040 mod p(x)` << 1, x^87104 mod p(x)` << 1 */ + { 0x000000019dd94dbe, 0x00000001cc7c88ce }, + /* x^86016 mod p(x)` << 1, x^86080 mod p(x)` << 1 */ + { 0x00000001373d206e, 0x00000001476b35a4 }, + /* x^84992 mod p(x)` << 1, x^85056 mod p(x)` << 1 */ + { 0x00000000668ccade, 0x000000013d52d508 }, + /* x^83968 mod p(x)` << 1, x^84032 mod p(x)` << 1 */ + { 0x00000001b192d268, 0x000000008e4be32e }, + /* x^82944 mod p(x)` << 1, x^83008 mod p(x)` << 1 */ + { 0x00000000e30f3a78, 0x00000000024120fe }, + /* x^81920 mod p(x)` << 1, x^81984 mod p(x)` << 1 */ + { 0x000000010ef1f7bc, 0x00000000ddecddb4 }, + /* x^80896 mod p(x)` << 1, x^80960 mod p(x)` << 1 */ + { 0x00000001f5ac7380, 0x00000000d4d403bc }, + /* x^79872 mod p(x)` << 1, x^79936 mod p(x)` << 1 */ + { 0x000000011822ea70, 0x00000001734b89aa }, + /* x^78848 mod p(x)` << 1, x^78912 mod p(x)` << 1 */ + { 0x00000000c3a33848, 0x000000010e7a58d6 }, + /* x^77824 mod p(x)` << 1, x^77888 mod p(x)` << 1 */ + { 0x00000001bd151c24, 0x00000001f9f04e9c }, + /* x^76800 mod p(x)` << 1, x^76864 mod p(x)` << 1 */ + { 0x0000000056002d76, 0x00000000b692225e }, + /* x^75776 mod p(x)` << 1, x^75840 mod p(x)` << 1 */ + { 0x000000014657c4f4, 0x000000019b8d3f3e }, + /* x^74752 mod p(x)` << 1, x^74816 mod p(x)` << 1 */ + { 0x0000000113742d7c, 0x00000001a874f11e }, + /* x^73728 mod p(x)` << 1, x^73792 mod p(x)` << 1 */ + { 0x000000019c5920ba, 0x000000010d5a4254 }, + /* x^72704 mod p(x)` << 1, x^72768 mod p(x)` << 1 */ + { 0x000000005216d2d6, 0x00000000bbb2f5d6 }, + /* x^71680 mod p(x)` << 1, x^71744 mod p(x)` << 1 */ + { 0x0000000136f5ad8a, 0x0000000179cc0e36 }, + /* x^70656 mod p(x)` << 1, x^70720 mod p(x)` << 1 */ + { 0x000000018b07beb6, 0x00000001dca1da4a }, + /* x^69632 mod p(x)` << 1, x^69696 mod p(x)` << 1 */ + { 0x00000000db1e93b0, 0x00000000feb1a192 }, + /* x^68608 mod p(x)` << 1, x^68672 mod p(x)` << 1 */ + { 0x000000000b96fa3a, 0x00000000d1eeedd6 }, + /* x^67584 mod p(x)` << 1, x^67648 mod p(x)` << 1 */ + { 0x00000001d9968af0, 0x000000008fad9bb4 }, + /* x^66560 mod p(x)` << 1, x^66624 mod p(x)` << 1 */ + { 0x000000000e4a77a2, 0x00000001884938e4 }, + /* x^65536 mod p(x)` << 1, x^65600 mod p(x)` << 1 */ + { 0x00000000508c2ac8, 0x00000001bc2e9bc0 }, + /* x^64512 mod p(x)` << 1, x^64576 mod p(x)` << 1 */ + { 0x0000000021572a80, 0x00000001f9658a68 }, + /* x^63488 mod p(x)` << 1, x^63552 mod p(x)` << 1 */ + { 0x00000001b859daf2, 0x000000001b9224fc }, + /* x^62464 mod p(x)` << 1, x^62528 mod p(x)` << 1 */ + { 0x000000016f788474, 0x0000000055b2fb84 }, + /* x^61440 mod p(x)` << 1, x^61504 mod p(x)` << 1 */ + { 0x00000001b438810e, 0x000000018b090348 }, + /* x^60416 mod p(x)` << 1, x^60480 mod p(x)` << 1 */ + { 0x0000000095ddc6f2, 0x000000011ccbd5ea }, + /* x^59392 mod p(x)` << 1, x^59456 mod p(x)` << 1 */ + { 0x00000001d977c20c, 0x0000000007ae47f8 }, + /* x^58368 mod p(x)` << 1, x^58432 mod p(x)` << 1 */ + { 0x00000000ebedb99a, 0x0000000172acbec0 }, + /* x^57344 mod p(x)` << 1, x^57408 mod p(x)` << 1 */ + { 0x00000001df9e9e92, 0x00000001c6e3ff20 }, + /* x^56320 mod p(x)` << 1, x^56384 mod p(x)` << 1 */ + { 0x00000001a4a3f952, 0x00000000e1b38744 }, + /* x^55296 mod p(x)` << 1, x^55360 mod p(x)` << 1 */ + { 0x00000000e2f51220, 0x00000000791585b2 }, + /* x^54272 mod p(x)` << 1, x^54336 mod p(x)` << 1 */ + { 0x000000004aa01f3e, 0x00000000ac53b894 }, + /* x^53248 mod p(x)` << 1, x^53312 mod p(x)` << 1 */ + { 0x00000000b3e90a58, 0x00000001ed5f2cf4 }, + /* x^52224 mod p(x)` << 1, x^52288 mod p(x)` << 1 */ + { 0x000000000c9ca2aa, 0x00000001df48b2e0 }, + /* x^51200 mod p(x)` << 1, x^51264 mod p(x)` << 1 */ + { 0x0000000151682316, 0x00000000049c1c62 }, + /* x^50176 mod p(x)` << 1, x^50240 mod p(x)` << 1 */ + { 0x0000000036fce78c, 0x000000017c460c12 }, + /* x^49152 mod p(x)` << 1, x^49216 mod p(x)` << 1 */ + { 0x000000009037dc10, 0x000000015be4da7e }, + /* x^48128 mod p(x)` << 1, x^48192 mod p(x)` << 1 */ + { 0x00000000d3298582, 0x000000010f38f668 }, + /* x^47104 mod p(x)` << 1, x^47168 mod p(x)` << 1 */ + { 0x00000001b42e8ad6, 0x0000000039f40a00 }, + /* x^46080 mod p(x)` << 1, x^46144 mod p(x)` << 1 */ + { 0x00000000142a9838, 0x00000000bd4c10c4 }, + /* x^45056 mod p(x)` << 1, x^45120 mod p(x)` << 1 */ + { 0x0000000109c7f190, 0x0000000042db1d98 }, + /* x^44032 mod p(x)` << 1, x^44096 mod p(x)` << 1 */ + { 0x0000000056ff9310, 0x00000001c905bae6 }, + /* x^43008 mod p(x)` << 1, x^43072 mod p(x)` << 1 */ + { 0x00000001594513aa, 0x00000000069d40ea }, + /* x^41984 mod p(x)` << 1, x^42048 mod p(x)` << 1 */ + { 0x00000001e3b5b1e8, 0x000000008e4fbad0 }, + /* x^40960 mod p(x)` << 1, x^41024 mod p(x)` << 1 */ + { 0x000000011dd5fc08, 0x0000000047bedd46 }, + /* x^39936 mod p(x)` << 1, x^40000 mod p(x)` << 1 */ + { 0x00000001675f0cc2, 0x0000000026396bf8 }, + /* x^38912 mod p(x)` << 1, x^38976 mod p(x)` << 1 */ + { 0x00000000d1c8dd44, 0x00000000379beb92 }, + /* x^37888 mod p(x)` << 1, x^37952 mod p(x)` << 1 */ + { 0x0000000115ebd3d8, 0x000000000abae54a }, + /* x^36864 mod p(x)` << 1, x^36928 mod p(x)` << 1 */ + { 0x00000001ecbd0dac, 0x0000000007e6a128 }, + /* x^35840 mod p(x)` << 1, x^35904 mod p(x)` << 1 */ + { 0x00000000cdf67af2, 0x000000000ade29d2 }, + /* x^34816 mod p(x)` << 1, x^34880 mod p(x)` << 1 */ + { 0x000000004c01ff4c, 0x00000000f974c45c }, + /* x^33792 mod p(x)` << 1, x^33856 mod p(x)` << 1 */ + { 0x00000000f2d8657e, 0x00000000e77ac60a }, + /* x^32768 mod p(x)` << 1, x^32832 mod p(x)` << 1 */ + { 0x000000006bae74c4, 0x0000000145895816 }, + /* x^31744 mod p(x)` << 1, x^31808 mod p(x)` << 1 */ + { 0x0000000152af8aa0, 0x0000000038e362be }, + /* x^30720 mod p(x)` << 1, x^30784 mod p(x)` << 1 */ + { 0x0000000004663802, 0x000000007f991a64 }, + /* x^29696 mod p(x)` << 1, x^29760 mod p(x)` << 1 */ + { 0x00000001ab2f5afc, 0x00000000fa366d3a }, + /* x^28672 mod p(x)` << 1, x^28736 mod p(x)` << 1 */ + { 0x0000000074a4ebd4, 0x00000001a2bb34f0 }, + /* x^27648 mod p(x)` << 1, x^27712 mod p(x)` << 1 */ + { 0x00000001d7ab3a4c, 0x0000000028a9981e }, + /* x^26624 mod p(x)` << 1, x^26688 mod p(x)` << 1 */ + { 0x00000001a8da60c6, 0x00000001dbc672be }, + /* x^25600 mod p(x)` << 1, x^25664 mod p(x)` << 1 */ + { 0x000000013cf63820, 0x00000000b04d77f6 }, + /* x^24576 mod p(x)` << 1, x^24640 mod p(x)` << 1 */ + { 0x00000000bec12e1e, 0x0000000124400d96 }, + /* x^23552 mod p(x)` << 1, x^23616 mod p(x)` << 1 */ + { 0x00000001c6368010, 0x000000014ca4b414 }, + /* x^22528 mod p(x)` << 1, x^22592 mod p(x)` << 1 */ + { 0x00000001e6e78758, 0x000000012fe2c938 }, + /* x^21504 mod p(x)` << 1, x^21568 mod p(x)` << 1 */ + { 0x000000008d7f2b3c, 0x00000001faed01e6 }, + /* x^20480 mod p(x)` << 1, x^20544 mod p(x)` << 1 */ + { 0x000000016b4a156e, 0x000000007e80ecfe }, + /* x^19456 mod p(x)` << 1, x^19520 mod p(x)` << 1 */ + { 0x00000001c63cfeb6, 0x0000000098daee94 }, + /* x^18432 mod p(x)` << 1, x^18496 mod p(x)` << 1 */ + { 0x000000015f902670, 0x000000010a04edea }, + /* x^17408 mod p(x)` << 1, x^17472 mod p(x)` << 1 */ + { 0x00000001cd5de11e, 0x00000001c00b4524 }, + /* x^16384 mod p(x)` << 1, x^16448 mod p(x)` << 1 */ + { 0x000000001acaec54, 0x0000000170296550 }, + /* x^15360 mod p(x)` << 1, x^15424 mod p(x)` << 1 */ + { 0x000000002bd0ca78, 0x0000000181afaa48 }, + /* x^14336 mod p(x)` << 1, x^14400 mod p(x)` << 1 */ + { 0x0000000032d63d5c, 0x0000000185a31ffa }, + /* x^13312 mod p(x)` << 1, x^13376 mod p(x)` << 1 */ + { 0x000000001c6d4e4c, 0x000000002469f608 }, + /* x^12288 mod p(x)` << 1, x^12352 mod p(x)` << 1 */ + { 0x0000000106a60b92, 0x000000006980102a }, + /* x^11264 mod p(x)` << 1, x^11328 mod p(x)` << 1 */ + { 0x00000000d3855e12, 0x0000000111ea9ca8 }, + /* x^10240 mod p(x)` << 1, x^10304 mod p(x)` << 1 */ + { 0x00000000e3125636, 0x00000001bd1d29ce }, + /* x^9216 mod p(x)` << 1, x^9280 mod p(x)` << 1 */ + { 0x000000009e8f7ea4, 0x00000001b34b9580 }, + /* x^8192 mod p(x)` << 1, x^8256 mod p(x)` << 1 */ + { 0x00000001c82e562c, 0x000000003076054e }, + /* x^7168 mod p(x)` << 1, x^7232 mod p(x)` << 1 */ + { 0x00000000ca9f09ce, 0x000000012a608ea4 }, + /* x^6144 mod p(x)` << 1, x^6208 mod p(x)` << 1 */ + { 0x00000000c63764e6, 0x00000000784d05fe }, + /* x^5120 mod p(x)` << 1, x^5184 mod p(x)` << 1 */ + { 0x0000000168d2e49e, 0x000000016ef0d82a }, + /* x^4096 mod p(x)` << 1, x^4160 mod p(x)` << 1 */ + { 0x00000000e986c148, 0x0000000075bda454 }, + /* x^3072 mod p(x)` << 1, x^3136 mod p(x)` << 1 */ + { 0x00000000cfb65894, 0x000000003dc0a1c4 }, + /* x^2048 mod p(x)` << 1, x^2112 mod p(x)` << 1 */ + { 0x0000000111cadee4, 0x00000000e9a5d8be }, + /* x^1024 mod p(x)` << 1, x^1088 mod p(x)` << 1 */ + { 0x0000000171fb63ce, 0x00000001609bc4b4 } +#endif /* __LITTLE_ENDIAN__ */ + }; + +/* Reduce final 1024-2048 bits to 64 bits, shifting 32 bits to include the trailing 32 bits of zeros */ + +static const __vector unsigned long long vcrc_short_const[16] + __attribute__((aligned (16))) = { +#ifdef __LITTLE_ENDIAN__ + /* x^1952 mod p(x) , x^1984 mod p(x) , x^2016 mod p(x) , x^2048 mod p(x) */ + { 0x5cf015c388e56f72, 0x7fec2963e5bf8048 }, + /* x^1824 mod p(x) , x^1856 mod p(x) , x^1888 mod p(x) , x^1920 mod p(x) */ + { 0x963a18920246e2e6, 0x38e888d4844752a9 }, + /* x^1696 mod p(x) , x^1728 mod p(x) , x^1760 mod p(x) , x^1792 mod p(x) */ + { 0x419a441956993a31, 0x42316c00730206ad }, + /* x^1568 mod p(x) , x^1600 mod p(x) , x^1632 mod p(x) , x^1664 mod p(x) */ + { 0x924752ba2b830011, 0x543d5c543e65ddf9 }, + /* x^1440 mod p(x) , x^1472 mod p(x) , x^1504 mod p(x) , x^1536 mod p(x) */ + { 0x55bd7f9518e4a304, 0x78e87aaf56767c92 }, + /* x^1312 mod p(x) , x^1344 mod p(x) , x^1376 mod p(x) , x^1408 mod p(x) */ + { 0x6d76739fe0553f1e, 0x8f68fcec1903da7f }, + /* x^1184 mod p(x) , x^1216 mod p(x) , x^1248 mod p(x) , x^1280 mod p(x) */ + { 0xc133722b1fe0b5c3, 0x3f4840246791d588 }, + /* x^1056 mod p(x) , x^1088 mod p(x) , x^1120 mod p(x) , x^1152 mod p(x) */ + { 0x64b67ee0e55ef1f3, 0x34c96751b04de25a }, + /* x^928 mod p(x) , x^960 mod p(x) , x^992 mod p(x) , x^1024 mod p(x) */ + { 0x069db049b8fdb1e7, 0x156c8e180b4a395b }, + /* x^800 mod p(x) , x^832 mod p(x) , x^864 mod p(x) , x^896 mod p(x) */ + { 0xa11bfaf3c9e90b9e, 0xe0b99ccbe661f7be }, + /* x^672 mod p(x) , x^704 mod p(x) , x^736 mod p(x) , x^768 mod p(x) */ + { 0x817cdc5119b29a35, 0x041d37768cd75659 }, + /* x^544 mod p(x) , x^576 mod p(x) , x^608 mod p(x) , x^640 mod p(x) */ + { 0x1ce9d94b36c41f1c, 0x3a0777818cfaa965 }, + /* x^416 mod p(x) , x^448 mod p(x) , x^480 mod p(x) , x^512 mod p(x) */ + { 0x4f256efcb82be955, 0x0e148e8252377a55 }, + /* x^288 mod p(x) , x^320 mod p(x) , x^352 mod p(x) , x^384 mod p(x) */ + { 0xec1631edb2dea967, 0x9c25531d19e65dde }, + /* x^160 mod p(x) , x^192 mod p(x) , x^224 mod p(x) , x^256 mod p(x) */ + { 0x5d27e147510ac59a, 0x790606ff9957c0a6 }, + /* x^32 mod p(x) , x^64 mod p(x) , x^96 mod p(x) , x^128 mod p(x) */ + { 0xa66805eb18b8ea18, 0x82f63b786ea2d55c } +#else /* __LITTLE_ENDIAN__ */ + /* x^1952 mod p(x) , x^1984 mod p(x) , x^2016 mod p(x) , x^2048 mod p(x) */ + { 0x7fec2963e5bf8048, 0x5cf015c388e56f72 }, + /* x^1824 mod p(x) , x^1856 mod p(x) , x^1888 mod p(x) , x^1920 mod p(x) */ + { 0x38e888d4844752a9, 0x963a18920246e2e6 }, + /* x^1696 mod p(x) , x^1728 mod p(x) , x^1760 mod p(x) , x^1792 mod p(x) */ + { 0x42316c00730206ad, 0x419a441956993a31 }, + /* x^1568 mod p(x) , x^1600 mod p(x) , x^1632 mod p(x) , x^1664 mod p(x) */ + { 0x543d5c543e65ddf9, 0x924752ba2b830011 }, + /* x^1440 mod p(x) , x^1472 mod p(x) , x^1504 mod p(x) , x^1536 mod p(x) */ + { 0x78e87aaf56767c92, 0x55bd7f9518e4a304 }, + /* x^1312 mod p(x) , x^1344 mod p(x) , x^1376 mod p(x) , x^1408 mod p(x) */ + { 0x8f68fcec1903da7f, 0x6d76739fe0553f1e }, + /* x^1184 mod p(x) , x^1216 mod p(x) , x^1248 mod p(x) , x^1280 mod p(x) */ + { 0x3f4840246791d588, 0xc133722b1fe0b5c3 }, + /* x^1056 mod p(x) , x^1088 mod p(x) , x^1120 mod p(x) , x^1152 mod p(x) */ + { 0x34c96751b04de25a, 0x64b67ee0e55ef1f3 }, + /* x^928 mod p(x) , x^960 mod p(x) , x^992 mod p(x) , x^1024 mod p(x) */ + { 0x156c8e180b4a395b, 0x069db049b8fdb1e7 }, + /* x^800 mod p(x) , x^832 mod p(x) , x^864 mod p(x) , x^896 mod p(x) */ + { 0xe0b99ccbe661f7be, 0xa11bfaf3c9e90b9e }, + /* x^672 mod p(x) , x^704 mod p(x) , x^736 mod p(x) , x^768 mod p(x) */ + { 0x041d37768cd75659, 0x817cdc5119b29a35 }, + /* x^544 mod p(x) , x^576 mod p(x) , x^608 mod p(x) , x^640 mod p(x) */ + { 0x3a0777818cfaa965, 0x1ce9d94b36c41f1c }, + /* x^416 mod p(x) , x^448 mod p(x) , x^480 mod p(x) , x^512 mod p(x) */ + { 0x0e148e8252377a55, 0x4f256efcb82be955 }, + /* x^288 mod p(x) , x^320 mod p(x) , x^352 mod p(x) , x^384 mod p(x) */ + { 0x9c25531d19e65dde, 0xec1631edb2dea967 }, + /* x^160 mod p(x) , x^192 mod p(x) , x^224 mod p(x) , x^256 mod p(x) */ + { 0x790606ff9957c0a6, 0x5d27e147510ac59a }, + /* x^32 mod p(x) , x^64 mod p(x) , x^96 mod p(x) , x^128 mod p(x) */ + { 0x82f63b786ea2d55c, 0xa66805eb18b8ea18 } +#endif /* __LITTLE_ENDIAN__ */ + }; + +/* Barrett constants */ +/* 33 bit reflected Barrett constant m - (4^32)/n */ + +static const __vector unsigned long long v_Barrett_const[2] + __attribute__((aligned (16))) = { + /* x^64 div p(x) */ +#ifdef __LITTLE_ENDIAN__ + { 0x00000000dea713f1, 0x0000000000000000 }, + { 0x0000000105ec76f1, 0x0000000000000000 } +#else /* __LITTLE_ENDIAN__ */ + { 0x0000000000000000, 0x00000000dea713f1 }, + { 0x0000000000000000, 0x0000000105ec76f1 } +#endif /* __LITTLE_ENDIAN__ */ + }; +#endif /* POWER8_INTRINSICS */ + +#endif /* __ASSEMBLER__ */ diff --git a/mysys/crc32ieee.cc b/mysys/crc32ieee.cc new file mode 100644 index 00000000000..bbafa1230f8 --- /dev/null +++ b/mysys/crc32ieee.cc @@ -0,0 +1,61 @@ +/* Copyright (c) 2020, 2021, MariaDB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + + +#include <my_global.h> +#include <my_sys.h> +#include <zlib.h> + +/* TODO: remove this once zlib adds inherent support for hardware accelerated +crc32 for all architectures. */ +static unsigned int my_crc32_zlib(unsigned int crc, const void *data, + size_t len) +{ + return (unsigned int) crc32(crc, (const Bytef *)data, (unsigned int) len); +} + +#ifdef HAVE_PCLMUL +extern "C" int crc32_pclmul_enabled(); +extern "C" unsigned int crc32_pclmul(unsigned int, const void *, size_t); +#elif defined(__GNUC__) && defined(HAVE_ARMV8_CRC) +extern "C" int crc32_aarch64_available(); +extern "C" unsigned int crc32_aarch64(unsigned int, const void *, size_t); +#endif + + +typedef unsigned int (*my_crc32_t)(unsigned int, const void *, size_t); + +static my_crc32_t init_crc32() +{ +#ifdef HAVE_PCLMUL + if (crc32_pclmul_enabled()) + return crc32_pclmul; +#elif defined(__GNUC__) && defined(HAVE_ARMV8_CRC) + if (crc32_aarch64_available()) + return crc32_aarch64; +#endif + return my_crc32_zlib; +} + +static const my_crc32_t my_checksum_func= init_crc32(); + +#ifdef __powerpc64__ +# error "my_checksum() is defined in mysys/crc32/crc32_ppc64.c" +#endif +extern "C" +unsigned int my_checksum(unsigned int crc, const void *data, size_t len) +{ + return my_checksum_func(crc, data, len); +} diff --git a/mysys/errors.c b/mysys/errors.c index 45a77d8c129..d88540fe277 100644 --- a/mysys/errors.c +++ b/mysys/errors.c @@ -55,7 +55,11 @@ const char *globerrs[GLOBERRS]= "Can't change permissions of the file '%s' (Errcode: %M)", "Can't seek in file '%s' (Errcode: %M)", "Can't change mode for file '%s' to 0x%lx (Errcode: %M)", - "Warning: Can't copy ownership for file '%s' (Errcode: %M)" + "Warning: Can't copy ownership for file '%s' (Errcode: %M)", + "Failed to release memory pointer %p, %zu bytes (Errcode: %M)", + "Lock Pages in memory access rights required", + "Memcntl %s cmd %s error", + "Warning: Charset id '%d' csname '%s' trying to replace existing csname '%s'", }; void init_glob_errs(void) @@ -101,6 +105,10 @@ void init_glob_errs() EE(EE_CANT_SEEK) = "Can't seek in file '%s' (Errcode: %M)"; EE(EE_CANT_CHMOD) = "Can't change mode for file '%s' to 0x%lx (Errcode: %M)"; EE(EE_CANT_COPY_OWNERSHIP)= "Warning: Can't copy ownership for file '%s' (Errcode: %M)"; + EE(EE_BADMEMORYRELEASE)= "Failed to release memory pointer %p, %zu bytes (Errcode: %M)"; + EE(EE_PERM_LOCK_MEMORY)= "Lock Pages in memory access rights required"; + EE(EE_MEMCNTL) = "Memcntl %s cmd %s error"; + EE(EE_DUPLICATE_CHARSET)= "Warning: Charset id %d trying to replace csname %s with %s"; } #endif diff --git a/mysys/file_logger.c b/mysys/file_logger.c index 476ed44089e..a753c049f68 100644 --- a/mysys/file_logger.c +++ b/mysys/file_logger.c @@ -84,7 +84,8 @@ LOGGER_HANDLE *logger_open(const char *path, return 0; } - if (!(l_perm= (LOGGER_HANDLE *) my_malloc(sizeof(LOGGER_HANDLE), MYF(0)))) + if (!(l_perm= (LOGGER_HANDLE *) my_malloc(PSI_INSTRUMENT_ME, + sizeof(LOGGER_HANDLE), MYF(0)))) { my_close(new_log.file, MYF(0)); new_log.file= -1; diff --git a/mysys/get_password.c b/mysys/get_password.c index e20800f4213..2a5ddc9a4d7 100644 --- a/mysys/get_password.c +++ b/mysys/get_password.c @@ -90,7 +90,7 @@ char *get_tty_password(const char *opt_message) pos--; /* Allow dummy space at end */ *pos=0; _cputs("\n"); - DBUG_RETURN(my_strdup(to,MYF(MY_FAE))); + DBUG_RETURN(my_strdup(PSI_INSTRUMENT_ME, to,MYF(MY_FAE))); } #else @@ -201,7 +201,7 @@ char *get_tty_password(const char *opt_message) fputc('\n',stderr); #endif /* HAVE_GETPASS */ - DBUG_RETURN(my_strdup(buff,MYF(MY_FAE))); + DBUG_RETURN(my_strdup(PSI_INSTRUMENT_ME, buff, MYF(MY_FAE))); } #endif /*__WIN__*/ diff --git a/mysys/hash.c b/mysys/hash.c index d29297a5233..abc11b42500 100644 --- a/mysys/hash.c +++ b/mysys/hash.c @@ -1,5 +1,5 @@ /* Copyright (c) 2000, 2010, Oracle and/or its affiliates. - Copyright (c) 2011, 2013, Monty Program Ab. + Copyright (c) 2011, 2020, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -45,7 +45,7 @@ my_hash_value_type my_hash_sort(CHARSET_INFO *cs, const uchar *key, size_t length) { ulong nr1= 1, nr2= 4; - cs->coll->hash_sort(cs, (uchar*) key, length, &nr1, &nr2); + my_ci_hash_sort(cs, (uchar*) key, length, &nr1, &nr2); return (my_hash_value_type) nr1; } @@ -76,14 +76,14 @@ my_hash_value_type my_hash_sort(CHARSET_INFO *cs, const uchar *key, @retval 1 failure */ my_bool -my_hash_init2(HASH *hash, uint growth_size, CHARSET_INFO *charset, - ulong size, size_t key_offset, size_t key_length, - my_hash_get_key get_key, +my_hash_init2(PSI_memory_key psi_key, HASH *hash, uint growth_size, + CHARSET_INFO *charset, ulong size, size_t key_offset, + size_t key_length, my_hash_get_key get_key, my_hash_function hash_function, void (*free_element)(void*), uint flags) { my_bool res; - DBUG_ENTER("my_hash_init"); + DBUG_ENTER("my_hash_init2"); DBUG_PRINT("enter",("hash:%p size: %u", hash, (uint) size)); hash->records=0; @@ -95,7 +95,7 @@ my_hash_init2(HASH *hash, uint growth_size, CHARSET_INFO *charset, hash->free=free_element; hash->flags=flags; hash->charset=charset; - res= init_dynamic_array2(&hash->array, sizeof(HASH_LINK), NULL, size, + res= init_dynamic_array2(psi_key, &hash->array, sizeof(HASH_LINK), NULL, size, growth_size, MYF((flags & HASH_THREAD_SPECIFIC ? MY_THREAD_SPECIFIC : 0))); DBUG_RETURN(res); @@ -894,7 +894,7 @@ int main(int argc __attribute__((unused)),char **argv __attribute__((unused))) DBUG_PUSH("d:t:O,/tmp/test_hash.trace"); printf("my_hash_init\n"); - if (my_hash_init2(&hash_test, 100, &my_charset_bin, 20, + if (my_hash_init2(PSI_INSTRUMENT_ME, &hash_test, 100, &my_charset_bin, 20, 0, 0, (my_hash_get_key) test_get_key, 0, 0, HASH_UNIQUE)) { fprintf(stderr, "hash init failed\n"); diff --git a/mysys/lf_alloc-pin.c b/mysys/lf_alloc-pin.c index 646e584c152..2162e1771ba 100644 --- a/mysys/lf_alloc-pin.c +++ b/mysys/lf_alloc-pin.c @@ -99,8 +99,7 @@ between THD's (LF_PINS::stack_ends_here being a primary reason for this limitation). */ -#include <my_global.h> -#include <my_sys.h> +#include "mysys_priv.h" #include <lf.h> #include "my_cpu.h" @@ -501,7 +500,8 @@ void *lf_alloc_new(LF_PINS *pins) } while (node != allocator->top && LF_BACKOFF()); if (!node) { - node= (void *)my_malloc(allocator->element_size, MYF(MY_WME)); + node= (void *)my_malloc(key_memory_lf_node, allocator->element_size, + MYF(MY_WME)); if (allocator->constructor) allocator->constructor(node); #ifdef MY_LF_EXTRA_DEBUG diff --git a/mysys/lf_dynarray.c b/mysys/lf_dynarray.c index be23690c70b..6a4ea3d3d96 100644 --- a/mysys/lf_dynarray.c +++ b/mysys/lf_dynarray.c @@ -35,9 +35,8 @@ Actually, it's wait-free, not lock-free ;-) */ -#include <my_global.h> +#include "mysys_priv.h" #include <m_string.h> -#include <my_sys.h> #include <lf.h> void lf_dynarray_init(LF_DYNARRAY *array, uint element_size) @@ -106,8 +105,8 @@ void *lf_dynarray_lvalue(LF_DYNARRAY *array, uint idx) { if (!(ptr= *ptr_ptr)) { - void *alloc= my_malloc(LF_DYNARRAY_LEVEL_LENGTH * sizeof(void *), - MYF(MY_WME|MY_ZEROFILL)); + void *alloc= my_malloc(key_memory_lf_dynarray, LF_DYNARRAY_LEVEL_LENGTH * + sizeof(void *), MYF(MY_WME|MY_ZEROFILL)); if (unlikely(!alloc)) return(NULL); if (my_atomic_casptr(ptr_ptr, &ptr, alloc)) @@ -121,9 +120,10 @@ void *lf_dynarray_lvalue(LF_DYNARRAY *array, uint idx) if (!(ptr= *ptr_ptr)) { uchar *alloc, *data; - alloc= my_malloc(LF_DYNARRAY_LEVEL_LENGTH * array->size_of_element + - MY_MAX(array->size_of_element, sizeof(void *)), - MYF(MY_WME|MY_ZEROFILL)); + alloc= my_malloc(key_memory_lf_dynarray, + LF_DYNARRAY_LEVEL_LENGTH * array->size_of_element + + MY_MAX(array->size_of_element, sizeof(void *)), + MYF(MY_WME|MY_ZEROFILL)); if (unlikely(!alloc)) return(NULL); /* reserve the space for free() address */ diff --git a/mysys/lf_hash.c b/mysys/lf_hash.cc index 7e401a63194..bc3504bc0dd 100644 --- a/mysys/lf_hash.c +++ b/mysys/lf_hash.cc @@ -1,5 +1,5 @@ /* Copyright (c) 2006, 2018, Oracle and/or its affiliates. - Copyright (c) 2009, 2018, MariaDB + Copyright (c) 2009, 2020, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -22,13 +22,13 @@ for non-unique hash, count only _distinct_ values (but how to do it in lf_hash_delete ?) */ -#include <my_global.h> +#include "mysys_priv.h" #include <m_string.h> -#include <my_sys.h> #include <mysys_err.h> #include <my_bit.h> #include <lf.h> #include "my_cpu.h" +#include "assume_aligned.h" /* An element of the list */ typedef struct { @@ -99,11 +99,11 @@ static int l_find(LF_SLIST **head, CHARSET_INFO *cs, uint32 hashnr, DBUG_ASSERT(!keylen || !callback); /* should not be set both */ retry: - cursor->prev= (intptr *)head; + cursor->prev= (intptr *) my_assume_aligned<sizeof(intptr)>(head); do { /* PTR() isn't necessary below, head is a dummy node */ - cursor->curr= (LF_SLIST *)(*cursor->prev); + cursor->curr= my_assume_aligned<sizeof(LF_SLIST *)>((LF_SLIST *)(*cursor->prev)); lf_pin(pins, 1, cursor->curr); - } while (my_atomic_loadptr((void**)cursor->prev) != cursor->curr && + } while (my_atomic_loadptr((void **)my_assume_aligned<sizeof(LF_SLIST *)>(cursor->prev)) != cursor->curr && LF_BACKOFF()); for (;;) { @@ -112,11 +112,14 @@ retry: cur_hashnr= cursor->curr->hashnr; cur_keylen= cursor->curr->keylen; + /* The key element needs to be aligned, not necessary what it points to */ + my_assume_aligned<sizeof(const uchar *)>(&cursor->curr->key); cur_key= cursor->curr->key; do { + /* attempting to my_assume_aligned onlink below broke the implementation */ link= cursor->curr->link; - cursor->next= PTR(link); + cursor->next= my_assume_aligned<sizeof(LF_SLIST *)>(PTR(link)); lf_pin(pins, 0, cursor->next); } while (link != cursor->curr->link && LF_BACKOFF()); @@ -156,6 +159,10 @@ retry: } } + +/* static l_find is the only user my_assume_aligned, keep the rest as c scoped */ +C_MODE_START + /* DESCRIPTION insert a 'node' in the list that starts from 'head' in the correct @@ -314,7 +321,7 @@ static inline my_hash_value_type calc_hash(CHARSET_INFO *cs, size_t keylen) { ulong nr1= 1, nr2= 4; - cs->coll->hash_sort(cs, (uchar*) key, keylen, &nr1, &nr2); + my_ci_hash_sort(cs, (uchar*) key, keylen, &nr1, &nr2); return nr1; } @@ -327,6 +334,7 @@ static void default_initializer(LF_HASH *hash, void *dst, const void *src) memcpy(dst, src, hash->element_size); } + /* Initializes lf_hash, the arguments are compatible with hash_init @@ -408,7 +416,7 @@ int lf_hash_insert(LF_HASH *hash, LF_PINS *pins, const void *data) node->key= hash_key(hash, (uchar *)(node+1), &node->keylen); hashnr= hash->hash_function(hash->charset, node->key, node->keylen) & INT_MAX32; bucket= hashnr % hash->size; - el= lf_dynarray_lvalue(&hash->array, bucket); + el= (LF_SLIST **)lf_dynarray_lvalue(&hash->array, bucket); if (unlikely(!el)) return -1; if (*el == NULL && unlikely(initialize_bucket(hash, el, bucket, pins))) @@ -446,7 +454,7 @@ int lf_hash_delete(LF_HASH *hash, LF_PINS *pins, const void *key, uint keylen) /* hide OOM errors - if we cannot initialize a bucket, try the previous one */ for (bucket= hashnr % hash->size; ;bucket= my_clear_highest_bit(bucket)) { - el= lf_dynarray_lvalue(&hash->array, bucket); + el= (LF_SLIST **)lf_dynarray_lvalue(&hash->array, bucket); if (el && (*el || initialize_bucket(hash, el, bucket, pins) == 0)) break; if (unlikely(bucket == 0)) @@ -480,7 +488,7 @@ void *lf_hash_search_using_hash_value(LF_HASH *hash, LF_PINS *pins, /* hide OOM errors - if we cannot initialize a bucket, try the previous one */ for (bucket= hashnr % hash->size; ;bucket= my_clear_highest_bit(bucket)) { - el= lf_dynarray_lvalue(&hash->array, bucket); + el= (LF_SLIST **)lf_dynarray_lvalue(&hash->array, bucket); if (el && (*el || initialize_bucket(hash, el, bucket, pins) == 0)) break; if (unlikely(bucket == 0)) @@ -510,7 +518,7 @@ int lf_hash_iterate(LF_HASH *hash, LF_PINS *pins, int res; LF_SLIST **el; - el= lf_dynarray_lvalue(&hash->array, bucket); + el= (LF_SLIST **)lf_dynarray_lvalue(&hash->array, bucket); if (unlikely(!el)) return 0; /* if there's no bucket==0, the hash is empty */ if (*el == NULL && unlikely(initialize_bucket(hash, el, bucket, pins))) @@ -544,9 +552,10 @@ static int initialize_bucket(LF_HASH *hash, LF_SLIST **node, uint bucket, LF_PINS *pins) { uint parent= my_clear_highest_bit(bucket); - LF_SLIST *dummy= (LF_SLIST *)my_malloc(sizeof(LF_SLIST), MYF(MY_WME)); + LF_SLIST *dummy= (LF_SLIST *)my_malloc(key_memory_lf_slist, + sizeof(LF_SLIST), MYF(MY_WME)); LF_SLIST **tmp= 0, *cur; - LF_SLIST **el= lf_dynarray_lvalue(&hash->array, parent); + LF_SLIST **el= (LF_SLIST **)lf_dynarray_lvalue(&hash->array, parent); if (unlikely(!el || !dummy)) return -1; if (*el == NULL && bucket && @@ -574,3 +583,5 @@ static int initialize_bucket(LF_HASH *hash, LF_SLIST **node, */ return 0; } + +C_MODE_END diff --git a/mysys/list.c b/mysys/list.c index 65da5d452cc..2efe53c0368 100644 --- a/mysys/list.c +++ b/mysys/list.c @@ -70,7 +70,7 @@ void list_free(LIST *root, uint free_data) LIST *list_cons(void *data, LIST *list) { - LIST *new_charset=(LIST*) my_malloc(sizeof(LIST),MYF(MY_FAE)); + LIST *new_charset=(LIST*) my_malloc(key_memory_LIST, sizeof(LIST),MYF(MY_FAE)); if (!new_charset) return 0; new_charset->data=data; diff --git a/mysys/ma_dyncol.c b/mysys/ma_dyncol.c index b438d910157..8e8ac8a98a4 100644 --- a/mysys/ma_dyncol.c +++ b/mysys/ma_dyncol.c @@ -2417,8 +2417,8 @@ dynamic_column_list(DYNAMIC_COLUMN *str, DYNAMIC_ARRAY *array_of_uint) str->length) return ER_DYNCOL_FORMAT; - if (my_init_dynamic_array(array_of_uint, sizeof(uint), header.column_count, - 0, MYF(0))) + if (my_init_dynamic_array(PSI_INSTRUMENT_ME, array_of_uint, + sizeof(uint), header.column_count, 0, MYF(0))) return ER_DYNCOL_RESOURCE; for (i= 0, read= header.header; @@ -2463,7 +2463,7 @@ mariadb_dyncol_list_num(DYNAMIC_COLUMN *str, uint *count, uint **nums) str->length) return ER_DYNCOL_FORMAT; - if (!((*nums)= my_malloc(sizeof(uint) * header.column_count, MYF(0)))) + if (!((*nums)= my_malloc(PSI_INSTRUMENT_ME, sizeof(uint) * header.column_count, MYF(0)))) return ER_DYNCOL_RESOURCE; for (i= 0, read= header.header; @@ -2510,12 +2510,17 @@ mariadb_dyncol_list_named(DYNAMIC_COLUMN *str, uint *count, LEX_STRING **names) str->length) return ER_DYNCOL_FORMAT; - if (header.format == dyncol_fmt_num) - *names= my_malloc(sizeof(LEX_STRING) * header.column_count + - DYNCOL_NUM_CHAR * header.column_count, MYF(0)); - else - *names= my_malloc(sizeof(LEX_STRING) * header.column_count + - header.nmpool_size + header.column_count, MYF(0)); + { + size_t size; + if (header.format == dyncol_fmt_num) + size= DYNCOL_NUM_CHAR * header.column_count; + else + size= header.nmpool_size + header.column_count; + + *names= my_malloc(PSI_INSTRUMENT_ME, + sizeof(LEX_STRING) * header.column_count + size, MYF(0)); + } + if (!(*names)) return ER_DYNCOL_RESOURCE; pool= ((char *)(*names)) + sizeof(LEX_STRING) * header.column_count; @@ -3327,7 +3332,8 @@ dynamic_column_update_many_fmt(DYNAMIC_COLUMN *str, if (IN_PLACE_PLAN > add_column_count) plan= in_place_plan; else if (!(alloc_plan= plan= - my_malloc(sizeof(PLAN) * (add_column_count + 1), MYF(0)))) + my_malloc(PSI_INSTRUMENT_ME, + sizeof(PLAN) * (add_column_count + 1), MYF(0)))) return ER_DYNCOL_RESOURCE; not_null= add_column_count; @@ -3923,12 +3929,10 @@ mariadb_dyncol_val_str(DYNAMIC_STRING *str, DYNAMIC_COLUMN_VALUE *val, &dummy_errors); return ER_DYNCOL_OK; } - if ((alloc= (char *)my_malloc(bufflen, MYF(0)))) + if ((alloc= (char *)my_malloc(PSI_INSTRUMENT_ME, bufflen, MYF(0)))) { - len= my_convert(alloc, bufflen, cs, - from, (uint32)len, - val->x.string.charset, - &dummy_errors); + len= my_convert(alloc, bufflen, cs, from, (uint32)len, + val->x.string.charset, &dummy_errors); from= alloc; } else @@ -4300,16 +4304,19 @@ mariadb_dyncol_unpack(DYNAMIC_COLUMN *str, str->length) return ER_DYNCOL_FORMAT; - *vals= my_malloc(sizeof(DYNAMIC_COLUMN_VALUE)* header.column_count, MYF(0)); + *vals= my_malloc(PSI_INSTRUMENT_ME, + sizeof(DYNAMIC_COLUMN_VALUE)* header.column_count, MYF(0)); if (header.format == dyncol_fmt_num) { - *names= my_malloc(sizeof(LEX_STRING) * header.column_count + + *names= my_malloc(PSI_INSTRUMENT_ME, + sizeof(LEX_STRING) * header.column_count + DYNCOL_NUM_CHAR * header.column_count, MYF(0)); nm= (char *)((*names) + header.column_count); } else { - *names= my_malloc(sizeof(LEX_STRING) * header.column_count, MYF(0)); + *names= my_malloc(PSI_INSTRUMENT_ME, + sizeof(LEX_STRING) * header.column_count, MYF(0)); nm= 0; } if (!(*vals) || !(*names)) diff --git a/mysys/mf_dirname.c b/mysys/mf_dirname.c index 700a68e4f3a..12fd15f82b9 100644 --- a/mysys/mf_dirname.c +++ b/mysys/mf_dirname.c @@ -1,4 +1,5 @@ /* Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved. + Copyright (c) 2009, 2020, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -34,7 +35,7 @@ size_t dirname_length(const char *name) { #ifdef BASKSLASH_MBTAIL uint l; - if (use_mb(fs) && (l= my_ismbchar(fs, pos, pos + 3))) + if (my_ci_use_mb(fs) && (l= my_ismbchar(fs, pos, pos + 3))) { pos+= l - 1; continue; @@ -123,7 +124,7 @@ char *convert_dirname(char *to, const char *from, const char *from_end) { #ifdef BACKSLASH_MBTAIL uint l; - if (use_mb(fs) && (l= my_ismbchar(fs, from, from + 3))) + if (my_ci_use_mb(fs) && (l= my_ismbchar(fs, from, from + 3))) { memmove(to, from, l); to+= l; diff --git a/mysys/mf_iocache.c b/mysys/mf_iocache.c index 06c6fc088c0..3eaf15803b6 100644 --- a/mysys/mf_iocache.c +++ b/mysys/mf_iocache.c @@ -1,6 +1,6 @@ /* Copyright (c) 2000, 2011, Oracle and/or its affiliates - Copyright (c) 2010, 2020, MariaDB + Copyright (c) 2010, 2021, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -50,11 +50,10 @@ TODO: #include "mysys_priv.h" #include <m_string.h> -#ifdef HAVE_AIOWAIT -#include "mysys_err.h" -static void my_aiowait(my_aio_result *result); -#endif #include <errno.h> +#include "mysql/psi/mysql_file.h" + +PSI_file_key key_file_io_cache; #define lock_append_buffer(info) \ mysql_mutex_lock(&(info)->append_buffer_lock) @@ -131,7 +130,7 @@ init_functions(IO_CACHE* info) Initialize an IO_CACHE object SYNOPSOS - init_io_cache() + init_io_cache_ext() info cache handler to initialize file File that should be associated to to the handler If == -1 then real_open_cached_file() @@ -144,20 +143,22 @@ init_functions(IO_CACHE* info) cache_myflags Bitmap of different flags MY_WME | MY_FAE | MY_NABP | MY_FNABP | MY_DONT_CHECK_FILESIZE + file_key Instrumented file key for temporary cache file RETURN 0 ok # error */ -int init_io_cache(IO_CACHE *info, File file, size_t cachesize, - enum cache_type type, my_off_t seek_offset, - my_bool use_async_io, myf cache_myflags) +int init_io_cache_ext(IO_CACHE *info, File file, size_t cachesize, + enum cache_type type, my_off_t seek_offset, + pbool use_async_io, myf cache_myflags, + PSI_file_key file_key) { size_t min_cache; my_off_t pos; my_off_t end_of_file= ~(my_off_t) 0; - DBUG_ENTER("init_io_cache"); + DBUG_ENTER("init_io_cache_ext"); DBUG_PRINT("enter",("cache:%p type: %d pos: %llu", info, (int) type, (ulonglong) seek_offset)); @@ -244,7 +245,7 @@ int init_io_cache(IO_CACHE *info, File file, size_t cachesize, if (cachesize == min_cache) flags|= (myf) MY_WME; - if ((info->buffer= (uchar*) my_malloc(buffer_block, flags)) != 0) + if ((info->buffer= (uchar*) my_malloc(key_memory_IO_CACHE, buffer_block, flags)) != 0) { if (type == SEQ_READ_APPEND) info->write_buffer= info->buffer + cachesize; @@ -260,7 +261,7 @@ int init_io_cache(IO_CACHE *info, File file, size_t cachesize, } } - DBUG_PRINT("info",("init_io_cache: cachesize = %lu", (ulong) cachesize)); + DBUG_PRINT("info",("init_io_cache_ext: cachesize = %lu", (ulong) cachesize)); info->read_length=info->buffer_length=cachesize; info->myflags=cache_myflags & ~(MY_NABP | MY_FNABP); info->request_pos= info->read_pos= info->write_pos = info->buffer; @@ -290,20 +291,16 @@ int init_io_cache(IO_CACHE *info, File file, size_t cachesize, info->error=0; info->type= type; init_functions(info); -#ifdef HAVE_AIOWAIT - if (use_async_io && ! my_disable_async_io) - { - DBUG_PRINT("info",("Using async io")); - DBUG_ASSERT(!(cache_myflags & MY_ENCRYPT)); - info->read_length/=2; - info->read_function=_my_b_async_read; - } - info->inited=info->aio_result.pending=0; -#endif DBUG_RETURN(0); -} /* init_io_cache */ - +} +int init_io_cache(IO_CACHE *info, File file, size_t cachesize, + enum cache_type type, my_off_t seek_offset, + my_bool use_async_io, myf cache_myflags) +{ + return init_io_cache_ext(info, file, cachesize, type, seek_offset, + use_async_io, cache_myflags, key_file_io_cache); +} /* Initialize the slave IO_CACHE to read the same file (and data) @@ -324,7 +321,7 @@ int init_slave_io_cache(IO_CACHE *master, IO_CACHE *slave) DBUG_ASSERT(!master->share); DBUG_ASSERT(master->alloced_buffer); - if (!(slave_buf= (uchar*)my_malloc(master->alloced_buffer, MYF(0)))) + if (!(slave_buf= (uchar*)my_malloc(PSI_INSTRUMENT_ME, master->alloced_buffer, MYF(0)))) { return 1; } @@ -409,33 +406,6 @@ void seek_io_cache(IO_CACHE *cache, my_off_t needed_offset) } } - /* Wait until current request is ready */ - -#ifdef HAVE_AIOWAIT -static void my_aiowait(my_aio_result *result) -{ - if (result->pending) - { - struct aio_result_t *tmp; - for (;;) - { - if ((int) (tmp=aiowait((struct timeval *) 0)) == -1) - { - if (errno == EINTR) - continue; - DBUG_PRINT("error",("No aio request, error: %d",errno)); - result->pending=0; /* Assume everything is ok */ - break; - } - ((my_aio_result*) tmp)->pending=0; - if ((my_aio_result*) tmp == result) - break; - } - } - return; -} -#endif - /* Use this to reset cache to re-start reading or to change the type @@ -488,9 +458,6 @@ my_bool reinit_io_cache(IO_CACHE *info, enum cache_type type, info->write_pos=pos; else info->read_pos= pos; -#ifdef HAVE_AIOWAIT - my_aiowait(&info->aio_result); /* Wait for outstanding req */ -#endif } else { @@ -545,18 +512,6 @@ my_bool reinit_io_cache(IO_CACHE *info, enum cache_type type, info->type=type; info->error=0; init_functions(info); - -#ifdef HAVE_AIOWAIT - if (use_async_io && ! my_disable_async_io && - ((ulong) info->buffer_length < - (ulong) (info->end_of_file - seek_offset))) - { - DBUG_ASSERT(!(cache_myflags & MY_ENCRYPT)); - info->read_length=info->buffer_length/2; - info->read_function=_my_b_async_read; - } - info->inited=0; -#endif DBUG_RETURN(0); } /* reinit_io_cache */ @@ -1523,193 +1478,6 @@ read_append_buffer: } -#ifdef HAVE_AIOWAIT - -/* - Read from the IO_CACHE into a buffer and feed asynchronously - from disk when needed. - - SYNOPSIS - _my_b_async_read() - info IO_CACHE pointer - Buffer Buffer to retrieve count bytes from file - Count Number of bytes to read into Buffer - - RETURN VALUE - -1 An error has occurred; my_errno is set. - 0 Success - 1 An error has occurred; IO_CACHE to error state. -*/ - -int _my_b_async_read(IO_CACHE *info, uchar *Buffer, size_t Count) -{ - size_t length, read_length, diff_length, left_length=0, use_length, org_Count; - size_t max_length; - my_off_t next_pos_in_file; - uchar *read_buffer; - - org_Count=Count; - - if (info->inited) - { /* wait for read block */ - info->inited=0; /* No more block to read */ - my_aiowait(&info->aio_result); /* Wait for outstanding req */ - if (info->aio_result.result.aio_errno) - { - if (info->myflags & MY_WME) - my_error(EE_READ, MYF(ME_BELL), my_filename(info->file), - info->aio_result.result.aio_errno); - my_errno=info->aio_result.result.aio_errno; - info->error= -1; - return(1); - } - if (! (read_length= (size_t) info->aio_result.result.aio_return) || - read_length == (size_t) -1) - { - my_errno=0; /* For testing */ - info->error= (read_length == (size_t) -1 ? -1 : - (int) (read_length+left_length)); - return(1); - } - info->pos_in_file+= (size_t) (info->read_end - info->request_pos); - - if (info->request_pos != info->buffer) - info->request_pos=info->buffer; - else - info->request_pos=info->buffer+info->read_length; - info->read_pos=info->request_pos; - next_pos_in_file=info->aio_read_pos+read_length; - - /* Check if pos_in_file is changed - (_ni_read_cache may have skipped some bytes) */ - - if (info->aio_read_pos < info->pos_in_file) - { /* Fix if skipped bytes */ - if (info->aio_read_pos + read_length < info->pos_in_file) - { - read_length=0; /* Skip block */ - next_pos_in_file=info->pos_in_file; - } - else - { - my_off_t offset= (info->pos_in_file - info->aio_read_pos); - info->pos_in_file=info->aio_read_pos; /* Whe are here */ - info->read_pos=info->request_pos+offset; - read_length-=offset; /* Bytes left from read_pos */ - } - } -#ifndef DBUG_OFF - if (info->aio_read_pos > info->pos_in_file) - { - my_errno=EINVAL; - return(info->read_length= (size_t) -1); - } -#endif - /* Copy found bytes to buffer */ - length=MY_MIN(Count,read_length); - memcpy(Buffer,info->read_pos,(size_t) length); - Buffer+=length; - Count-=length; - left_length+=length; - info->read_end=info->read_pos+read_length; - info->read_pos+=length; - } - else - next_pos_in_file=(info->pos_in_file+ (size_t) - (info->read_end - info->request_pos)); - - /* If reading large blocks, or first read or read with skip */ - if (Count) - { - if (next_pos_in_file == info->end_of_file) - { - info->error=(int) (read_length+left_length); - return 1; - } - - if (mysql_file_seek(info->file, next_pos_in_file, MY_SEEK_SET, MYF(0)) - == MY_FILEPOS_ERROR) - { - info->error= -1; - return (1); - } - - read_length=IO_SIZE*2- (size_t) (next_pos_in_file & (IO_SIZE-1)); - if (Count < read_length) - { /* Small block, read to cache */ - if ((read_length=mysql_file_read(info->file,info->request_pos, - read_length, info->myflags)) == (size_t) -1) - return info->error= -1; - use_length=MY_MIN(Count,read_length); - memcpy(Buffer,info->request_pos,(size_t) use_length); - info->read_pos=info->request_pos+Count; - info->read_end=info->request_pos+read_length; - info->pos_in_file=next_pos_in_file; /* Start of block in cache */ - next_pos_in_file+=read_length; - - if (Count != use_length) - { /* Didn't find hole block */ - if (info->myflags & (MY_WME | MY_FAE | MY_FNABP) && Count != org_Count) - my_error(EE_EOFERR, MYF(ME_BELL), my_filename(info->file), my_errno); - info->error=(int) (read_length+left_length); - return 1; - } - } - else - { /* Big block, don't cache it */ - if ((read_length= mysql_file_read(info->file, Buffer, Count,info->myflags)) - != Count) - { - info->error= read_length == (size_t) -1 ? -1 : read_length+left_length; - return 1; - } - info->read_pos=info->read_end=info->request_pos; - info->pos_in_file=(next_pos_in_file+=Count); - } - } - - /* Read next block with asyncronic io */ - diff_length=(next_pos_in_file & (IO_SIZE-1)); - max_length= info->read_length - diff_length; - if (max_length > info->end_of_file - next_pos_in_file) - max_length= (size_t) (info->end_of_file - next_pos_in_file); - - if (info->request_pos != info->buffer) - read_buffer=info->buffer; - else - read_buffer=info->buffer+info->read_length; - info->aio_read_pos=next_pos_in_file; - if (max_length) - { - info->aio_result.result.aio_errno=AIO_INPROGRESS; /* Marker for test */ - DBUG_PRINT("aioread",("filepos: %ld length: %lu", - (ulong) next_pos_in_file, (ulong) max_length)); - if (aioread(info->file,read_buffer, max_length, - (my_off_t) next_pos_in_file,MY_SEEK_SET, - &info->aio_result.result)) - { /* Skip async io */ - my_errno=errno; - DBUG_PRINT("error",("got error: %d, aio_result: %d from aioread, async skipped", - errno, info->aio_result.result.aio_errno)); - if (info->request_pos != info->buffer) - { - bmove(info->buffer,info->request_pos, - (size_t) (info->read_end - info->read_pos)); - info->request_pos=info->buffer; - info->read_pos-=info->read_length; - info->read_end-=info->read_length; - } - info->read_length=info->buffer_length; /* Use hole buffer */ - info->read_function=_my_b_cache_read; /* Use normal IO_READ next */ - } - else - info->inited=info->aio_result.pending=1; - } - return 0; /* Block read, async in use */ -} /* _my_b_async_read */ -#endif - - /* Read one byte when buffer is empty */ int _my_b_get(IO_CACHE *info) @@ -1799,6 +1567,8 @@ int my_b_append(IO_CACHE *info, const uchar *Buffer, size_t Count) { size_t rest_length,length; + MEM_CHECK_DEFINED(Buffer, Count); + /* Assert that we cannot come here with a shared cache. If we do one day, we might need to add a call to copy_to_read_buffer(). @@ -1968,13 +1738,6 @@ int my_b_flush_io_cache(IO_CACHE *info, int need_append_buffer_lock) DBUG_RETURN(info->error); } } -#ifdef HAVE_AIOWAIT - else if (info->type != READ_NET) - { - my_aiowait(&info->aio_result); /* Wait for outstanding req */ - info->inited=0; - } -#endif UNLOCK_APPEND_BUFFER; DBUG_RETURN(0); } diff --git a/mysys/mf_keycache.c b/mysys/mf_keycache.c index 07dd8f3e1ef..ec5e39938b6 100644 --- a/mysys/mf_keycache.c +++ b/mysys/mf_keycache.c @@ -162,6 +162,7 @@ typedef struct st_simple_key_cache_cb my_bool resize_in_flush; /* true during flush of resize operation */ my_bool can_be_used; /* usage of cache for read/write is allowed */ size_t key_cache_mem_size; /* specified size of the cache memory */ + size_t allocated_mem_size; /* size of the memory actually allocated */ uint key_cache_block_size; /* size of the page buffer of a cache block */ ulong min_warm_blocks; /* min number of warm blocks; */ ulong age_threshold; /* age threshold for hot blocks */ @@ -545,16 +546,15 @@ int init_simple_key_cache(SIMPLE_KEY_CACHE_CB *keycache, sizeof(BLOCK_LINK*)* (changed_blocks_hash_size*2))) + ((size_t) blocks * keycache->key_cache_block_size) > use_mem && blocks > 8) blocks--; - /* Allocate memory for cache page buffers */ - if ((keycache->block_mem= - my_large_malloc((size_t) blocks * keycache->key_cache_block_size, - MYF(0)))) + keycache->allocated_mem_size= blocks * keycache->key_cache_block_size; + if ((keycache->block_mem= my_large_malloc(&keycache->allocated_mem_size, + MYF(0)))) { /* Allocate memory for blocks, hash_links and hash entries; For each block 2 hash links are allocated */ - if (my_multi_malloc_large(MYF(MY_ZEROFILL), + if (my_multi_malloc_large(key_memory_KEY_CACHE, MYF(MY_ZEROFILL), &keycache->block_root, (ulonglong) (blocks * sizeof(BLOCK_LINK)), &keycache->hash_root, @@ -570,7 +570,7 @@ int init_simple_key_cache(SIMPLE_KEY_CACHE_CB *keycache, changed_blocks_hash_size), NullS)) break; - my_large_free(keycache->block_mem); + my_large_free(keycache->block_mem, keycache->allocated_mem_size); keycache->block_mem= 0; } if (blocks < 8) @@ -631,7 +631,7 @@ err: keycache->blocks= 0; if (keycache->block_mem) { - my_large_free((uchar*) keycache->block_mem); + my_large_free((uchar*) keycache->block_mem, keycache->allocated_mem_size); keycache->block_mem= NULL; } if (keycache->block_root) @@ -965,7 +965,7 @@ void end_simple_key_cache(SIMPLE_KEY_CACHE_CB *keycache, my_bool cleanup) { if (keycache->block_mem) { - my_large_free((uchar*) keycache->block_mem); + my_large_free((uchar*) keycache->block_mem, keycache->allocated_mem_size); keycache->block_mem= NULL; my_free(keycache->block_root); keycache->block_root= NULL; @@ -3949,8 +3949,8 @@ static int flush_key_blocks_int(SIMPLE_KEY_CACHE_CB *keycache, changed blocks appear while we need to wait for something. */ if ((count > FLUSH_CACHE) && - !(cache= (BLOCK_LINK**) my_malloc(sizeof(BLOCK_LINK*)*count, - MYF(0)))) + !(cache= (BLOCK_LINK**) my_malloc(key_memory_KEY_CACHE, + sizeof(BLOCK_LINK*)*count, MYF(0)))) cache= cache_buff; /* After a restart there could be more changed blocks than now. @@ -5113,7 +5113,8 @@ int init_partitioned_key_cache(PARTITIONED_KEY_CACHE_CB *keycache, else { if(!(partition_ptr= - (SIMPLE_KEY_CACHE_CB **) my_malloc(sizeof(SIMPLE_KEY_CACHE_CB *) * + (SIMPLE_KEY_CACHE_CB **) my_malloc(key_memory_KEY_CACHE, + sizeof(SIMPLE_KEY_CACHE_CB *) * partitions, MYF(MY_WME)))) DBUG_RETURN(-1); bzero(partition_ptr, sizeof(SIMPLE_KEY_CACHE_CB *) * partitions); @@ -5131,7 +5132,8 @@ int init_partitioned_key_cache(PARTITIONED_KEY_CACHE_CB *keycache, else { if (!(partition= - (SIMPLE_KEY_CACHE_CB *) my_malloc(sizeof(SIMPLE_KEY_CACHE_CB), + (SIMPLE_KEY_CACHE_CB *) my_malloc(key_memory_KEY_CACHE, + sizeof(SIMPLE_KEY_CACHE_CB), MYF(MY_WME)))) continue; partition->key_cache_inited= 0; @@ -5909,7 +5911,8 @@ int init_key_cache_internal(KEY_CACHE *keycache, uint key_cache_block_size, { if (partitions == 0) { - if (!(keycache_cb= (void *) my_malloc(sizeof(SIMPLE_KEY_CACHE_CB), + if (!(keycache_cb= (void *) my_malloc(key_memory_KEY_CACHE, + sizeof(SIMPLE_KEY_CACHE_CB), MYF(0)))) return 0; ((SIMPLE_KEY_CACHE_CB *) keycache_cb)->key_cache_inited= 0; @@ -5918,7 +5921,8 @@ int init_key_cache_internal(KEY_CACHE *keycache, uint key_cache_block_size, } else { - if (!(keycache_cb= (void *) my_malloc(sizeof(PARTITIONED_KEY_CACHE_CB), + if (!(keycache_cb= (void *) my_malloc(key_memory_KEY_CACHE, + sizeof(PARTITIONED_KEY_CACHE_CB), MYF(0)))) return 0; ((PARTITIONED_KEY_CACHE_CB *) keycache_cb)->key_cache_inited= 0; diff --git a/mysys/mf_pack.c b/mysys/mf_pack.c index d79aa18dab2..b8722a2fdde 100644 --- a/mysys/mf_pack.c +++ b/mysys/mf_pack.c @@ -1,5 +1,5 @@ /* Copyright (c) 2000, 2013, Oracle and/or its affiliates. - Copyright (c) 2012, 2013, Monty Program Ab. + Copyright (c) 2012, 2020, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -152,7 +152,7 @@ size_t cleanup_dirname(register char *to, const char *from) { #ifdef BACKSLASH_MBTAIL uint l; - if (use_mb(fs) && (l= my_ismbchar(fs, from_ptr - 1, from_ptr + 2))) + if (my_ci_use_mb(fs) && (l= my_ismbchar(fs, from_ptr - 1, from_ptr + 2))) { for (l-- ; l ; *++pos= *from_ptr++, l--); start= pos + 1; /* Don't look inside multi-byte char */ diff --git a/mysys/mf_sort.c b/mysys/mf_sort.c index 3cbad020cb1..24e875b813e 100644 --- a/mysys/mf_sort.c +++ b/mysys/mf_sort.c @@ -24,7 +24,8 @@ void my_string_ptr_sort(uchar *base, uint items, size_t size) uchar **ptr=0; if (radixsort_is_appliccable(items, size) && - (ptr= (uchar**) my_malloc(items*sizeof(char*),MYF(0)))) + (ptr= (uchar**) my_malloc(PSI_NOT_INSTRUMENTED, + items * sizeof(char*),MYF(0)))) { radixsort_for_str_ptr((uchar**) base,items,size,ptr); my_free(ptr); diff --git a/mysys/mf_tempdir.c b/mysys/mf_tempdir.c index 39de3570fad..d09b307c4c7 100644 --- a/mysys/mf_tempdir.c +++ b/mysys/mf_tempdir.c @@ -30,7 +30,8 @@ my_bool init_tmpdir(MY_TMPDIR *tmpdir, const char *pathlist) DBUG_PRINT("enter", ("pathlist: %s", pathlist ? pathlist : "NULL")); mysql_mutex_init(key_TMPDIR_mutex, &tmpdir->mutex, MY_MUTEX_INIT_FAST); - if (my_init_dynamic_array(&tmpdir->full_list, sizeof(char*), 1, 5, MYF(0))) + if (my_init_dynamic_array(key_memory_MY_TMPDIR_full_list, &tmpdir->full_list, + sizeof(char*), 1, 5, MYF(0))) goto err; if (!pathlist || !pathlist[0]) { @@ -51,7 +52,7 @@ my_bool init_tmpdir(MY_TMPDIR *tmpdir, const char *pathlist) end=strcend(pathlist, DELIM); strmake(buff, pathlist, (uint) (end-pathlist)); length= cleanup_dirname(buff, buff); - if (!(copy= my_strndup(buff, length, MYF(MY_WME))) || + if (!(copy= my_strndup(key_memory_MY_TMPDIR_full_list, buff, length, MYF(MY_WME))) || insert_dynamic(&tmpdir->full_list, (uchar*) ©)) DBUG_RETURN(TRUE); pathlist=end+1; diff --git a/mysys/mulalloc.c b/mysys/mulalloc.c index 26f8253bc87..357f9315f2b 100644 --- a/mysys/mulalloc.c +++ b/mysys/mulalloc.c @@ -32,7 +32,7 @@ NULL */ -void* my_multi_malloc(myf myFlags, ...) +void* my_multi_malloc(PSI_memory_key key, myf myFlags, ...) { va_list args; char **ptr,*start,*res; @@ -48,7 +48,7 @@ void* my_multi_malloc(myf myFlags, ...) } va_end(args); - if (!(start=(char *) my_malloc(tot_length,myFlags))) + if (!(start=(char *) my_malloc(key, tot_length,myFlags))) DBUG_RETURN(0); /* purecov: inspected */ va_start(args,myFlags); @@ -76,7 +76,7 @@ void* my_multi_malloc(myf myFlags, ...) NULL */ -void *my_multi_malloc_large(myf myFlags, ...) +void *my_multi_malloc_large(PSI_memory_key key, myf myFlags, ...) { va_list args; char **ptr,*start,*res; @@ -92,7 +92,7 @@ void *my_multi_malloc_large(myf myFlags, ...) } va_end(args); - if (!(start=(char *) my_malloc((size_t) tot_length, myFlags))) + if (!(start=(char *) my_malloc(key, (size_t) tot_length, myFlags))) DBUG_RETURN(0); /* purecov: inspected */ va_start(args,myFlags); diff --git a/mysys/my_addr_resolve.c b/mysys/my_addr_resolve.c index f0c0d214171..0603a50fac7 100644 --- a/mysys/my_addr_resolve.c +++ b/mysys/my_addr_resolve.c @@ -170,7 +170,6 @@ static pid_t pid; static char addr2line_binary[1024]; static char output[1024]; static struct pollfd poll_fds; -static Dl_info info; int start_addr2line_fork(const char *binary_path) { @@ -211,7 +210,9 @@ int start_addr2line_fork(const char *binary_path) return 0; } -int my_addr_resolve(void *ptr, my_addr_loc *loc) +static int first_error= 0; + +static int addr_resolve(void *ptr, my_addr_loc *loc) { char input[32]; size_t len; @@ -225,31 +226,16 @@ int my_addr_resolve(void *ptr, my_addr_loc *loc) int filename_start = -1; int line_number_start = -1; - void *offset; - poll_fds.fd = out[0]; poll_fds.events = POLLIN | POLLRDBAND; - if (!dladdr(ptr, &info)) - return 1; - - if (strcmp(addr2line_binary, info.dli_fname)) - { - /* We use dli_fname in case the path is longer than the length of our static - string. We don't want to allocate anything dynamicaly here as we are in - a "crashed" state. */ - if (start_addr2line_fork(info.dli_fname)) - { - addr2line_binary[0] = '\0'; - return 2; - } - /* Save result for future comparisons. */ - strnmov(addr2line_binary, info.dli_fname, sizeof(addr2line_binary)); - } - offset = info.dli_fbase; - len= my_snprintf(input, sizeof(input), "%08x\n", (ulonglong)(ptr - offset)); + len= my_snprintf(input, sizeof(input), "%p\n", ptr); if (write(in[1], input, len) <= 0) + { + if (!first_error++) + fputs("Printing to addr2line failed\n", stderr); return 3; + } /* 500 ms should be plenty of time for addr2line to issue a response. */ @@ -299,7 +285,7 @@ int my_addr_resolve(void *ptr, my_addr_loc *loc) loc->line= atoi(output + line_number_start); /* Addr2line was unable to extract any meaningful information. */ - if (strcmp(loc->file, "??") == 0) + if (strcmp(loc->file, "??") == 0 && loc->func[0] == '?') return 6; loc->file= strip_path(loc->file); @@ -307,6 +293,38 @@ int my_addr_resolve(void *ptr, my_addr_loc *loc) return 0; } + +int my_addr_resolve(void *ptr, my_addr_loc *loc) +{ + Dl_info info; + int error; + + if (!dladdr(ptr, &info)) + return 1; + + if (strcmp(addr2line_binary, info.dli_fname)) + { + /* + We use dli_fname in case the path is longer than the length of + our static string. We don't want to allocate anything + dynamicaly here as we are in a "crashed" state. + */ + if (start_addr2line_fork(info.dli_fname)) + { + if (!first_error++) + fputs("Can't start addr2line\n", stderr); + addr2line_binary[0] = '\0'; + return 2; + } + /* Save result for future comparisons. */ + strnmov(addr2line_binary, info.dli_fname, sizeof(addr2line_binary)); + } + if (!(error= addr_resolve((void*) (ptr - info.dli_fbase), loc))) + return 0; + return error; +} + + const char *my_addr_resolve_init() { return 0; diff --git a/mysys/my_alloc.c b/mysys/my_alloc.c index a6f38dcb145..3e0b774b0c7 100644 --- a/mysys/my_alloc.c +++ b/mysys/my_alloc.c @@ -55,35 +55,32 @@ Because of this, we store in MY_THREAD_SPECIFIC as bit 1 in block_size */ -void init_alloc_root(MEM_ROOT *mem_root, const char *name, size_t block_size, +void init_alloc_root(PSI_memory_key key, MEM_ROOT *mem_root, size_t block_size, size_t pre_alloc_size __attribute__((unused)), myf my_flags) { DBUG_ENTER("init_alloc_root"); - DBUG_PRINT("enter",("root: %p name: %s prealloc: %zu", mem_root, - name, pre_alloc_size)); + DBUG_PRINT("enter",("root: %p prealloc: %zu", mem_root, pre_alloc_size)); mem_root->free= mem_root->used= mem_root->pre_alloc= 0; mem_root->min_malloc= 32; mem_root->block_size= (block_size - ALLOC_ROOT_MIN_BLOCK_SIZE) & ~1; - if (MY_TEST(my_flags & MY_THREAD_SPECIFIC)) + if (my_flags & MY_THREAD_SPECIFIC) mem_root->block_size|= 1; mem_root->error_handler= 0; mem_root->block_num= 4; /* We shift this with >>2 */ mem_root->first_block_usage= 0; - mem_root->total_alloc= 0; - mem_root->name= name; + mem_root->m_psi_key= key; #if !(defined(HAVE_valgrind) && defined(EXTRA_DEBUG)) if (pre_alloc_size) { + size_t size= pre_alloc_size + ALIGN_SIZE(sizeof(USED_MEM)); if ((mem_root->free= mem_root->pre_alloc= - (USED_MEM*) my_malloc(pre_alloc_size + ALIGN_SIZE(sizeof(USED_MEM)), - MYF(my_flags)))) + (USED_MEM*) my_malloc(key, size, MYF(my_flags)))) { - mem_root->free->size= pre_alloc_size+ALIGN_SIZE(sizeof(USED_MEM)); - mem_root->total_alloc= pre_alloc_size+ALIGN_SIZE(sizeof(USED_MEM)); + mem_root->free->size= size; mem_root->free->left= pre_alloc_size; mem_root->free->next= 0; TRASH_MEM(mem_root->free); @@ -142,19 +139,17 @@ void reset_root_defaults(MEM_ROOT *mem_root, size_t block_size, { /* remove block from the list and free it */ *prev= mem->next; - mem_root->total_alloc-= mem->size; my_free(mem); } else prev= &mem->next; } /* Allocate new prealloc block and add it to the end of free list */ - if ((mem= (USED_MEM *) my_malloc(size, + if ((mem= (USED_MEM *) my_malloc(mem_root->m_psi_key, size, MYF(MALLOC_FLAG(mem_root-> block_size))))) { mem->size= size; - mem_root->total_alloc+= size; mem->left= pre_alloc_size; mem->next= *prev; *prev= mem_root->pre_alloc= mem; @@ -179,7 +174,7 @@ void *alloc_root(MEM_ROOT *mem_root, size_t length) #if defined(HAVE_valgrind) && defined(EXTRA_DEBUG) reg1 USED_MEM *next; DBUG_ENTER("alloc_root"); - DBUG_PRINT("enter",("root: %p name: %s", mem_root, mem_root->name)); + DBUG_PRINT("enter",("root: %p", mem_root)); DBUG_ASSERT(alloc_root_inited(mem_root)); @@ -192,7 +187,7 @@ void *alloc_root(MEM_ROOT *mem_root, size_t length) }); length+=ALIGN_SIZE(sizeof(USED_MEM)); - if (!(next = (USED_MEM*) my_malloc(length, + if (!(next = (USED_MEM*) my_malloc(mem_root->m_psi_key, length, MYF(MY_WME | ME_FATAL | MALLOC_FLAG(mem_root->block_size))))) { @@ -204,10 +199,8 @@ void *alloc_root(MEM_ROOT *mem_root, size_t length) next->left= 0; next->size= length; mem_root->used= next; - mem_root->total_alloc+= length; - DBUG_PRINT("exit",("ptr: %p", (((char*) next)+ - ALIGN_SIZE(sizeof(USED_MEM))))); - DBUG_RETURN((uchar*) (((char*) next)+ALIGN_SIZE(sizeof(USED_MEM)))); + DBUG_PRINT("exit",("ptr: %p", (((char*)next)+ALIGN_SIZE(sizeof(USED_MEM))))); + DBUG_RETURN((((uchar*) next)+ALIGN_SIZE(sizeof(USED_MEM)))); #else size_t get_size, block_size; uchar* point; @@ -215,7 +208,7 @@ void *alloc_root(MEM_ROOT *mem_root, size_t length) reg2 USED_MEM **prev; size_t original_length __attribute__((unused)) = length; DBUG_ENTER("alloc_root"); - DBUG_PRINT("enter",("root: %p name: %s", mem_root, mem_root->name)); + DBUG_PRINT("enter",("root: %p", mem_root)); DBUG_ASSERT(alloc_root_inited(mem_root)); DBUG_EXECUTE_IF("simulate_out_of_memory", @@ -248,7 +241,7 @@ void *alloc_root(MEM_ROOT *mem_root, size_t length) get_size= length+ALIGN_SIZE(sizeof(USED_MEM)); get_size= MY_MAX(get_size, block_size); - if (!(next = (USED_MEM*) my_malloc(get_size, + if (!(next = (USED_MEM*) my_malloc(mem_root->m_psi_key, get_size, MYF(MY_WME | ME_FATAL | MALLOC_FLAG(mem_root-> block_size))))) @@ -258,7 +251,6 @@ void *alloc_root(MEM_ROOT *mem_root, size_t length) DBUG_RETURN((void*) 0); /* purecov: inspected */ } mem_root->block_num++; - mem_root->total_alloc+= get_size; next->next= *prev; next->size= get_size; next->left= get_size-ALIGN_SIZE(sizeof(USED_MEM)); @@ -395,8 +387,7 @@ void free_root(MEM_ROOT *root, myf MyFlags) { reg1 USED_MEM *next,*old; DBUG_ENTER("free_root"); - DBUG_PRINT("enter",("root: %p name: %s flags: %u", root, root->name, - (uint) MyFlags)); + DBUG_PRINT("enter",("root: %p flags: %lu", root, MyFlags)); #if !(defined(HAVE_valgrind) && defined(EXTRA_DEBUG)) /* @@ -416,19 +407,13 @@ void free_root(MEM_ROOT *root, myf MyFlags) { old=next; next= next->next ; if (old != root->pre_alloc) - { - root->total_alloc-= old->size; my_free(old); - } } for (next=root->free ; next ;) { old=next; next= next->next; if (old != root->pre_alloc) - { - root->total_alloc-= old->size; my_free(old); - } } root->used=root->free=0; if (root->pre_alloc) diff --git a/mysys/my_atomic_writes.c b/mysys/my_atomic_writes.c index b383af11ba8..cfd7dbb3cbf 100644 --- a/mysys/my_atomic_writes.c +++ b/mysys/my_atomic_writes.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2016, MariaDB Corporation +/* Copyright (c) 2016, 2021, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -19,10 +19,15 @@ my_bool my_may_have_atomic_write= IF_WIN(1,0); #ifdef __linux__ -my_bool has_shannon_atomic_write= 0, has_fusion_io_atomic_write= 0; +my_bool has_shannon_atomic_write= 0, has_fusion_io_atomic_write= 0, + has_sfx_atomic_write= 0; #include <sys/ioctl.h> +/* Linux seems to allow up to 15 partitions per block device. +Partition number 0 is the whole block device. */ +# define SAME_DEV(fs_dev, blk_dev) \ + (fs_dev == blk_dev) || ((fs_dev & ~15U) == blk_dev) /*********************************************************************** FUSION_IO @@ -114,9 +119,9 @@ static my_bool test_if_shannon_card_exists() sprintf(path, "/dev/df%c", dev_part); #ifdef TEST_SHANNON - if (lstat(path, &stat_buff) < 0) + if (stat(path, &stat_buff) < 0) { - printf("%s(): lstat failed.\n", __func__); + printf("%s(): stat %s failed.\n", __func__, path); break; } #endif @@ -146,7 +151,7 @@ static my_bool test_if_shannon_card_exists() for (dev_no= 1 ; dev_no < 9 ; dev_no++) { sprintf(path, "/dev/df%c%d", dev_part, dev_no); - if (lstat(path, &stat_buff) < 0) + if (stat(path, &stat_buff) < 0) break; shannon_devices[shannon_found_devices].st_dev= stat_buff.st_rdev; @@ -193,12 +198,13 @@ static my_bool shannon_dev_has_atomic_write(struct shannon_dev *dev, int fd= open(dev->dev_name, 0); if (fd < 0) { - perror("open() failed!"); + fprintf(stderr, "Unable to determine if atomic writes are supported:" + " open(\"%s\"): %m\n", dev->dev_name); dev->atomic_size= 0; /* Don't try again */ return FALSE; } - dev->atomic_size= ioctl(fd, SHANNON_IOCQATOMIC_SIZE); - close(fd); + dev->atomic_size= ioctl(fd, SHANNON_IOCQATOMIC_SIZE); + close(fd); } #ifdef TEST_SHANNON @@ -247,7 +253,7 @@ static my_bool shannon_has_atomic_write(File file, int page_size) #ifdef TEST_SHANNON printf("%s(): st_rdev=0x%lx\n", __func__, (ulong) dev->st_dev); #endif - if (stat_buff.st_dev == dev->st_dev) + if (SAME_DEV(stat_buff.st_dev, dev->st_dev)) return shannon_dev_has_atomic_write(dev, page_size); } return 0; @@ -255,6 +261,104 @@ static my_bool shannon_has_atomic_write(File file, int page_size) /*********************************************************************** + ScaleFlux +************************************************************************/ + +#define SFX_GET_ATOMIC_SIZE _IO('N', 0x244) +#define SFX_MAX_DEVICES 32 +#define SFX_NO_ATOMIC_SIZE_YET -2 + +struct sfx_dev +{ + char dev_name[32]; + dev_t st_dev; + int atomic_size; +}; + +static struct sfx_dev sfx_devices[SFX_MAX_DEVICES + 1]; + +/** + Check if the system has a ScaleFlux card + If card exists, record device numbers to allow us to later check if + a given file is on this device. + @return TRUE Card exists +*/ + +static my_bool test_if_sfx_card_exists() +{ + uint sfx_found_devices = 0; + uint dev_num; + + for (dev_num = 0; dev_num < SFX_MAX_DEVICES; dev_num++) + { + struct stat stat_buff; + + sprintf(sfx_devices[sfx_found_devices].dev_name, "/dev/sfdv%dn1", + dev_num); + if (stat(sfx_devices[sfx_found_devices].dev_name, &stat_buff) < 0) + break; + + sfx_devices[sfx_found_devices].st_dev= stat_buff.st_rdev; + /* + The atomic size will be checked on first access. This is needed + as a normal user can't open the /dev/sfdvXn1 file + */ + sfx_devices[sfx_found_devices].atomic_size = SFX_NO_ATOMIC_SIZE_YET; + if (++sfx_found_devices == SFX_MAX_DEVICES) + goto end; + } +end: + sfx_devices[sfx_found_devices].st_dev= 0; + return sfx_found_devices > 0; +} + +static my_bool sfx_dev_has_atomic_write(struct sfx_dev *dev, + int page_size) +{ + if (dev->atomic_size == SFX_NO_ATOMIC_SIZE_YET) + { + int fd= open(dev->dev_name, 0); + if (fd < 0) + { + fprintf(stderr, "Unable to determine if atomic writes are supported:" + " open(\"%s\"): %m\n", dev->dev_name); + dev->atomic_size= 0; /* Don't try again */ + } + else + { + dev->atomic_size= ioctl(fd, SFX_GET_ATOMIC_SIZE); + close(fd); + } + } + + return (page_size <= dev->atomic_size); +} + + +/** + Check if a file is on a ScaleFlux device and that it supports atomic_write + @param[in] file OS file handle + @param[in] page_size page size + @return TRUE Atomic write supported + + @notes + This is called only at first open of a file. In this case it's doesn't + matter so much that we loop over all cards. + We update the atomic size on first access. +*/ + +static my_bool sfx_has_atomic_write(File file, int page_size) +{ + struct sfx_dev *dev; + struct stat stat_buff; + + if (fstat(file, &stat_buff) == 0) + for (dev= sfx_devices; dev->st_dev; dev++) + if (SAME_DEV(stat_buff.st_dev, dev->st_dev)) + return sfx_dev_has_atomic_write(dev, page_size); + return 0; +} +/*********************************************************************** Generic atomic write code ************************************************************************/ @@ -265,9 +369,13 @@ static my_bool shannon_has_atomic_write(File file, int page_size) void my_init_atomic_write(void) { - if ((has_shannon_atomic_write= test_if_shannon_card_exists()) || - (has_fusion_io_atomic_write= test_if_fusion_io_card_exists())) - my_may_have_atomic_write= 1; + has_shannon_atomic_write= test_if_shannon_card_exists(); + has_fusion_io_atomic_write= test_if_fusion_io_card_exists(); + has_sfx_atomic_write= test_if_sfx_card_exists(); + + my_may_have_atomic_write= has_shannon_atomic_write || + has_fusion_io_atomic_write || has_sfx_atomic_write; + #ifdef TEST_SHANNON printf("%s(): has_shannon_atomic_write=%d, my_may_have_atomic_write=%d\n", __func__, @@ -294,6 +402,7 @@ my_bool my_test_if_atomic_write(File handle, int page_size) #endif if (!my_may_have_atomic_write) return 0; + if (has_shannon_atomic_write && shannon_has_atomic_write(handle, page_size)) return 1; @@ -302,6 +411,10 @@ my_bool my_test_if_atomic_write(File handle, int page_size) fusion_io_has_atomic_write(handle, page_size)) return 1; + if (has_sfx_atomic_write && + sfx_has_atomic_write(handle, page_size)) + return 1; + return 0; } diff --git a/mysys/my_bitmap.c b/mysys/my_bitmap.c index c69eb4748f7..bf2bacd213c 100644 --- a/mysys/my_bitmap.c +++ b/mysys/my_bitmap.c @@ -173,7 +173,8 @@ my_bool my_bitmap_init(MY_BITMAP *map, my_bitmap_map *buf, uint n_bits, size_in_bytes= ALIGN_SIZE(size_in_bytes); extra= sizeof(mysql_mutex_t); } - if (!(buf= (my_bitmap_map*) my_malloc(size_in_bytes+extra, MYF(MY_WME)))) + if (!(buf= (my_bitmap_map*) my_malloc(key_memory_MY_BITMAP_bitmap, + size_in_bytes+extra, MYF(MY_WME)))) DBUG_RETURN(1); if (thread_safe) { diff --git a/mysys/my_compare.c b/mysys/my_compare.c index 358cb77008e..d4ecfd7b4aa 100644 --- a/mysys/my_compare.c +++ b/mysys/my_compare.c @@ -1,5 +1,5 @@ /* Copyright (c) 2011, Oracle and/or its affiliates. - Copyright (C) 2009-2011 Monty Program Ab + Copyright (c) 2009, 2020, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -24,9 +24,9 @@ int ha_compare_text(CHARSET_INFO *charset_info, const uchar *a, size_t a_length, const uchar *b, size_t b_length, my_bool part_key) { if (!part_key) - return charset_info->coll->strnncollsp(charset_info, a, a_length, - b, b_length); - return charset_info->coll->strnncoll(charset_info, a, a_length, + return my_ci_strnncollsp(charset_info, a, a_length, + b, b_length); + return my_ci_strnncoll(charset_info, a, a_length, b, b_length, part_key); } diff --git a/mysys/my_compress.c b/mysys/my_compress.c index 7cd85d57dd8..11603e7ba24 100644 --- a/mysys/my_compress.c +++ b/mysys/my_compress.c @@ -15,7 +15,7 @@ /* Written by Sinisa Milivojevic <sinisa@mysql.com> */ -#include <my_global.h> +#include <mysys_priv.h> #ifdef HAVE_COMPRESS #include <my_sys.h> #ifndef SCO @@ -84,7 +84,8 @@ my_bool my_compress(uchar *packet, size_t *len, size_t *complen) void *my_az_allocator(void *dummy __attribute__((unused)), unsigned int items, unsigned int size) { - return my_malloc((size_t)items*(size_t)size, IF_VALGRIND(MY_ZEROFILL, MYF(0))); + return my_malloc(key_memory_my_compress_alloc, (size_t)items*(size_t)size, + IF_VALGRIND(MY_ZEROFILL, MYF(0))); } void my_az_free(void *dummy __attribute__((unused)), void *address) @@ -133,8 +134,9 @@ uchar *my_compress_alloc(const uchar *packet, size_t *len, size_t *complen) int res; *complen= *len * 120 / 100 + 12; - if (!(compbuf= (uchar *) my_malloc(*complen, MYF(MY_WME)))) - return 0; /* Not enough memory */ + if (!(compbuf= (uchar *) my_malloc(key_memory_my_compress_alloc, + *complen, MYF(MY_WME)))) + return 0; res= my_compress_buffer(compbuf, complen, packet, *len); @@ -180,7 +182,8 @@ my_bool my_uncompress(uchar *packet, size_t len, size_t *complen) if (*complen) /* If compressed */ { - uchar *compbuf= (uchar *) my_malloc(*complen,MYF(MY_WME)); + uchar *compbuf= (uchar *) my_malloc(key_memory_my_compress_alloc, + *complen,MYF(MY_WME)); int error; if (!compbuf) DBUG_RETURN(1); /* Not enough memory */ diff --git a/mysys/my_default.c b/mysys/my_default.c index 7c6ef8fffa0..c76ab8c6eb1 100644 --- a/mysys/my_default.c +++ b/mysys/my_default.c @@ -43,52 +43,23 @@ #include <winbase.h> #endif -/** - arguments separator - - load_defaults() loads arguments from config file and put them - before the arguments from command line, this separator is used to - separate the arguments loaded from config file and arguments user - provided on command line. - - Options with value loaded from config file are always in the form - '--option=value', while for command line options, the value can be - given as the next argument. Thus we used a separator so that - handle_options() can distinguish them. - - Note: any other places that does not need to distinguish them - should skip the separator. - - The content of arguments separator does not matter, one should only - check the pointer, use "----args-separator----" here to ease debug - if someone misused it. - - The args separator will only be added when - my_getopt_use_args_seprator is set to TRUE before calling - load_defaults(); - - See BUG#25192 +/* + Mark file names in argv[]. File marker is *always* followed by a file name + All options after it come from that file. + Empty file name ("") means command line. */ -static const char *args_separator= "----args-separator----"; -inline static void set_args_separator(char** arg) +static char *file_marker= (char*)"----file-marker----"; +my_bool my_defaults_mark_files= FALSE; +my_bool is_file_marker(const char* arg) { - DBUG_ASSERT(my_getopt_use_args_separator); - *arg= (char*)args_separator; -} -my_bool my_getopt_use_args_separator= FALSE; -my_bool my_getopt_is_args_separator(const char* arg) -{ - return (arg == args_separator); + return arg == file_marker; } + +my_bool my_no_defaults=FALSE, my_print_defaults= FALSE; const char *my_defaults_file=0; const char *my_defaults_group_suffix=0; const char *my_defaults_extra_file=0; -static char my_defaults_file_buffer[FN_REFLEN]; -static char my_defaults_extra_file_buffer[FN_REFLEN]; - -static my_bool defaults_already_read= FALSE; - /* Which directories are searched for options (and in which order) */ #define MAX_DEFAULT_DIRS 7 @@ -103,15 +74,6 @@ static const char *f_extensions[]= { ".cnf", 0 }; #define NEWLINE "\n" #endif -static int handle_default_option(void *, const char *, const char *); - -/* - This structure defines the context that we pass to callback - function 'handle_default_option' used in search_default_file - to process each option. This context is used if search_default_file - was called from load_defaults. -*/ - struct handle_option_ctx { MEM_ROOT *alloc; @@ -119,12 +81,11 @@ struct handle_option_ctx TYPELIB *group; }; -static int search_default_file(Process_option_func func, void *func_ctx, - const char *dir, const char *config_file); -static int search_default_file_with_ext(Process_option_func func, - void *func_ctx, - const char *dir, const char *ext, - const char *config_file, int recursion_level); +static int search_default_file(struct handle_option_ctx *, + const char *, const char *); +static int search_default_file_with_ext(struct handle_option_ctx *, + const char *, const char *, + const char *, int); /** @@ -159,33 +120,6 @@ static char *remove_end_comment(char *ptr); /* - Expand a file name so that the current working directory is added if - the name is relative. - - RETURNS - 0 All OK - 2 Out of memory or path to long - 3 Not able to get working directory - */ - -static int -fn_expand(const char *filename, char *result_buf) -{ - char dir[FN_REFLEN]; - const int flags= MY_UNPACK_FILENAME | MY_SAFE_PATH | MY_RELATIVE_PATH; - DBUG_ENTER("fn_expand"); - DBUG_PRINT("enter", ("filename: %s, result_buf: %p", - filename, result_buf)); - if (my_getwd(dir, sizeof(dir), MYF(0))) - DBUG_RETURN(3); - DBUG_PRINT("debug", ("dir: %s", dir)); - if (fn_format(result_buf, filename, dir, "", flags) == NULL) - DBUG_RETURN(2); - DBUG_PRINT("return", ("result: %s", result_buf)); - DBUG_RETURN(0); -} - -/* Process config files in default directories. SYNOPSIS @@ -194,8 +128,6 @@ fn_expand(const char *filename, char *result_buf) If this is a path, then only this file is read. argc Pointer to argc of original program argv Pointer to argv of original program - args_used Pointer to variable for storing the number of - arguments used. func Pointer to the function to process options func_ctx It's context. Usually it is the structure to store additional options. @@ -219,53 +151,20 @@ fn_expand(const char *filename, char *result_buf) --defaults_group_suffix */ -int my_search_option_files(const char *conf_file, int *argc, char ***argv, - uint *args_used, Process_option_func func, - void *func_ctx, const char **default_directories) +static int my_search_option_files(const char *conf_file, + struct handle_option_ctx *ctx, + const char **default_directories) { - const char **dirs, *forced_default_file, *forced_extra_defaults; + const char **dirs; int error= 0; DBUG_ENTER("my_search_option_files"); - /* Check if we want to force the use a specific default file */ - *args_used+= get_defaults_options(*argc - *args_used, *argv + *args_used, - (char **) &forced_default_file, - (char **) &forced_extra_defaults, - (char **) &my_defaults_group_suffix); - - if (! my_defaults_group_suffix) - my_defaults_group_suffix= getenv("MYSQL_GROUP_SUFFIX"); - - if (forced_extra_defaults && !defaults_already_read) - { - int error= fn_expand(forced_extra_defaults, my_defaults_extra_file_buffer); - if (error) - DBUG_RETURN(error); - my_defaults_extra_file= my_defaults_extra_file_buffer; - } - - if (forced_default_file && !defaults_already_read) - { - int error= fn_expand(forced_default_file, my_defaults_file_buffer); - if (error) - DBUG_RETURN(error); - my_defaults_file= my_defaults_file_buffer; - } - - defaults_already_read= TRUE; - - /* - We can only handle 'defaults-group-suffix' if we are called from - load_defaults() as otherwise we can't know the type of 'func_ctx' - */ - - if (my_defaults_group_suffix && func == handle_default_option) + if (my_defaults_group_suffix) { /* Handle --defaults-group-suffix= */ uint i; const char **extra_groups; const size_t instance_len= strlen(my_defaults_group_suffix); - struct handle_option_ctx *ctx= (struct handle_option_ctx*) func_ctx; char *ptr; TYPELIB *group= ctx->group; @@ -297,7 +196,7 @@ int my_search_option_files(const char *conf_file, int *argc, char ***argv, if (my_defaults_file) { - if ((error= search_default_file_with_ext(func, func_ctx, "", "", + if ((error= search_default_file_with_ext(ctx, "", "", my_defaults_file, 0)) < 0) goto err; if (error > 0) @@ -309,7 +208,7 @@ int my_search_option_files(const char *conf_file, int *argc, char ***argv, } else if (dirname_length(conf_file)) { - if ((error= search_default_file(func, func_ctx, NullS, conf_file)) < 0) + if ((error= search_default_file(ctx, NullS, conf_file)) < 0) goto err; } else @@ -318,12 +217,12 @@ int my_search_option_files(const char *conf_file, int *argc, char ***argv, { if (**dirs) { - if (search_default_file(func, func_ctx, *dirs, conf_file) < 0) + if (search_default_file(ctx, *dirs, conf_file) < 0) goto err; } else if (my_defaults_extra_file) { - if ((error= search_default_file_with_ext(func, func_ctx, "", "", + if ((error= search_default_file_with_ext(ctx, "", "", my_defaults_extra_file, 0)) < 0) goto err; /* Fatal error */ if (error > 0) @@ -345,47 +244,23 @@ err: /* - The option handler for load_defaults. + adds an option to the array of options SYNOPSIS - handle_deault_option() - in_ctx Handler context. In this case it is a - handle_option_ctx structure. - group_name The name of the group the option belongs to. + add_option() + in_ctx Handler context. option The very option to be processed. It is already - prepared to be used in argv (has -- prefix). If it - is NULL, we are handling a new group (section). - - DESCRIPTION - This handler checks whether a group is one of the listed and adds an option - to the array if yes. Some other handler can record, for instance, all - groups and their options, not knowing in advance the names and amount of - groups. + prepared to be used in argv (has -- prefix). RETURN 0 - ok 1 - error occurred */ -static int handle_default_option(void *in_ctx, const char *group_name, - const char *option) +static int add_option(struct handle_option_ctx *ctx, const char *option) { - char *tmp; - struct handle_option_ctx *ctx= (struct handle_option_ctx *) in_ctx; - - if (!option) - return 0; - - if (find_type((char *)group_name, ctx->group, FIND_TYPE_NO_PREFIX)) - { - if (!(tmp= alloc_root(ctx->alloc, strlen(option) + 1))) - return 1; - if (insert_dynamic(ctx->args, (uchar*) &tmp)) - return 1; - strmov(tmp, option); - } - - return 0; + char *tmp= strdup_root(ctx->alloc, option); + return !tmp || insert_dynamic(ctx->args, (uchar*) &tmp); } @@ -394,50 +269,70 @@ static int handle_default_option(void *in_ctx, const char *group_name, SYNOPSIS get_defaults_options() - argc Pointer to argc of original program argv Pointer to argv of original program - defaults --defaults-file option - extra_defaults --defaults-extra-file option + + DESCRIPTION + Sets my_no_defaults, my_defaults_file, my_defaults_extra_file, + my_defaults_group_suffix, my_print_defaults RETURN # Number of arguments used from *argv - defaults and extra_defaults will be set to option of the appropriate - items of argv array, or to NULL if there are no such options */ -int get_defaults_options(int argc, char **argv, - char **defaults, - char **extra_defaults, - char **group_suffix) +int get_defaults_options(char **argv) { - int org_argc= argc; - *defaults= *extra_defaults= *group_suffix= 0; + static char file_buffer[FN_REFLEN]; + static char extra_file_buffer[FN_REFLEN]; + char **orig_argv= argv; + + argv++; /* Skip program name */ - while (argc >= 2) + my_defaults_file= my_defaults_group_suffix= my_defaults_extra_file= 0; + my_no_defaults= my_print_defaults= FALSE; + + if (*argv && !strcmp(*argv, "--no-defaults")) { - /* Skip program name or previously handled argument */ + my_no_defaults= 1; argv++; - if (!*defaults && is_prefix(*argv,"--defaults-file=")) - { - *defaults= *argv + sizeof("--defaults-file=")-1; - argc--; - continue; - } - if (!*extra_defaults && is_prefix(*argv,"--defaults-extra-file=")) - { - *extra_defaults= *argv + sizeof("--defaults-extra-file=")-1; - argc--; - continue; - } - if (!*group_suffix && is_prefix(*argv, "--defaults-group-suffix=")) + } + else + for(; *argv; argv++) { - *group_suffix= *argv + sizeof("--defaults-group-suffix=")-1; - argc--; - continue; + if (!my_defaults_file && is_prefix(*argv, "--defaults-file=")) + my_defaults_file= *argv + sizeof("--defaults-file=")-1; + else + if (!my_defaults_extra_file && is_prefix(*argv, "--defaults-extra-file=")) + my_defaults_extra_file= *argv + sizeof("--defaults-extra-file=")-1; + else + if (!my_defaults_group_suffix && is_prefix(*argv, "--defaults-group-suffix=")) + my_defaults_group_suffix= *argv + sizeof("--defaults-group-suffix=")-1; + else + break; } - break; + + if (*argv && !strcmp(*argv, "--print-defaults")) + { + my_print_defaults= 1; + my_defaults_mark_files= FALSE; + argv++; + } + + if (! my_defaults_group_suffix) + my_defaults_group_suffix= getenv("MYSQL_GROUP_SUFFIX"); + + if (my_defaults_extra_file && my_defaults_extra_file != extra_file_buffer) + { + my_realpath(extra_file_buffer, my_defaults_extra_file, MYF(0)); + my_defaults_extra_file= extra_file_buffer; + } + + if (my_defaults_file && my_defaults_file != file_buffer) + { + my_realpath(file_buffer, my_defaults_file, MYF(0)); + my_defaults_file= file_buffer; } - return org_argc - argc; + + return (int)(argv - orig_argv); } /* @@ -504,132 +399,85 @@ int load_defaults(const char *conf_file, const char **groups, */ -int my_load_defaults(const char *conf_file, const char **groups, - int *argc, char ***argv, const char ***default_directories) +int my_load_defaults(const char *conf_file, const char **groups, int *argc, + char ***argv, const char ***default_directories) { DYNAMIC_ARRAY args; - TYPELIB group; - my_bool found_print_defaults= 0; - uint args_used= 0; + int args_used= 0; int error= 0; MEM_ROOT alloc; char *ptr,**res; - struct handle_option_ctx ctx; const char **dirs; - uint args_sep= my_getopt_use_args_separator ? 1 : 0; - DBUG_ENTER("load_defaults"); + DBUG_ENTER("my_load_defaults"); - init_alloc_root(&alloc, "my_load_defaults", 512, 0, MYF(0)); + init_alloc_root(key_memory_defaults, &alloc, 512, 0, MYF(0)); if ((dirs= init_default_directories(&alloc)) == NULL) goto err; - /* - Check if the user doesn't want any default option processing - --no-defaults is always the first option - */ - if (*argc >= 2 && !strcmp(argv[0][1],"--no-defaults")) - { - /* remove the --no-defaults argument and return only the other arguments */ - uint i, j; - if (!(ptr=(char*) alloc_root(&alloc,sizeof(alloc)+ - (*argc + 1)*sizeof(char*)))) - goto err; - res= (char**) (ptr+sizeof(alloc)); - res[0]= **argv; /* Copy program name */ - j= 1; /* Start from 1 for the reset result args */ - if (my_getopt_use_args_separator) - { - /* set arguments separator */ - set_args_separator(&res[1]); - j++; - } - for (i=2 ; i < (uint) *argc ; i++, j++) - res[j]=argv[0][i]; - res[j]=0; /* End pointer */ - /* - Update the argc, if have not added args separator, then we have - to decrease argc because we have removed the "--no-defaults". - */ - if (!my_getopt_use_args_separator) - (*argc)--; - *argv=res; - *(MEM_ROOT*) ptr= alloc; /* Save alloc root for free */ - if (default_directories) - *default_directories= dirs; - DBUG_RETURN(0); - } - - group.count=0; - group.name= "defaults"; - group.type_names= groups; - for (; *groups ; groups++) - group.count++; + args_used= get_defaults_options(*argv); - if (my_init_dynamic_array(&args, sizeof(char*), 128, 64, MYF(0))) + if (my_init_dynamic_array(key_memory_defaults, &args, sizeof(char*), 128, 64, + MYF(0))) goto err; - ctx.alloc= &alloc; - ctx.args= &args; - ctx.group= &group; + insert_dynamic(&args, *argv);/* Name MUST be set, even by embedded library */ - if ((error= my_search_option_files(conf_file, argc, argv, &args_used, - handle_default_option, (void *) &ctx, - dirs))) + *argc-= args_used; + *argv+= args_used; + + if (!my_no_defaults) { - delete_dynamic(&args); - free_root(&alloc,MYF(0)); - DBUG_RETURN(error); + TYPELIB group; // XXX + struct handle_option_ctx ctx; + + group.count=0; + group.name= "defaults"; + group.type_names= groups; + + for (; *groups ; groups++) + group.count++; + + ctx.alloc= &alloc; + ctx.args= &args; + ctx.group= &group; + + if ((error= my_search_option_files(conf_file, &ctx, dirs))) + { + delete_dynamic(&args); + free_root(&alloc,MYF(0)); + DBUG_RETURN(error); + } } - /* - Here error contains <> 0 only if we have a fully specified conf_file - or a forced default file - */ - if (!(ptr=(char*) alloc_root(&alloc,sizeof(alloc)+ - (args.elements + *argc + 1 + args_sep) *sizeof(char*)))) + + if (!(ptr=(char*) alloc_root(&alloc, sizeof(alloc) + + (args.elements + *argc + 3) * sizeof(char*)))) goto err; res= (char**) (ptr+sizeof(alloc)); - /* copy name + found arguments + command line arguments to new array */ - res[0]= argv[0][0]; /* Name MUST be set, even by embedded library */ - memcpy((uchar*) (res+1), args.buffer, args.elements*sizeof(char*)); - /* Skip --defaults-xxx options */ - (*argc)-= args_used; - (*argv)+= args_used; + /* found arguments + command line arguments to new array */ + memcpy(res, args.buffer, args.elements * sizeof(char*)); - /* - Check if we want to see the new argument list - This options must always be the last of the default options - */ - if (*argc >= 2 && !strcmp(argv[0][1],"--print-defaults")) + if (my_defaults_mark_files) { - found_print_defaults=1; - --*argc; ++*argv; /* skip argument */ - } - - if (my_getopt_use_args_separator) - { - /* set arguments separator for arguments from config file and - command line */ - set_args_separator(&res[args.elements+1]); + res[args.elements++]= file_marker; + res[args.elements++]= (char*)""; } if (*argc) - memcpy((uchar*) (res+1+args.elements+args_sep), (char*) ((*argv)+1), - (*argc-1)*sizeof(char*)); - res[args.elements+ *argc+args_sep]=0; /* last null */ + memcpy(res + args.elements, *argv, *argc * sizeof(char*)); - (*argc)+=args.elements+args_sep; - *argv= (char**) res; + (*argc)+= args.elements; + *argv= res; + (*argv)[*argc]= 0; *(MEM_ROOT*) ptr= alloc; /* Save alloc root for free */ delete_dynamic(&args); - if (found_print_defaults) + if (my_print_defaults) { int i; printf("%s would have been started with the following arguments:\n", **argv); for (i=1 ; i < *argc ; i++) - if (!my_getopt_is_args_separator((*argv)[i])) /* skip arguments separator */ - printf("%s ", (*argv)[i]); + printf("%s ", (*argv)[i]); puts(""); DBUG_RETURN(4); } @@ -653,9 +501,7 @@ void free_defaults(char **argv) } -static int search_default_file(Process_option_func opt_handler, - void *handler_ctx, - const char *dir, +static int search_default_file(struct handle_option_ctx *ctx, const char *dir, const char *config_file) { char **ext; @@ -666,9 +512,7 @@ static int search_default_file(Process_option_func opt_handler, for (ext= (char**) exts_to_use; *ext; ext++) { int error; - if ((error= search_default_file_with_ext(opt_handler, handler_ctx, - dir, *ext, - config_file, 0)) < 0) + if ((error= search_default_file_with_ext(ctx, dir, *ext, config_file, 0)) < 0) return error; } return 0; @@ -730,9 +574,7 @@ static char *get_argument(const char *keyword, size_t kwlen, SYNOPSIS search_default_file_with_ext() - opt_handler Option handler function. It is used to process - every separate option. - handler_ctx Pointer to the structure to store actual + ctx Pointer to the structure to store actual parameters of the function. dir directory to read ext Extension for configuration file @@ -747,10 +589,8 @@ static char *get_argument(const char *keyword, size_t kwlen, 1 File not found (Warning) */ -static int search_default_file_with_ext(Process_option_func opt_handler, - void *handler_ctx, - const char *dir, - const char *ext, +static int search_default_file_with_ext(struct handle_option_ctx *ctx, + const char *dir, const char *ext, const char *config_file, int recursion_level) { @@ -761,12 +601,12 @@ static int search_default_file_with_ext(Process_option_func opt_handler, const int max_recursion_level= 10; MYSQL_FILE *fp; uint line=0; - my_bool found_group=0; + enum { NONE, PARSE, SKIP } found_group= NONE; uint i; MY_DIR *search_dir; FILEINFO *search_file; - if ((dir ? strlen(dir) : 0 )+strlen(config_file) >= FN_REFLEN-3) + if (safe_strlen(dir) + strlen(config_file) >= FN_REFLEN-3) return 0; /* Ignore wrong paths */ if (dir) { @@ -802,6 +642,11 @@ static int search_default_file_with_ext(Process_option_func opt_handler, if (!(fp= mysql_file_fopen(key_file_cnf, name, O_RDONLY, MYF(0)))) return 1; /* Ignore wrong files */ + if (my_defaults_mark_files) + if (insert_dynamic(ctx->args, (uchar*) &file_marker) || + add_option(ctx, name)) + goto err; + while (mysql_file_fgets(buff, sizeof(buff) - 1, fp)) { line++; @@ -862,8 +707,7 @@ static int search_default_file_with_ext(Process_option_func opt_handler, fn_format(tmp, search_file->name, ptr, "", MY_UNPACK_FILENAME | MY_SAFE_PATH); - search_default_file_with_ext(opt_handler, handler_ctx, "", "", tmp, - recursion_level + 1); + search_default_file_with_ext(ctx, "", "", tmp, recursion_level + 1); } } @@ -877,8 +721,7 @@ static int search_default_file_with_ext(Process_option_func opt_handler, name, line))) goto err; - search_default_file_with_ext(opt_handler, handler_ctx, "", "", ptr, - recursion_level + 1); + search_default_file_with_ext(ctx, "", "", ptr, recursion_level + 1); } continue; @@ -886,7 +729,6 @@ static int search_default_file_with_ext(Process_option_func opt_handler, if (*ptr == '[') /* Group name */ { - found_group=1; if (!(end=(char *) strchr(++ptr,']'))) { fprintf(stderr, @@ -898,33 +740,30 @@ static int search_default_file_with_ext(Process_option_func opt_handler, for ( ; my_isspace(&my_charset_latin1,end[-1]) ; end--) ; end[0]=0; - strmake(curr_gr, ptr, MY_MIN((size_t) (end-ptr)+1, sizeof(curr_gr)-1)); - - /* signal that a new group is found */ - opt_handler(handler_ctx, curr_gr, NULL); - + strmake(curr_gr, ptr, MY_MIN((size_t) (end-ptr), sizeof(curr_gr)-1)); + found_group= find_type(curr_gr, ctx->group, FIND_TYPE_NO_PREFIX) + ? PARSE : SKIP; continue; } - if (!found_group) + switch (found_group) { + case NONE: fprintf(stderr, "error: Found option without preceding group in config file: %s at line: %d\n", name,line); goto err; + case PARSE: + break; + case SKIP: + continue; } - - + end= remove_end_comment(ptr); if ((value= strchr(ptr, '='))) end= value; for ( ; my_isspace(&my_charset_latin1,end[-1]) ; end--) ; - if (!value) - { - strmake(strmov(option,"--"),ptr, (size_t) (end-ptr)); - if (opt_handler(handler_ctx, curr_gr, option)) - goto err; - } - else + ptr= strmake(strmov(option,"--"), ptr, (size_t) (end-ptr)); + if (value) { /* Remove pre- and end space */ char *value_end; @@ -946,9 +785,7 @@ static int search_default_file_with_ext(Process_option_func opt_handler, value++; value_end--; } - ptr=strnmov(strmov(option,"--"),ptr,(size_t) (end-ptr)); *ptr++= '='; - for ( ; value != value_end; value++) { if (*value == '\\' && value != value_end-1) @@ -988,9 +825,10 @@ static int search_default_file_with_ext(Process_option_func opt_handler, *ptr++= *value; } *ptr=0; - if (opt_handler(handler_ctx, curr_gr, option)) - goto err; } + + if (add_option(ctx, option)) + goto err; } mysql_file_fclose(fp, MYF(0)); return(0); @@ -1047,7 +885,7 @@ void my_print_default_files(const char *conf_file) { const char **dirs; MEM_ROOT alloc; - init_alloc_root(&alloc, "my_print_defaults", 512, 0, MYF(0)); + init_alloc_root(key_memory_defaults, &alloc, 512, 0, MYF(0)); if ((dirs= init_default_directories(&alloc)) == NULL) { diff --git a/mysys/my_delete.c b/mysys/my_delete.c index face960c1eb..b9b0e112077 100644 --- a/mysys/my_delete.c +++ b/mysys/my_delete.c @@ -43,9 +43,12 @@ int my_delete(const char *name, myf MyFlags) err= unlink(name); #endif - if(err) + if ((MyFlags & MY_IGNORE_ENOENT) && errno == ENOENT) + DBUG_RETURN(0); + + if (err) { - my_errno=errno; + my_errno= errno; if (MyFlags & (MY_FAE+MY_WME)) my_error(EE_DELETE, MYF(ME_BELL), name, errno); } diff --git a/mysys/my_error.c b/mysys/my_error.c index 2bb9305b2fb..106e51de581 100644 --- a/mysys/my_error.c +++ b/mysys/my_error.c @@ -104,7 +104,7 @@ const char *my_get_err_msg(uint nr) @param MyFlags Flags @param ... variable list matching that error format string */ - + void my_error(uint nr, myf MyFlags, ...) { const char *format; @@ -112,13 +112,12 @@ void my_error(uint nr, myf MyFlags, ...) char ebuff[ERRMSGSIZE]; DBUG_ENTER("my_error"); DBUG_PRINT("my", ("nr: %d MyFlags: %lu errno: %d", nr, MyFlags, errno)); - if (!(format = my_get_err_msg(nr))) (void) my_snprintf(ebuff, sizeof(ebuff), "Unknown error %d", nr); else { va_start(args,MyFlags); - (void) my_vsnprintf_ex(&my_charset_utf8_general_ci, ebuff, + (void) my_vsnprintf_ex(&my_charset_utf8mb3_general_ci, ebuff, sizeof(ebuff), format, args); va_end(args); } @@ -148,7 +147,7 @@ void my_printf_error(uint error, const char *format, myf MyFlags, ...) error, MyFlags, errno, format)); va_start(args,MyFlags); - (void) my_vsnprintf_ex(&my_charset_utf8_general_ci, ebuff, + (void) my_vsnprintf_ex(&my_charset_utf8mb3_general_ci, ebuff, sizeof(ebuff), format, args); va_end(args); (*error_handler_hook)(error, ebuff, MyFlags); @@ -224,7 +223,8 @@ int my_error_register(const char** (*get_errmsgs)(int error), uint first, struct my_err_head **search_meh_pp; /* Allocate a new header structure. */ - if (! (meh_p= (struct my_err_head*) my_malloc(sizeof(struct my_err_head), + if (! (meh_p= (struct my_err_head*) my_malloc(key_memory_my_err_head, + sizeof(struct my_err_head), MYF(MY_WME)))) return 1; meh_p->get_errmsgs= get_errmsgs; diff --git a/mysys/my_file.c b/mysys/my_file.c index 17b2493962a..c2b358f5e1e 100644 --- a/mysys/my_file.c +++ b/mysys/my_file.c @@ -102,7 +102,8 @@ uint my_set_max_open_files(uint files) if (files <= MY_NFILE) DBUG_RETURN(files); - if (!(tmp= (struct st_my_file_info*) my_malloc(sizeof(*tmp) * files, + if (!(tmp= (struct st_my_file_info*) my_malloc(key_memory_my_file_info, + sizeof(*tmp) * files, MYF(MY_WME)))) DBUG_RETURN(MY_NFILE); diff --git a/mysys/my_fopen.c b/mysys/my_fopen.c index a22ffc50626..25b492846e5 100644 --- a/mysys/my_fopen.c +++ b/mysys/my_fopen.c @@ -65,7 +65,7 @@ FILE *my_fopen(const char *filename, int flags, myf MyFlags) statistic_increment(my_stream_opened,&THR_LOCK_open); DBUG_RETURN(fd); /* safeguard */ } - my_file_info[filedesc].name= (char*) my_strdup(filename,MyFlags); + my_file_info[filedesc].name= my_strdup(key_memory_my_file_info, filename, MyFlags); statistic_increment(my_stream_opened, &THR_LOCK_open); statistic_increment(my_file_total_opened, &THR_LOCK_open); my_file_info[filedesc].type= STREAM_BY_FOPEN; @@ -228,9 +228,10 @@ FILE *my_fdopen(File Filedes, const char *name, int Flags, myf MyFlags) } else { - my_file_info[Filedes].name= my_strdup(name,MyFlags); + my_file_info[Filedes].name= my_strdup(key_memory_my_file_info, + name, MyFlags); } - my_file_info[Filedes].type = STREAM_BY_FDOPEN; + my_file_info[Filedes].type= STREAM_BY_FDOPEN; } } diff --git a/mysys/my_getopt.c b/mysys/my_getopt.c index c4f83475f7b..9747ee4214e 100644 --- a/mysys/my_getopt.c +++ b/mysys/my_getopt.c @@ -1,6 +1,6 @@ /* Copyright (c) 2002, 2013, Oracle and/or its affiliates - Copyright (c) 2009, 2015, MariaDB + Copyright (c) 2009, 2020, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -15,15 +15,15 @@ along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */ -#include <my_global.h> +#include <mysys_priv.h> #include <my_default.h> #include <m_string.h> #include <stdlib.h> -#include <my_sys.h> #include <mysys_err.h> #include <my_getopt.h> #include <errno.h> +my_bool is_file_marker(const char* arg); typedef void (*init_func_p)(const struct my_option *option, void *variable, longlong value); @@ -31,7 +31,7 @@ static void default_reporter(enum loglevel level, const char *format, ...); my_error_reporter my_getopt_error_reporter= &default_reporter; static int findopt(char *, uint, const struct my_option **, const char **); -my_bool getopt_compare_strings(const char *, const char *, uint); +static my_bool getopt_compare_strings(const char *, const char *, uint); static longlong getopt_ll(char *arg, const struct my_option *optp, int *err); static ulonglong getopt_ull(char *, const struct my_option *, int *); static double getopt_double(char *arg, const struct my_option *optp, int *err); @@ -85,8 +85,9 @@ my_bool my_getopt_prefix_matching= 1; */ my_bool my_handle_options_init_variables = 1; -static void default_reporter(enum loglevel level, - const char *format, ...) +my_getopt_value my_getopt_get_addr= 0; + +static void default_reporter(enum loglevel level, const char *format, ...) { va_list args; DBUG_ENTER("default_reporter"); @@ -103,13 +104,6 @@ static void default_reporter(enum loglevel level, DBUG_VOID_RETURN; } -static my_getopt_value getopt_get_addr; - -void my_getopt_register_get_addr(my_getopt_value func_addr) -{ - getopt_get_addr= func_addr; -} - union ull_dbl { ulonglong ull; @@ -146,7 +140,7 @@ double getopt_ulonglong2double(ulonglong v) or until the end of argv. Parse options, check that the given option matches with one of the options in struct 'my_option'. Check that option was given an argument if it requires one - Call the optional 'get_one_option()' function once for each option. + Call the 'get_one_option()' function once for each option. Note that handle_options() can be invoked multiple times to parse a command line in several steps. @@ -193,19 +187,18 @@ double getopt_ulonglong2double(ulonglong v) @param [in, out] argc command line options (count) @param [in, out] argv command line options (values) @param [in] longopts descriptor of all valid options - @param [in] get_one_option optional callback function to process each option, - can be NULL. + @param [in] get_one_option callback function to process each option @return error in case of ambiguous or unknown options, 0 on success. */ -int handle_options(int *argc, char ***argv, - const struct my_option *longopts, +int handle_options(int *argc, char ***argv, const struct my_option *longopts, my_get_one_option get_one_option) { uint UNINIT_VAR(opt_found), argvpos= 0, length; my_bool end_of_options= 0, must_be_var, set_maximum_value, option_is_loose, option_is_autoset; char **pos, **pos_end, *optend, *opt_str, key_name[FN_REFLEN]; + char *filename= (char*)""; const char *UNINIT_VAR(prev_found); const struct my_option *optp; void *value; @@ -214,41 +207,36 @@ int handle_options(int *argc, char ***argv, DBUG_ENTER("handle_options"); /* handle_options() assumes arg0 (program name) always exists */ - DBUG_ASSERT(argc && *argc >= 1); - DBUG_ASSERT(argv && *argv); + DBUG_ASSERT(*argc >= 1); + DBUG_ASSERT(*argv); (*argc)--; /* Skip the program name */ (*argv)++; /* --- || ---- */ if (my_handle_options_init_variables) init_variables(longopts, init_one_value); - /* - Search for args_separator, if found, then the first part of the - arguments are loaded from configs - */ - for (pos= *argv, pos_end=pos+ *argc; pos != pos_end ; pos++) - { - if (my_getopt_is_args_separator(*pos)) - { - is_cmdline_arg= 0; - break; - } - } + is_cmdline_arg= !is_file_marker(**argv); for (pos= *argv, pos_end=pos+ *argc; pos != pos_end ; pos++) { char **first= pos; char *cur_arg= *pos; opt_found= 0; - if (!is_cmdline_arg && (my_getopt_is_args_separator(cur_arg))) + if (!is_cmdline_arg) { - is_cmdline_arg= 1; - - /* save the separator too if skip unknown options */ - if (my_getopt_skip_unknown) - (*argv)[argvpos++]= cur_arg; - else - (*argc)--; - continue; + if (is_file_marker(cur_arg)) + { + pos++; + filename= *pos; + is_cmdline_arg= *filename == 0; /* empty file name = command line */ + if (my_getopt_skip_unknown) + { + (*argv)[argvpos++]= cur_arg; + (*argv)[argvpos++]= filename; + } + else + (*argc)-= 2; + continue; + } } if (cur_arg[0] == '-' && cur_arg[1] && !end_of_options) /* must be opt */ { @@ -415,9 +403,9 @@ int handle_options(int *argc, char ***argv, DBUG_RETURN(EXIT_OPTION_DISABLED); } error= 0; - value= optp->var_type & GET_ASK_ADDR ? - (*getopt_get_addr)(key_name, (uint) strlen(key_name), optp, &error) : - optp->value; + value= optp->var_type & GET_ASK_ADDR + ? (*my_getopt_get_addr)(key_name, (uint)strlen(key_name), optp, &error) + : optp->value; if (error) DBUG_RETURN(error); @@ -460,9 +448,9 @@ int handle_options(int *argc, char ***argv, my_progname, optp->name, optend); continue; } - if (get_one_option && get_one_option(optp->id, optp, - *((my_bool*) value) ? - enabled_my_option : disabled_my_option)) + if (get_one_option(optp, *((my_bool*) value) ? + enabled_my_option : disabled_my_option, + filename)) DBUG_RETURN(EXIT_ARGUMENT_INVALID); continue; } @@ -479,12 +467,7 @@ int handle_options(int *argc, char ***argv, DBUG_RETURN(EXIT_NO_ARGUMENT_ALLOWED); } - /* - We support automatic setup only via get_one_option and only for - marked options. - */ - if (!get_one_option || - !(optp->var_type & GET_AUTO)) + if (!(optp->var_type & GET_AUTO)) { my_getopt_error_reporter(option_is_loose ? WARNING_LEVEL : ERROR_LEVEL, @@ -500,10 +483,11 @@ int handle_options(int *argc, char ***argv, } else if (optp->arg_type == REQUIRED_ARG && !optend) { - /* Check if there are more arguments after this one, - Note: options loaded from config file that requires value - should always be in the form '--option=value'. - */ + /* + Check if there are more arguments after this one, + Note: options loaded from config file that requires value + should always be in the form '--option=value'. + */ if (!is_cmdline_arg || !*++pos) { if (my_getopt_print_errors) @@ -541,7 +525,7 @@ int handle_options(int *argc, char ***argv, optp->arg_type == NO_ARG) { *((my_bool*) optp->value)= (my_bool) 1; - if (get_one_option && get_one_option(optp->id, optp, argument)) + if (get_one_option(optp, argument, filename)) DBUG_RETURN(EXIT_UNSPECIFIED_ERROR); continue; } @@ -561,7 +545,7 @@ int handle_options(int *argc, char ***argv, { if (optp->var_type == GET_BOOL) *((my_bool*) optp->value)= (my_bool) 1; - if (get_one_option && get_one_option(optp->id, optp, argument)) + if (get_one_option(optp, argument, filename)) DBUG_RETURN(EXIT_UNSPECIFIED_ERROR); continue; } @@ -582,7 +566,7 @@ int handle_options(int *argc, char ***argv, if ((error= setval(optp, optp->value, argument, set_maximum_value))) DBUG_RETURN(error); - if (get_one_option && get_one_option(optp->id, optp, argument)) + if (get_one_option(optp, argument, filename)) DBUG_RETURN(EXIT_UNSPECIFIED_ERROR); break; } @@ -629,7 +613,7 @@ int handle_options(int *argc, char ***argv, ((error= setval(optp, value, argument, set_maximum_value))) && !option_is_loose) DBUG_RETURN(error); - if (get_one_option && get_one_option(optp->id, optp, argument)) + if (get_one_option(optp, argument, filename)) DBUG_RETURN(EXIT_UNSPECIFIED_ERROR); (*argc)--; /* option handled (long), decrease argument count */ @@ -776,7 +760,8 @@ static int setval(const struct my_option *opts, void *value, char *argument, break; case GET_STR_ALLOC: my_free(*((char**) value)); - if (!(*((char**) value)= my_strdup(argument == enabled_my_option ? "" : + if (!(*((char**) value)= my_strdup(key_memory_defaults, + argument == enabled_my_option ? "" : argument, MYF(MY_WME)))) { res= EXIT_OUT_OF_MEMORY; @@ -1365,7 +1350,7 @@ static void init_one_value(const struct my_option *option, void *variable, { char **pstr= (char **) variable; my_free(*pstr); - *pstr= my_strdup((char*) (intptr) value, MYF(MY_WME)); + *pstr= my_strdup(key_memory_defaults, (char*) (intptr) value, MYF(MY_WME)); } break; default: /* dummy default to avoid compiler warnings */ @@ -1437,8 +1422,8 @@ static void init_variables(const struct my_option *options, */ if (options->u_max_value) func_init_one_value(options, options->u_max_value, options->max_value); - value= (options->var_type & GET_ASK_ADDR ? - (*getopt_get_addr)("", 0, options, 0) : options->value); + value= options->var_type & GET_ASK_ADDR ? + (*my_getopt_get_addr)("", 0, options, 0) : options->value; if (value) func_init_one_value(options, value, options->def_value); } @@ -1642,8 +1627,8 @@ void my_print_variables(const struct my_option *options) for (optp= options; optp->name; optp++) { - void *value= (optp->var_type & GET_ASK_ADDR ? - (*getopt_get_addr)("", 0, optp, 0) : optp->value); + void *value= optp->var_type & GET_ASK_ADDR ? + (*my_getopt_get_addr)("", 0, optp, 0) : optp->value; if (value) { length= print_name(optp); diff --git a/mysys/my_getsystime.c b/mysys/my_getsystime.c index 18218c2e8de..ce8a9ca786f 100644 --- a/mysys/my_getsystime.c +++ b/mysys/my_getsystime.c @@ -21,10 +21,8 @@ #ifdef _WIN32 #define OFFSET_TO_EPOC 116444736000000000LL static ulonglong query_performance_frequency; -typedef void (WINAPI* get_system_time_as_filetime_t)(LPFILETIME); -static get_system_time_as_filetime_t - my_GetSystemTimePreciseAsFileTime= GetSystemTimeAsFileTime; #endif + #ifdef HAVE_LINUX_UNISTD_H #include <linux/unistd.h> #endif @@ -57,20 +55,12 @@ ulonglong my_interval_timer() #elif defined(HAVE_GETHRTIME) return gethrtime(); #elif defined(_WIN32) + DBUG_ASSERT(query_performance_frequency); LARGE_INTEGER t_cnt; - if (query_performance_frequency) - { - QueryPerformanceCounter(&t_cnt); - return (t_cnt.QuadPart / query_performance_frequency * 1000000000ULL) + + QueryPerformanceCounter(&t_cnt); + return (t_cnt.QuadPart / query_performance_frequency * 1000000000ULL) + ((t_cnt.QuadPart % query_performance_frequency) * 1000000000ULL / query_performance_frequency); - } - else - { - ulonglong newtime; - my_GetSystemTimePreciseAsFileTime((FILETIME*)&newtime); - return newtime*100ULL; - } #else /* TODO: check for other possibilities for hi-res timestamping */ struct timeval tv; @@ -87,7 +77,7 @@ my_hrtime_t my_hrtime() my_hrtime_t hrtime; #if defined(_WIN32) ulonglong newtime; - my_GetSystemTimePreciseAsFileTime((FILETIME*)&newtime); + GetSystemTimePreciseAsFileTime((FILETIME*)&newtime); hrtime.val= (newtime - OFFSET_TO_EPOC)/10; #elif defined(HAVE_CLOCK_GETTIME) struct timespec tp; @@ -129,14 +119,8 @@ void my_time_init() #ifdef _WIN32 compile_time_assert(sizeof(LARGE_INTEGER) == sizeof(query_performance_frequency)); - if (QueryPerformanceFrequency((LARGE_INTEGER *)&query_performance_frequency) == 0) - query_performance_frequency= 0; - - get_system_time_as_filetime_t f= (get_system_time_as_filetime_t) - GetProcAddress(GetModuleHandle("kernel32"), - "GetSystemTimePreciseAsFileTime"); - if (f) - my_GetSystemTimePreciseAsFileTime= f; + QueryPerformanceFrequency((LARGE_INTEGER *)&query_performance_frequency); + DBUG_ASSERT(query_performance_frequency); #endif } diff --git a/mysys/my_init.c b/mysys/my_init.c index d8fb2003052..2b420da03be 100644 --- a/mysys/my_init.c +++ b/mysys/my_init.c @@ -22,14 +22,13 @@ #include <m_ctype.h> #include <signal.h> #include <mysql/psi/mysql_stage.h> -#ifdef __WIN__ +#ifdef _WIN32 #ifdef _MSC_VER #include <locale.h> #include <crtdbg.h> /* WSAStartup needs winsock library*/ #pragma comment(lib, "ws2_32") #endif -my_bool have_tcpip=0; static void my_win_init(void); static my_bool win32_init_tcp_ip(); #else @@ -60,7 +59,6 @@ static ulong atoi_octal(const char *str) MYSQL_FILE *mysql_stdin= NULL; static MYSQL_FILE instrumented_stdin; - /** Initialize my_sys functions, resources and variables @@ -119,8 +117,9 @@ my_bool my_init(void) my_time_init(); my_win_init(); DBUG_PRINT("exit", ("home: '%s'", home_dir)); -#ifdef __WIN__ - win32_init_tcp_ip(); +#ifdef _WIN32 + if (win32_init_tcp_ip()) + DBUG_RETURN(1); #endif #ifdef CHECK_UNLIKELY init_my_likely(); @@ -218,7 +217,7 @@ Voluntary context switches %ld, Involuntary context switches %ld\n", rus.ru_msgsnd, rus.ru_msgrcv, rus.ru_nsignals, rus.ru_nvcsw, rus.ru_nivcsw); #endif -#if defined(__WIN__) && defined(_MSC_VER) +#if defined(_MSC_VER) _CrtSetReportMode( _CRT_WARN, _CRTDBG_MODE_FILE ); _CrtSetReportFile( _CRT_WARN, _CRTDBG_FILE_STDERR ); _CrtSetReportMode( _CRT_ERROR, _CRTDBG_MODE_FILE ); @@ -245,10 +244,9 @@ Voluntary context switches %ld, Involuntary context switches %ld\n", (FILE *) 0); #endif /* defined(SAFE_MUTEX) */ -#ifdef __WIN__ - if (have_tcpip) - WSACleanup(); -#endif /* __WIN__ */ +#ifdef _WIN32 + WSACleanup(); +#endif /* At very last, delete mysys key, it is used everywhere including DBUG */ pthread_key_delete(THR_KEY_mysys); @@ -263,16 +261,14 @@ void my_debug_put_break_here(void) } #endif -#ifdef __WIN__ +#ifdef _WIN32 /* my_parameter_handler Invalid parameter handler we will use instead of the one "baked" - into the CRT for MSC v8. This one just prints out what invalid - parameter was encountered. By providing this routine, routines like - lseek will return -1 when we expect them to instead of crash. + into the CRT. */ void my_parameter_handler(const wchar_t * expression, const wchar_t * function, @@ -311,78 +307,14 @@ int handle_rtc_failure(int err_type, const char *file, int line, #pragma runtime_checks("", restore) #endif -/* - Open HKEY_LOCAL_MACHINE\SOFTWARE\MySQL and set any strings found - there as environment variables -*/ -static void win_init_registry(void) -{ - HKEY key_handle; - - if (RegOpenKeyEx(HKEY_LOCAL_MACHINE, (LPCTSTR)"SOFTWARE\\MySQL", - 0, KEY_READ, &key_handle) == ERROR_SUCCESS) - { - LONG ret; - DWORD index= 0; - DWORD type; - char key_name[256], key_data[1024]; - DWORD key_name_len= sizeof(key_name) - 1; - DWORD key_data_len= sizeof(key_data) - 1; - - while ((ret= RegEnumValue(key_handle, index++, - key_name, &key_name_len, - NULL, &type, (LPBYTE)&key_data, - &key_data_len)) != ERROR_NO_MORE_ITEMS) - { - char env_string[sizeof(key_name) + sizeof(key_data) + 2]; - - if (ret == ERROR_MORE_DATA) - { - /* Registry value larger than 'key_data', skip it */ - DBUG_PRINT("error", ("Skipped registry value that was too large")); - } - else if (ret == ERROR_SUCCESS) - { - if (type == REG_SZ) - { - strxmov(env_string, key_name, "=", key_data, NullS); - - /* variable for putenv must be allocated ! */ - putenv(strdup(env_string)) ; - } - } - else - { - /* Unhandled error, break out of loop */ - break; - } - - key_name_len= sizeof(key_name) - 1; - key_data_len= sizeof(key_data) - 1; - } - - RegCloseKey(key_handle); - } -} - static void my_win_init(void) { DBUG_ENTER("my_win_init"); #if defined(_MSC_VER) -#if _MSC_VER < 1300 - /* - Clear the OS system variable TZ and avoid the 100% CPU usage - Only for old versions of Visual C++ - */ - _putenv("TZ="); -#endif -#if _MSC_VER >= 1400 - /* this is required to make crt functions return -1 appropriately */ _set_invalid_parameter_handler(my_parameter_handler); #endif -#endif #ifdef __MSVC_RUNTIME_CHECKS /* @@ -394,75 +326,22 @@ static void my_win_init(void) _tzset(); - win_init_registry(); - DBUG_VOID_RETURN; } -/*------------------------------------------------------------------ - Name: CheckForTcpip| Desc: checks if tcpip has been installed on system - According to Microsoft Developers documentation the first registry - entry should be enough to check if TCP/IP is installed, but as expected - this doesn't work on all Win32 machines :( -------------------------------------------------------------------*/ - -#define TCPIPKEY "SYSTEM\\CurrentControlSet\\Services\\Tcpip\\Parameters" -#define WINSOCK2KEY "SYSTEM\\CurrentControlSet\\Services\\Winsock2\\Parameters" -#define WINSOCKKEY "SYSTEM\\CurrentControlSet\\Services\\Winsock\\Parameters" - -static my_bool win32_have_tcpip(void) -{ - HKEY hTcpipRegKey; - if (RegOpenKeyEx ( HKEY_LOCAL_MACHINE, TCPIPKEY, 0, KEY_READ, - &hTcpipRegKey) != ERROR_SUCCESS) - { - if (RegOpenKeyEx ( HKEY_LOCAL_MACHINE, WINSOCK2KEY, 0, KEY_READ, - &hTcpipRegKey) != ERROR_SUCCESS) - { - if (RegOpenKeyEx ( HKEY_LOCAL_MACHINE, WINSOCKKEY, 0, KEY_READ, - &hTcpipRegKey) != ERROR_SUCCESS) - if (!getenv("HAVE_TCPIP") || have_tcpip) /* Provide a workaround */ - return (FALSE); - } - } - RegCloseKey ( hTcpipRegKey); - return (TRUE); -} - - static my_bool win32_init_tcp_ip() { - if (win32_have_tcpip()) + WORD wVersionRequested = MAKEWORD( 2, 2 ); + WSADATA wsaData; + if (WSAStartup(wVersionRequested, &wsaData)) { - WORD wVersionRequested = MAKEWORD( 2, 2 ); - WSADATA wsaData; - /* Be a good citizen: maybe another lib has already initialised - sockets, so don't clobber them unless necessary */ - if (WSAStartup( wVersionRequested, &wsaData )) - { - /* Load failed, maybe because of previously loaded - incompatible version; try again */ - WSACleanup( ); - if (!WSAStartup( wVersionRequested, &wsaData )) - have_tcpip=1; - } - else - { - if (wsaData.wVersion != wVersionRequested) - { - /* Version is no good, try again */ - WSACleanup( ); - if (!WSAStartup( wVersionRequested, &wsaData )) - have_tcpip=1; - } - else - have_tcpip=1; - } + fprintf(stderr, "WSAStartup() failed with error: %d\n", WSAGetLastError()); + return 1; } return(0); } -#endif /* __WIN__ */ +#endif /* _WIN32 */ PSI_stage_info stage_waiting_for_table_level_lock= {0, "Waiting for table level lock", 0}; @@ -550,16 +429,10 @@ static PSI_thread_info all_mysys_threads[]= }; -#ifdef HUGETLB_USE_PROC_MEMINFO -PSI_file_key key_file_proc_meminfo; -#endif /* HUGETLB_USE_PROC_MEMINFO */ PSI_file_key key_file_charset, key_file_cnf; static PSI_file_info all_mysys_files[]= { -#ifdef HUGETLB_USE_PROC_MEMINFO - { &key_file_proc_meminfo, "proc_meminfo", 0}, -#endif /* HUGETLB_USE_PROC_MEMINFO */ { &key_file_charset, "charset", 0}, { &key_file_cnf, "cnf", 0} }; diff --git a/mysys/my_largepage.c b/mysys/my_largepage.c index 33d0843cd58..0fdc4e17a26 100644 --- a/mysys/my_largepage.c +++ b/mysys/my_largepage.c @@ -1,4 +1,5 @@ /* Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved. + Copyright (c) 2019, 2020 IBM. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -14,153 +15,446 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ #include "mysys_priv.h" +#include <mysys_err.h> -#ifdef HAVE_LINUX_LARGE_PAGES +#ifdef __linux__ +#include <dirent.h> +#endif +#if defined(__linux__) || defined(MAP_ALIGNED) +#include "my_bit.h" +#endif +#ifdef HAVE_LINUX_MMAN_H +#include <linux/mman.h> +#endif + +#ifdef HAVE_SOLARIS_LARGE_PAGES +#if defined(__sun__) && defined(__GNUC__) && defined(__cplusplus) \ + && defined(_XOPEN_SOURCE) +/* memcntl exist within sys/mman.h, but under-defines what is need to use it */ +extern int memcntl(caddr_t, size_t, int, caddr_t, int, int); +#endif /* __sun__ ... */ +#endif /* HAVE_SOLARIS_LARGE_PAGES */ -#ifdef HAVE_SYS_IPC_H -#include <sys/ipc.h> +#if defined(_WIN32) +static size_t my_large_page_size; +#define HAVE_LARGE_PAGES +#elif defined(HAVE_MMAP) +#define HAVE_LARGE_PAGES #endif -#ifdef HAVE_SYS_SHM_H -#include <sys/shm.h> +#ifdef HAVE_LARGE_PAGES +static my_bool my_use_large_pages= 0; +#else +#define my_use_large_pages 0 #endif -static uint my_get_large_page_size_int(void); -static uchar* my_large_malloc_int(size_t size, myf my_flags); -static my_bool my_large_free_int(uchar* ptr); +#if defined(HAVE_GETPAGESIZES) || defined(__linux__) +/* Descending sort */ + +static int size_t_cmp(const void *a, const void *b) +{ + const size_t ia= *(const size_t *) a; + const size_t ib= *(const size_t *) b; + if (ib > ia) + { + return 1; + } + else if (ib < ia) + { + return -1; + } + return 0; +} +#endif /* defined(HAVE_GETPAGESIZES) || defined(__linux__) */ + -/* Gets the size of large pages from the OS */ +#if defined(__linux__) || defined(HAVE_GETPAGESIZES) +#define my_large_page_sizes_length 8 +static size_t my_large_page_sizes[my_large_page_sizes_length]; +#endif -uint my_get_large_page_size(void) +/** + Linux-specific function to determine the sizes of large pages +*/ +#ifdef __linux__ +static inline my_bool my_is_2pow(size_t n) { return !((n) & ((n) - 1)); } + +static void my_get_large_page_sizes(size_t sizes[my_large_page_sizes_length]) { - uint size; - DBUG_ENTER("my_get_large_page_size"); - - if (!(size = my_get_large_page_size_int())) - fprintf(stderr, "Warning: Failed to determine large page size\n"); + DIR *dirp; + struct dirent *r; + int i= 0; + DBUG_ENTER("my_get_large_page_sizes"); - DBUG_RETURN(size); + dirp= opendir("/sys/kernel/mm/hugepages"); + if (dirp == NULL) + { + my_error(EE_DIR, MYF(ME_BELL), "/sys/kernel/mm/hugepages", errno); + } + else + { + while (i < my_large_page_sizes_length && (r= readdir(dirp))) + { + if (strncmp("hugepages-", r->d_name, 10) == 0) + { + sizes[i]= strtoull(r->d_name + 10, NULL, 10) * 1024ULL; + if (!my_is_2pow(sizes[i])) + { + my_printf_error(0, + "non-power of 2 large page size (%zu) found," + " skipping", MYF(ME_NOTE | ME_ERROR_LOG_ONLY), + sizes[i]); + sizes[i]= 0; + continue; + } + ++i; + } + } + if (closedir(dirp)) + { + my_error(EE_BADCLOSE, MYF(ME_BELL), "/sys/kernel/mm/hugepages", errno); + } + qsort(sizes, i, sizeof(size_t), size_t_cmp); + } + DBUG_VOID_RETURN; } -/* - General large pages allocator. - Tries to allocate memory from large pages pool and falls back to - my_malloc_lock() in case of failure -*/ -uchar* my_large_malloc(size_t size, myf my_flags) +#elif defined(HAVE_GETPAGESIZES) +static void my_get_large_page_sizes(size_t sizes[my_large_page_sizes_length]) { - uchar* ptr; - DBUG_ENTER("my_large_malloc"); - - if (my_use_large_pages && my_large_page_size) + int nelem; + + nelem= getpagesizes(NULL, 0); + + assert(nelem <= my_large_page_sizes_length); + getpagesizes(sizes, my_large_page_sizes_length); + qsort(sizes, nelem, sizeof(size_t), size_t_cmp); + if (nelem < my_large_page_sizes_length) { - if ((ptr = my_large_malloc_int(size, my_flags)) != NULL) - DBUG_RETURN(ptr); - if (my_flags & MY_WME) - fprintf(stderr, "Warning: Using conventional memory pool\n"); + sizes[nelem]= 0; } - - DBUG_RETURN(my_malloc_lock(size, my_flags)); } -/* - General large pages deallocator. - Tries to deallocate memory as if it was from large pages pool and falls back - to my_free_lock() in case of failure - */ -void my_large_free(uchar* ptr) +#elif defined(_WIN32) +#define my_large_page_sizes_length 0 +#define my_get_large_page_sizes(A) do {} while(0) + +#else +#define my_large_page_sizes_length 1 +static size_t my_large_page_sizes[my_large_page_sizes_length]; +static void my_get_large_page_sizes(size_t sizes[]) { - DBUG_ENTER("my_large_free"); - - /* - my_large_free_int() can only fail if ptr was not allocated with - my_large_malloc_int(), i.e. my_malloc_lock() was used so we should free it - with my_free_lock() - */ - if (!my_use_large_pages || !my_large_page_size || !my_large_free_int(ptr)) - my_free_lock(ptr); + sizes[0]= my_getpagesize(); +} +#endif - DBUG_VOID_RETURN; + +/** + Returns the next large page size smaller or equal to the passed in size. + + The search starts at my_large_page_sizes[*start]. + + Assumes my_get_large_page_sizes(my_large_page_sizes) has been called before + use. + + For first use, have *start=0. There is no need to increment *start. + + @param[in] sz size to be searched for. + @param[in,out] start ptr to int representing offset in my_large_page_sizes to + start from. + *start is updated during search and can be used to search again if 0 isn't + returned. + + @returns the next size found. *start will be incremented to the next potential + size. + @retval a large page size that is valid on this system or 0 if no large page + size possible. +*/ +#if defined(HAVE_MMAP) && !defined(_WIN32) +static size_t my_next_large_page_size(size_t sz, int *start) +{ + DBUG_ENTER("my_next_large_page_size"); + + while (*start < my_large_page_sizes_length && my_large_page_sizes[*start] > 0) + { + size_t cur= *start; + (*start)++; + if (my_large_page_sizes[cur] <= sz) + { + DBUG_RETURN(my_large_page_sizes[cur]); + } + } + DBUG_RETURN(0); } +#endif /* defined(MMAP) || !defined(_WIN32) */ -#ifdef HUGETLB_USE_PROC_MEMINFO -/* Linux-specific function to determine the size of large pages */ -uint my_get_large_page_size_int(void) +int my_init_large_pages(my_bool super_large_pages) { - MYSQL_FILE *f; - uint size = 0; - char buf[256]; - DBUG_ENTER("my_get_large_page_size_int"); +#ifdef _WIN32 + if (!my_obtain_privilege(SE_LOCK_MEMORY_NAME)) + { + my_printf_error(EE_PERM_LOCK_MEMORY, + "Lock Pages in memory access rights required for use with" + " large-pages, see https://mariadb.com/kb/en/library/" + "mariadb-memory-allocation/#huge-pages", MYF(MY_WME)); + } + my_large_page_size= GetLargePageMinimum(); +#endif - if (!(f= mysql_file_fopen(key_file_proc_meminfo, "/proc/meminfo", - O_RDONLY, MYF(MY_WME)))) - goto finish; + my_use_large_pages= 1; + my_get_large_page_sizes(my_large_page_sizes); - while (mysql_file_fgets(buf, sizeof(buf), f)) - if (sscanf(buf, "Hugepagesize: %u kB", &size)) - break; +#ifndef HAVE_LARGE_PAGES + my_printf_error(EE_OUTOFMEMORY, "No large page support on this platform", + MYF(MY_WME)); +#endif + +#ifdef HAVE_SOLARIS_LARGE_PAGES + /* + tell the kernel that we want to use 4/256MB page for heap storage + and also for the stack. We use 4 MByte as default and if the + super-large-page is set we increase it to 256 MByte. 256 MByte + is for server installations with GBytes of RAM memory where + the MySQL Server will have page caches and other memory regions + measured in a number of GBytes. + We use as big pages as possible which isn't bigger than the above + desired page sizes. + */ + int nelem= 0; + size_t max_desired_page_size= (super_large_pages ? 256 : 4) * 1024 * 1024; + size_t max_page_size= my_next_large_page_size(max_desired_page_size, &nelem); - mysql_file_fclose(f, MYF(MY_WME)); - -finish: - DBUG_RETURN(size * 1024); + if (max_page_size > 0) + { + struct memcntl_mha mpss; + + mpss.mha_cmd= MHA_MAPSIZE_BSSBRK; + mpss.mha_pagesize= max_page_size; + mpss.mha_flags= 0; + if (memcntl(NULL, 0, MC_HAT_ADVISE, (caddr_t) &mpss, 0, 0)) + { + my_error(EE_MEMCNTL, MYF(ME_WARNING | ME_ERROR_LOG_ONLY), "MC_HAT_ADVISE", + "MHA_MAPSIZE_BSSBRK"); + } + mpss.mha_cmd= MHA_MAPSIZE_STACK; + if (memcntl(NULL, 0, MC_HAT_ADVISE, (caddr_t) &mpss, 0, 0)) + { + my_error(EE_MEMCNTL, MYF(ME_WARNING | ME_ERROR_LOG_ONLY), "MC_HAT_ADVISE", + "MHA_MAPSIZE_STACK"); + } + } +#endif /* HAVE_SOLARIS_LARGE_PAGES */ + return 0; } -#endif /* HUGETLB_USE_PROC_MEMINFO */ -#if HAVE_DECL_SHM_HUGETLB -/* Linux-specific large pages allocator */ - -uchar* my_large_malloc_int(size_t size, myf my_flags) + +#if defined(HAVE_MMAP) && !defined(_WIN32) +/* Solaris for example has only MAP_ANON, FreeBSD has MAP_ANONYMOUS and +MAP_ANON but MAP_ANONYMOUS is marked "for compatibility" */ +#if defined(MAP_ANONYMOUS) +#define OS_MAP_ANON MAP_ANONYMOUS +#elif defined(MAP_ANON) +#define OS_MAP_ANON MAP_ANON +#else +#error unsupported mmap - no MAP_ANON{YMOUS} +#endif +#endif /* HAVE_MMAP && !_WIN32 */ + +/** + General large pages allocator. + Tries to allocate memory from large pages pool and falls back to + my_malloc_lock() in case of failure. + Every implementation returns a zero filled buffer here. +*/ +uchar *my_large_malloc(size_t *size, myf my_flags) { - int shmid; - uchar* ptr; - struct shmid_ds buf; - DBUG_ENTER("my_large_malloc_int"); + uchar *ptr= NULL; - /* Align block size to my_large_page_size */ - size= MY_ALIGN(size, (size_t) my_large_page_size); - - shmid = shmget(IPC_PRIVATE, size, SHM_HUGETLB | SHM_R | SHM_W); - if (shmid < 0) +#ifdef _WIN32 + DWORD alloc_type= MEM_COMMIT | MEM_RESERVE; + size_t orig_size= *size; + DBUG_ENTER("my_large_malloc"); + + if (my_use_large_pages) + { + alloc_type|= MEM_LARGE_PAGES; + /* Align block size to my_large_page_size */ + *size= MY_ALIGN(*size, (size_t) my_large_page_size); + } + ptr= VirtualAlloc(NULL, *size, alloc_type, PAGE_READWRITE); + if (!ptr) { if (my_flags & MY_WME) - fprintf(stderr, - "Warning: Failed to allocate %lu bytes from HugeTLB memory." - " errno %d\n", (ulong) size, errno); - - DBUG_RETURN(NULL); + { + if (my_use_large_pages) + { + my_printf_error(EE_OUTOFMEMORY, + "Couldn't allocate %zu bytes (MEM_LARGE_PAGES page " + "size %zu); Windows error %lu", + MYF(ME_WARNING | ME_ERROR_LOG_ONLY), *size, + my_large_page_size, GetLastError()); + } + else + { + my_error(EE_OUTOFMEMORY, MYF(ME_BELL+ME_ERROR_LOG), *size); + } + } + if (my_use_large_pages) + { + *size= orig_size; + ptr= VirtualAlloc(NULL, *size, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE); + if (!ptr && my_flags & MY_WME) + { + my_error(EE_OUTOFMEMORY, MYF(ME_BELL+ME_ERROR_LOG), *size); + } + } } +#elif defined(HAVE_MMAP) + int mapflag; + int page_i= 0; + size_t large_page_size= 0; + size_t aligned_size= *size; + DBUG_ENTER("my_large_malloc"); - ptr = (uchar*) shmat(shmid, NULL, 0); - if (ptr == (uchar *) -1) + while (1) { - if (my_flags& MY_WME) - fprintf(stderr, "Warning: Failed to attach shared memory segment," - " errno %d\n", errno); - shmctl(shmid, IPC_RMID, &buf); + mapflag= MAP_PRIVATE | OS_MAP_ANON; + if (my_use_large_pages) + { + large_page_size= my_next_large_page_size(*size, &page_i); + /* this might be 0, in which case we do a standard mmap */ + if (large_page_size) + { +#if defined(MAP_HUGETLB) /* linux 2.6.32 */ + mapflag|= MAP_HUGETLB; +#if defined(MAP_HUGE_SHIFT) /* Linux-3.8+ */ + mapflag|= my_bit_log2_size_t(large_page_size) << MAP_HUGE_SHIFT; +#else +# warning "No explicit large page (HUGETLB pages) support in Linux < 3.8" +#endif +#elif defined(MAP_ALIGNED) + mapflag|= MAP_ALIGNED(my_bit_log2_size_t(large_page_size)); +#if defined(MAP_ALIGNED_SUPER) + mapflag|= MAP_ALIGNED_SUPER; +#endif +#endif + aligned_size= MY_ALIGN(*size, (size_t) large_page_size); + } + else + { + aligned_size= *size; + } + } + ptr= mmap(NULL, aligned_size, PROT_READ | PROT_WRITE, mapflag, -1, 0); + if (ptr == (void*) -1) + { + ptr= NULL; + if (my_flags & MY_WME) + { + if (large_page_size) + { + my_printf_error(EE_OUTOFMEMORY, + "Couldn't allocate %zu bytes (Large/HugeTLB memory " + "page size %zu); errno %u; continuing to smaller size", + MYF(ME_WARNING | ME_ERROR_LOG_ONLY), + aligned_size, large_page_size, errno); + } + else + { + my_error(EE_OUTOFMEMORY, MYF(ME_BELL+ME_ERROR_LOG), aligned_size); + } + } + /* try next smaller memory size */ + if (large_page_size && errno == ENOMEM) + continue; - DBUG_RETURN(NULL); + /* other errors are more serious */ + break; + } + else /* success */ + { + if (large_page_size) + { + /* + we do need to record the adjustment so that munmap gets called with + the right size. This is only the case for HUGETLB pages. + */ + *size= aligned_size; + } + break; + } + if (large_page_size == 0) + { + break; /* no more options to try */ + } } +#else + DBUG_RETURN(my_malloc_lock(*size, my_flags)); +#endif /* defined(HAVE_MMAP) */ - /* - Remove the shared memory segment so that it will be automatically freed - after memory is detached or process exits - */ - shmctl(shmid, IPC_RMID, &buf); + if (ptr != NULL) + { + MEM_MAKE_DEFINED(ptr, *size); + } DBUG_RETURN(ptr); } -/* Linux-specific large pages deallocator */ -my_bool my_large_free_int(uchar *ptr) +/** + General large pages deallocator. + Tries to deallocate memory as if it was from large pages pool and falls back + to my_free_lock() in case of failure +*/ +void my_large_free(void *ptr, size_t size) { - DBUG_ENTER("my_large_free_int"); - DBUG_RETURN(shmdt(ptr) == 0); -} -#endif /* HAVE_DECL_SHM_HUGETLB */ + DBUG_ENTER("my_large_free"); + + /* + The following implementations can only fail if ptr was not allocated with + my_large_malloc(), i.e. my_malloc_lock() was used so we should free it + with my_free_lock() + + For ASAN, we need to explicitly unpoison this memory region because the OS + may reuse that memory for some TLS or stack variable. It will remain + poisoned if it was explicitly poisioned before release. If this happens, + we'll have hard to debug false positives like in MDEV-21239. + For valgrind, we mark it as UNDEFINED rather than NOACCESS because of the + implict reuse possiblility. + */ +#if defined(HAVE_MMAP) && !defined(_WIN32) + if (munmap(ptr, size)) + { + my_error(EE_BADMEMORYRELEASE, MYF(ME_ERROR_LOG_ONLY), ptr, size, errno); + } +# if !__has_feature(memory_sanitizer) + else + { + MEM_MAKE_ADDRESSABLE(ptr, size); + } +# endif +#elif defined(_WIN32) + /* + When RELEASE memory, the size parameter must be 0. + Do not use MEM_RELEASE with MEM_DECOMMIT. + */ + if (ptr && !VirtualFree(ptr, 0, MEM_RELEASE)) + { + my_error(EE_BADMEMORYRELEASE, MYF(ME_ERROR_LOG_ONLY), ptr, size, + GetLastError()); + } +# if !__has_feature(memory_sanitizer) + else + { + MEM_MAKE_ADDRESSABLE(ptr, size); + } +# endif +#else + my_free_lock(ptr); +#endif -#endif /* HAVE_LINUX_LARGE_PAGES */ + DBUG_VOID_RETURN; +} diff --git a/mysys/my_lib.c b/mysys/my_lib.c index b61a02449bc..ca50699b4c3 100644 --- a/mysys/my_lib.c +++ b/mysys/my_lib.c @@ -128,16 +128,17 @@ MY_DIR *my_dir(const char *path, myf MyFlags) goto err_open; } - if (!(dirh= my_malloc(sizeof(*dirh), MyFlags | MY_ZEROFILL))) + if (!(dirh= my_malloc(key_memory_MY_DIR, sizeof(*dirh), + MYF(MyFlags | MY_ZEROFILL)))) goto err_alloc; - if (my_init_dynamic_array(&dirh->array, sizeof(FILEINFO), + if (my_init_dynamic_array(key_memory_MY_DIR, &dirh->array, sizeof(FILEINFO), ENTRIES_START_SIZE, ENTRIES_INCREMENT, MYF(MyFlags))) goto error; - init_alloc_root(&dirh->root, "dir", NAMES_START_SIZE, NAMES_START_SIZE, - MYF(MyFlags)); + init_alloc_root(key_memory_MY_DIR, &dirh->root, NAMES_START_SIZE, + NAMES_START_SIZE, MYF(MyFlags)); dp= (struct dirent*) dirent_tmp; @@ -231,15 +232,15 @@ MY_DIR *my_dir(const char *path, myf MyFlags) tmp_file[2]='*'; tmp_file[3]='\0'; - if (!(dirh= my_malloc(sizeof(*dirh), MyFlags | MY_ZEROFILL))) + if (!(dirh= my_malloc(PSI_INSTRUMENT_ME, sizeof(*dirh), MyFlags | MY_ZEROFILL))) goto error; - if (my_init_dynamic_array(&dirh->array, sizeof(FILEINFO), + if (my_init_dynamic_array(PSI_INSTRUMENT_ME, &dirh->array, sizeof(FILEINFO), ENTRIES_START_SIZE, ENTRIES_INCREMENT, MYF(MyFlags))) goto error; - init_alloc_root(&dirh->root, "dir", NAMES_START_SIZE, NAMES_START_SIZE, + init_alloc_root(PSI_INSTRUMENT_ME, &dirh->root, NAMES_START_SIZE, NAMES_START_SIZE, MYF(MyFlags)); if ((handle=_findfirst(tmp_path,&find)) == -1L) @@ -345,7 +346,8 @@ MY_STAT *my_stat(const char *path, MY_STAT *stat_area, myf my_flags) stat_area, my_flags)); if ((m_used= (stat_area == NULL))) - if (!(stat_area= (MY_STAT *) my_malloc(sizeof(MY_STAT), my_flags))) + if (!(stat_area= (MY_STAT *) my_malloc(key_memory_MY_STAT, sizeof(MY_STAT), + my_flags))) goto error; #ifndef _WIN32 if (! stat((char *) path, (struct stat *) stat_area) ) diff --git a/mysys/my_likely.c b/mysys/my_likely.c index c6fca5b7146..2ae587be18d 100644 --- a/mysys/my_likely.c +++ b/mysys/my_likely.c @@ -48,10 +48,9 @@ HASH likely_hash; void init_my_likely() { /* Allocate big enough to avoid malloc calls */ - my_hash_init2(&likely_hash, 10000, &my_charset_bin, - 1024, 0, 0, - (my_hash_get_key) get_likely_key, 0, - free, HASH_UNIQUE); + my_hash_init2(PSI_NOT_INSTRUMENTED, &likely_hash, 10000, &my_charset_bin, + 1024, 0, 0, (my_hash_get_key) get_likely_key, 0, free, + HASH_UNIQUE); likely_inited= 1; pthread_mutex_init(&likely_mutex, MY_MUTEX_INIT_FAST); } diff --git a/mysys/my_malloc.c b/mysys/my_malloc.c index d54270a7f20..befdcb0e5c3 100644 --- a/mysys/my_malloc.c +++ b/mysys/my_malloc.c @@ -19,35 +19,17 @@ #include "mysys_err.h" #include <m_string.h> -/* If we have our own safemalloc (for debugging) */ -#if defined(SAFEMALLOC) -#define MALLOC_SIZE_AND_FLAG(p,b) sf_malloc_usable_size(p,b) -#define MALLOC_PREFIX_SIZE 0 -#define MALLOC_STORE_SIZE(a,b,c,d) -#define MALLOC_FIX_POINTER_FOR_FREE(a) a -#else -/* - * We use double as prefix size as this guarantees the correct - * alignment on all platforms and will optimize things for - * memcpy(), memcmp() etc. - */ -#define MALLOC_PREFIX_SIZE (sizeof(double)) -#define MALLOC_SIZE(p) (*(size_t*) ((char*)(p) - MALLOC_PREFIX_SIZE)) -#define MALLOC_STORE_SIZE(p, type_of_p, size, flag) \ -{\ - *(size_t*) p= (size) | (flag); \ - (p)= (type_of_p) (((char*) (p)) + MALLOC_PREFIX_SIZE); \ -} -static inline size_t malloc_size_and_flag(void *p, my_bool *is_thread_specific) +struct my_memory_header { - size_t size= MALLOC_SIZE(p); - *is_thread_specific= (size & 1); - return size & ~ (ulonglong) 1; -} -#define MALLOC_SIZE_AND_FLAG(p,b) malloc_size_and_flag(p, b); -#define MALLOC_FIX_POINTER_FOR_FREE(p) (((char*) (p)) - MALLOC_PREFIX_SIZE) -#endif /* SAFEMALLOC */ + PSI_thread *m_owner; + size_t m_size; + PSI_memory_key m_key; +}; +typedef struct my_memory_header my_memory_header; +#define HEADER_SIZE 24 +#define USER_TO_HEADER(P) ((my_memory_header*)((char *)(P) - HEADER_SIZE)) +#define HEADER_TO_USER(P) ((char*)(P) + HEADER_SIZE) /** Inform application that memory usage has changed @@ -82,12 +64,13 @@ void set_malloc_size_cb(MALLOC_SIZE_CB func) @return A pointer to the allocated memory block, or NULL on failure. */ -void *my_malloc(size_t size, myf my_flags) +void *my_malloc(PSI_memory_key key, size_t size, myf my_flags) { - void* point; + my_memory_header *mh; + void *point; DBUG_ENTER("my_malloc"); - DBUG_PRINT("my",("size: %lu my_flags: %lu", (ulong) size, my_flags)); - compile_time_assert(sizeof(size_t) <= sizeof(double)); + DBUG_PRINT("my",("size: %zu flags: %lu", size, my_flags)); + compile_time_assert(sizeof(my_memory_header) <= HEADER_SIZE); if (!(my_flags & (MY_WME | MY_FAE))) my_flags|= my_global_flags; @@ -95,12 +78,18 @@ void *my_malloc(size_t size, myf my_flags) /* Safety */ if (!size) size=1; + if (size > SIZE_T_MAX - 1024L*1024L*16L) /* Wrong call */ + DBUG_RETURN(0); - /* We have to align size to be able to store markers in it */ + /* We have to align size as we store MY_THREAD_SPECIFIC flag in the LSB */ size= ALIGN_SIZE(size); - point= sf_malloc(size + MALLOC_PREFIX_SIZE, my_flags); - if (point == NULL) + if (DBUG_EVALUATE_IF("simulate_out_of_memory", 1, 0)) + mh= NULL; + else + mh= (my_memory_header*) sf_malloc(size + HEADER_SIZE, my_flags); + + if (mh == NULL) { my_errno=errno; if (my_flags & MY_FAE) @@ -109,22 +98,19 @@ void *my_malloc(size_t size, myf my_flags) my_error(EE_OUTOFMEMORY, MYF(ME_BELL+ME_ERROR_LOG+ME_FATAL),size); if (my_flags & MY_FAE) abort(); + point= NULL; } else { - MALLOC_STORE_SIZE(point, void*, size, - MY_TEST(my_flags & MY_THREAD_SPECIFIC)); - update_malloc_size(size + MALLOC_PREFIX_SIZE, - MY_TEST(my_flags & MY_THREAD_SPECIFIC)); - TRASH_ALLOC(point, size); - DBUG_EXECUTE_IF("simulate_out_of_memory", - { - /* my_free() handles memory accounting */ - my_free(point); - point= NULL; - }); + int flag= MY_TEST(my_flags & MY_THREAD_SPECIFIC); + mh->m_size= size | flag; + mh->m_key= PSI_CALL_memory_alloc(key, size, & mh->m_owner); + update_malloc_size(size + HEADER_SIZE, flag); + point= HEADER_TO_USER(mh); if (my_flags & MY_ZEROFILL) bzero(point, size); + else + TRASH_ALLOC(point, size); } DBUG_PRINT("exit",("ptr: %p", point)); DBUG_RETURN(point); @@ -134,62 +120,57 @@ void *my_malloc(size_t size, myf my_flags) /** @brief wrapper around realloc() - @param oldpoint pointer to currently allocated area + @param old_point pointer to currently allocated area @param size new size requested, must be >0 @param my_flags flags @note if size==0 realloc() may return NULL; my_realloc() treats this as an error which is not the intention of realloc() */ -void *my_realloc(void *oldpoint, size_t size, myf my_flags) +void *my_realloc(PSI_memory_key key, void *old_point, size_t size, myf my_flags) { + my_memory_header *old_mh, *mh; void *point; size_t old_size; my_bool old_flags; DBUG_ENTER("my_realloc"); - DBUG_PRINT("my",("ptr: %p size: %lu my_flags: %lu", oldpoint, - (ulong) size, my_flags)); + DBUG_PRINT("my",("ptr: %p size: %zu flags: %lu", old_point, size, my_flags)); DBUG_ASSERT(size > 0); - if (!oldpoint && (my_flags & MY_ALLOW_ZERO_PTR)) - DBUG_RETURN(my_malloc(size, my_flags)); + if (!old_point && (my_flags & MY_ALLOW_ZERO_PTR)) + DBUG_RETURN(my_malloc(key, size, my_flags)); + + old_mh= USER_TO_HEADER(old_point); + old_size= old_mh->m_size & ~1; + old_flags= old_mh->m_size & 1; + + DBUG_ASSERT(old_mh->m_key == key || old_mh->m_key == PSI_NOT_INSTRUMENTED); + DBUG_ASSERT(old_flags == MY_TEST(my_flags & MY_THREAD_SPECIFIC)); size= ALIGN_SIZE(size); - old_size= MALLOC_SIZE_AND_FLAG(oldpoint, &old_flags); - /* - Test that the new and old area are the same, if not MY_THREAD_MOVE is - given - */ - DBUG_ASSERT((MY_TEST(my_flags & MY_THREAD_SPECIFIC) == old_flags) || - (my_flags & MY_THREAD_MOVE)); - if ((point= sf_realloc(MALLOC_FIX_POINTER_FOR_FREE(oldpoint), - size + MALLOC_PREFIX_SIZE, my_flags)) == NULL) + mh= sf_realloc(old_mh, size + HEADER_SIZE, my_flags); + + if (mh == NULL) { + if (size < old_size) + DBUG_RETURN(old_point); + my_errno=errno; if (my_flags & MY_FREE_ON_ERROR) { /* my_free will take care of size accounting */ - my_free(oldpoint); - oldpoint= 0; + my_free(old_point); + old_point= 0; } - if (my_flags & MY_HOLD_ON_ERROR) - DBUG_RETURN(oldpoint); - my_errno=errno; if (my_flags & (MY_FAE+MY_WME)) my_error(EE_OUTOFMEMORY, MYF(ME_BELL + ME_FATAL), size); + point= NULL; } else { - MALLOC_STORE_SIZE(point, void*, size, - MY_TEST(my_flags & MY_THREAD_SPECIFIC)); - if (MY_TEST(my_flags & MY_THREAD_SPECIFIC) != old_flags) - { - /* memory moved between system and thread specific */ - update_malloc_size(-(longlong) old_size - MALLOC_PREFIX_SIZE, old_flags); - update_malloc_size((longlong) size + MALLOC_PREFIX_SIZE, - MY_TEST(my_flags & MY_THREAD_SPECIFIC)); - } - else - update_malloc_size((longlong)size - (longlong)old_size, old_flags); + mh->m_size= size | old_flags; + mh->m_key= PSI_CALL_memory_realloc(key, old_size, size, & mh->m_owner); + update_malloc_size((longlong)size - (longlong)old_size, old_flags); + point= HEADER_TO_USER(mh); } DBUG_PRINT("exit",("ptr: %p", point)); @@ -204,56 +185,63 @@ void *my_realloc(void *oldpoint, size_t size, myf my_flags) */ void my_free(void *ptr) { + my_memory_header *mh; + size_t old_size; + my_bool old_flags; DBUG_ENTER("my_free"); DBUG_PRINT("my",("ptr: %p", ptr)); - if (ptr) - { - size_t old_size; - my_bool old_flags; - old_size= MALLOC_SIZE_AND_FLAG(ptr, &old_flags); - update_malloc_size(- (longlong) old_size - MALLOC_PREFIX_SIZE, old_flags); + + if (ptr == NULL) + DBUG_VOID_RETURN; + + mh= USER_TO_HEADER(ptr); + old_size= mh->m_size & ~1; + old_flags= mh->m_size & 1; + PSI_CALL_memory_free(mh->m_key, old_size, mh->m_owner); + + update_malloc_size(- (longlong) old_size - HEADER_SIZE, old_flags); + #ifndef SAFEMALLOC - /* - Trash memory if not safemalloc. We don't have to do this if safemalloc - is used as safemalloc will also do trashing - */ - TRASH_FREE(ptr, old_size); + /* + Trash memory if not safemalloc. We don't have to do this if safemalloc + is used as safemalloc will also do trashing + */ + TRASH_FREE(ptr, old_size); #endif - sf_free(MALLOC_FIX_POINTER_FOR_FREE(ptr)); - } + sf_free(mh); DBUG_VOID_RETURN; } -void *my_memdup(const void *from, size_t length, myf my_flags) +void *my_memdup(PSI_memory_key key, const void *from, size_t length, myf my_flags) { void *ptr; DBUG_ENTER("my_memdup"); - if ((ptr= my_malloc(length,my_flags)) != 0) + if ((ptr= my_malloc(key, length,my_flags)) != 0) memcpy(ptr, from, length); DBUG_RETURN(ptr); } -char *my_strdup(const char *from, myf my_flags) +char *my_strdup(PSI_memory_key key, const char *from, myf my_flags) { char *ptr; size_t length= strlen(from)+1; DBUG_ENTER("my_strdup"); - if ((ptr= (char*) my_malloc(length, my_flags))) + if ((ptr= (char*) my_malloc(key, length, my_flags))) memcpy(ptr, from, length); DBUG_RETURN(ptr); } -char *my_strndup(const char *from, size_t length, myf my_flags) +char *my_strndup(PSI_memory_key key, const char *from, size_t length, myf my_flags) { char *ptr; DBUG_ENTER("my_strndup"); - if ((ptr= (char*) my_malloc(length+1, my_flags))) + if ((ptr= (char*) my_malloc(key, length+1, my_flags))) { memcpy(ptr, from, length); ptr[length]= 0; diff --git a/mysys/my_mkdir.c b/mysys/my_mkdir.c index ea1f66f3b7b..e14b85d98be 100644 --- a/mysys/my_mkdir.c +++ b/mysys/my_mkdir.c @@ -34,7 +34,7 @@ int my_mkdir(const char *dir, int Flags, myf MyFlags) #endif { my_errno=errno; - DBUG_PRINT("error",("error %d when creating direcory %s",my_errno,dir)); + DBUG_PRINT("error",("error %d when creating directory %s",my_errno,dir)); if (MyFlags & (MY_FFNF | MY_FAE | MY_WME)) my_error(EE_CANT_MKDIR, MYF(ME_BELL), dir, my_errno); DBUG_RETURN(-1); diff --git a/mysys/my_mmap.c b/mysys/my_mmap.c index 9d84c1c76f6..e04f4addd3d 100644 --- a/mysys/my_mmap.c +++ b/mysys/my_mmap.c @@ -29,6 +29,10 @@ int my_msync(int fd, void *addr, size_t len, int flags) #elif defined(_WIN32) +#ifndef FILE_DAX_VOLUME +#define FILE_DAX_VOLUME 0x20000000 +#endif + static SECURITY_ATTRIBUTES mmap_security_attributes= {sizeof(SECURITY_ATTRIBUTES), 0, TRUE}; @@ -61,6 +65,18 @@ void *my_mmap(void *addr, size_t len, int prot, */ CloseHandle(hFileMap); + if (flags & MAP_SYNC) + { + DWORD filesystemFlags; + if (!GetVolumeInformationByHandleW(hFile, NULL, 0, NULL, NULL, + &filesystemFlags, NULL, 0) || + !(filesystemFlags & FILE_DAX_VOLUME)) + { + UnmapViewOfFile(ptr); + ptr= NULL; + } + } + if (ptr) { DBUG_PRINT("mysys", ("mapped addr: %p", ptr)); @@ -79,7 +95,8 @@ int my_munmap(void *addr, size_t len) int my_msync(int fd, void *addr, size_t len, int flags) { - return FlushViewOfFile(addr, len) ? 0 : -1; + return FlushViewOfFile(addr, len) && + FlushFileBuffers(my_get_osfhandle(fd)) ? 0 : -1; } #else diff --git a/mysys/my_open.c b/mysys/my_open.c index 09fa201c0bd..8b5f4f9435e 100644 --- a/mysys/my_open.c +++ b/mysys/my_open.c @@ -137,7 +137,7 @@ File my_register_filename(File fd, const char *FileName, enum file_type my_atomic_add32_explicit(&my_file_opened, 1, MY_MEMORY_ORDER_RELAXED); if ((uint) fd >= my_file_limit) DBUG_RETURN(fd); - my_file_info[fd].name = (char*) my_strdup(FileName, MyFlags); + my_file_info[fd].name = my_strdup(key_memory_my_file_info, FileName, MyFlags); statistic_increment(my_file_total_opened,&THR_LOCK_open); my_file_info[fd].type = type_of_file; DBUG_PRINT("exit",("fd: %d",fd)); diff --git a/mysys/my_pthread.c b/mysys/my_pthread.c index e2795ed7bb9..c1567f2cd03 100644 --- a/mysys/my_pthread.c +++ b/mysys/my_pthread.c @@ -29,8 +29,6 @@ #define SCHED_POLICY SCHED_OTHER #endif -uint thd_lib_detected= 0; - /* Some functions for RTS threads, AIX, Siemens Unix and UnixWare 7 (and DEC OSF/1 3.2 too) @@ -405,3 +403,66 @@ int pthread_dummy(int ret) { return ret; } + + +/* + pthread_attr_setstacksize() without so much platform-dependency + + Return: The actual stack size if possible. +*/ + +size_t my_setstacksize(pthread_attr_t *attr, size_t stacksize) +{ + size_t guard_size __attribute__((unused))= 0; + +#if defined(__ia64__) || defined(__ia64) + /* + On IA64, half of the requested stack size is used for "normal stack" + and half for "register stack". The space measured by check_stack_overrun + is the "normal stack", so double the request to make sure we have the + caller-expected amount of normal stack. + + NOTE: there is no guarantee that the register stack can't grow faster + than normal stack, so it's very unclear that we won't dump core due to + stack overrun despite check_stack_overrun's efforts. Experimentation + shows that in the execution_constants test, the register stack grows + less than half as fast as normal stack, but perhaps other scenarios are + less forgiving. If it turns out that more space is needed for the + register stack, that could be forced (rather inefficiently) by using a + multiplier higher than 2 here. + */ + stacksize *= 2; +#endif + + /* + On many machines, the "guard space" is subtracted from the requested + stack size, and that space is quite large on some platforms. So add + it to our request, if we can find out what it is. + */ +#ifdef HAVE_PTHREAD_ATTR_GETGUARDSIZE + if (pthread_attr_getguardsize(attr, &guard_size)) + guard_size = 0; /* if can't find it out, treat as 0 */ +#endif /* HAVE_PTHREAD_ATTR_GETGUARDSIZE */ + + pthread_attr_setstacksize(attr, stacksize + guard_size); + + /* Retrieve actual stack size if possible */ +#ifdef HAVE_PTHREAD_ATTR_GETSTACKSIZE + { + size_t real_stack_size= 0; + /* We must ignore real_stack_size = 0 as Solaris 2.9 can return 0 here */ + if (pthread_attr_getstacksize(attr, &real_stack_size) == 0 && + real_stack_size > guard_size) + { + real_stack_size -= guard_size; + if (real_stack_size < stacksize) + stacksize= real_stack_size; + } + } +#endif /* HAVE_PTHREAD_ATTR_GETSTACKSIZE */ + +#if defined(__ia64__) || defined(__ia64) + stacksize /= 2; +#endif + return stacksize; +} diff --git a/mysys/my_rdtsc.c b/mysys/my_rdtsc.c index c707588da01..1503a5db442 100644 --- a/mysys/my_rdtsc.c +++ b/mysys/my_rdtsc.c @@ -173,9 +173,29 @@ ulonglong my_timer_microseconds(void) milliseconds. */ +#if defined(HAVE_CLOCK_GETTIME) +#if defined(CLOCK_MONOTONIC_FAST) +/* FreeBSD */ +#define MY_CLOCK_ID CLOCK_MONOTONIC_FAST +#elif defined(CLOCK_MONOTONIC_COARSE) +/* Linux */ +#define MY_CLOCK_ID CLOCK_MONOTONIC_COARSE +#elif defined(CLOCK_MONOTONIC) +/* POSIX (includes OSX) */ +#define MY_CLOCK_ID CLOCK_MONOTONIC +#elif defined(CLOCK_REALTIME) +/* Solaris (which doesn't seem to have MONOTONIC) */ +#define MY_CLOCK_ID CLOCK_REALTIME +#endif +#endif + ulonglong my_timer_milliseconds(void) { -#if defined(HAVE_SYS_TIMEB_H) && defined(HAVE_FTIME) +#if defined(MY_CLOCK_ID) + struct timespec tp; + clock_gettime(MY_CLOCK_ID, &tp); + return (ulonglong)tp.tv_sec * 1000 + (ulonglong)tp.tv_nsec / 1000000; +#elif defined(HAVE_SYS_TIMEB_H) && defined(HAVE_FTIME) /* ftime() is obsolete but maybe the platform is old */ struct timeb ft; ftime(&ft); @@ -428,7 +448,9 @@ void my_timer_init(MY_TIMER_INFO *mti) /* milliseconds */ mti->milliseconds.frequency= 1000; /* initial assumption */ -#if defined(HAVE_SYS_TIMEB_H) && defined(HAVE_FTIME) +#ifdef MY_CLOCK_ID + mti->milliseconds.routine= MY_TIMER_ROUTINE_CLOCK_GETTIME; +#elif defined(HAVE_SYS_TIMEB_H) && defined(HAVE_FTIME) mti->milliseconds.routine= MY_TIMER_ROUTINE_FTIME; #elif defined(_WIN32) mti->milliseconds.routine= MY_TIMER_ROUTINE_GETSYSTEMTIMEASFILETIME; diff --git a/mysys/my_safehash.c b/mysys/my_safehash.c index 7555f73fb30..7d37b707c3a 100644 --- a/mysys/my_safehash.c +++ b/mysys/my_safehash.c @@ -100,8 +100,8 @@ my_bool safe_hash_init(SAFE_HASH *hash, uint elements, uchar *default_value) { DBUG_ENTER("safe_hash_init"); - if (my_hash_init(&hash->hash, &my_charset_bin, elements, - 0, 0, (my_hash_get_key) safe_hash_entry_get, + if (my_hash_init(key_memory_SAFE_HASH_ENTRY, &hash->hash, &my_charset_bin, + elements, 0, 0, (my_hash_get_key) safe_hash_entry_get, (void (*)(void*)) safe_hash_entry_free, 0)) { hash->default_value= 0; @@ -224,7 +224,8 @@ my_bool safe_hash_set(SAFE_HASH *hash, const uchar *key, uint length, } else { - if (!(entry= (SAFE_HASH_ENTRY *) my_malloc(sizeof(*entry) + length, + if (!(entry= (SAFE_HASH_ENTRY *) my_malloc(key_memory_SAFE_HASH_ENTRY, + sizeof(*entry) + length, MYF(MY_WME)))) { error= 1; diff --git a/mysys/my_static.c b/mysys/my_static.c index fc4c60bc237..6c09ab8d94b 100644 --- a/mysys/my_static.c +++ b/mysys/my_static.c @@ -23,10 +23,40 @@ #include "my_static.h" #include "my_alarm.h" -my_bool timed_mutexes= 0; + +PSI_memory_key key_memory_DYNAMIC_STRING; +PSI_memory_key key_memory_IO_CACHE; +PSI_memory_key key_memory_KEY_CACHE; +PSI_memory_key key_memory_LIST; +PSI_memory_key key_memory_MY_BITMAP_bitmap; +PSI_memory_key key_memory_MY_DIR; +PSI_memory_key key_memory_MY_STAT; +PSI_memory_key key_memory_MY_TMPDIR_full_list; +PSI_memory_key key_memory_QUEUE; +PSI_memory_key key_memory_SAFE_HASH_ENTRY; +PSI_memory_key key_memory_THD_ALARM; +PSI_memory_key key_memory_TREE; +PSI_memory_key key_memory_charset_file; +PSI_memory_key key_memory_charset_loader; +PSI_memory_key key_memory_defaults; +PSI_memory_key key_memory_lf_dynarray; +PSI_memory_key key_memory_lf_node; +PSI_memory_key key_memory_lf_slist; +PSI_memory_key key_memory_max_alloca; +PSI_memory_key key_memory_my_compress_alloc; +PSI_memory_key key_memory_my_err_head; +PSI_memory_key key_memory_my_file_info; +PSI_memory_key key_memory_pack_frm; +PSI_memory_key key_memory_charsets; + +#ifdef _WIN32 +PSI_memory_key key_memory_win_SECURITY_ATTRIBUTES; +PSI_memory_key key_memory_win_PACL; +PSI_memory_key key_memory_win_IP_ADAPTER_ADDRESSES; +#endif /* _WIN32 */ /* from my_init */ -char * home_dir=0; +char *home_dir=0; char *mysql_data_home= (char*) "."; const char *my_progname= NULL, *my_progname_short= NULL; char curr_dir[FN_REFLEN]= {0}, @@ -62,12 +92,6 @@ const char *soundex_map= "01230120022455012623010202"; USED_MEM* my_once_root_block=0; /* pointer to first block */ uint my_once_extra=ONCE_ALLOC_INIT; /* Memory to alloc / block */ - /* from my_largepage.c */ -#ifdef HAVE_LINUX_LARGE_PAGES -my_bool my_use_large_pages= 0; -uint my_large_page_size= 0; -#endif - /* from my_alarm */ int volatile my_have_got_alarm=0; /* declare variable to reset */ ulong my_time_to_wait_for_lock=2; /* In seconds */ diff --git a/mysys/my_symlink.c b/mysys/my_symlink.c index cbee78a7f5c..323ae69a39c 100644 --- a/mysys/my_symlink.c +++ b/mysys/my_symlink.c @@ -154,7 +154,8 @@ int my_realpath(char *to, const char *filename, myf MyFlags) original name but will at least be able to resolve paths that starts with '.'. */ - DBUG_PRINT("error",("realpath failed with errno: %d", errno)); + if (MyFlags) + DBUG_PRINT("error",("realpath failed with errno: %d", errno)); my_errno=errno; if (MyFlags & MY_WME) my_error(EE_REALPATH, MYF(0), filename, my_errno); diff --git a/mysys/my_symlink2.c b/mysys/my_symlink2.c index aef77333e75..0b580ecd32f 100644 --- a/mysys/my_symlink2.c +++ b/mysys/my_symlink2.c @@ -170,6 +170,7 @@ int my_rename_with_symlink(const char *from, const char *to, myf MyFlags) in this case both the symlink and the symlinked file are deleted, but only if the symlinked file is not in the datadir. */ + int my_handler_delete_with_symlink(const char *filename, myf sync_dir) { char real[FN_REFLEN]; @@ -182,8 +183,9 @@ int my_handler_delete_with_symlink(const char *filename, myf sync_dir) Delete the symlinked file only if the symlink is not pointing into datadir. */ - if (!(my_realpath(real, filename, MYF(0)) || mysys_test_invalid_symlink(real))) + if (!(my_realpath(real, filename, MYF(0)) || + mysys_test_invalid_symlink(real))) res= my_delete(real, MYF(MY_NOSYMLINKS | sync_dir)); } - DBUG_RETURN(my_delete(filename, MYF(sync_dir)) || res); + DBUG_RETURN(my_delete(filename, sync_dir) || res); } diff --git a/mysys/my_thr_init.c b/mysys/my_thr_init.c index 66c2543c51d..24b4b3aff15 100644 --- a/mysys/my_thr_init.c +++ b/mysys/my_thr_init.c @@ -39,9 +39,6 @@ mysql_mutex_t LOCK_localtime_r; static void install_sigabrt_handler(); #endif - -static uint get_thread_lib(void); - /** True if @c my_thread_global_init() has been called. */ static my_bool my_thread_global_init_done= 0; /* True if THR_KEY_mysys is created */ @@ -96,7 +93,6 @@ static void my_thread_init_internal_mutex(void) mysql_cond_init(key_THR_COND_threads, &THR_COND_threads, NULL); } - void my_thread_destroy_internal_mutex(void) { mysql_mutex_destroy(&THR_LOCK_threads); @@ -190,7 +186,6 @@ my_bool my_thread_global_init(void) if (my_thread_init()) return 1; - thd_lib_detected= get_thread_lib(); my_thread_init_common_mutex(); @@ -455,21 +450,6 @@ safe_mutex_t **my_thread_var_mutex_in_use() return tmp ? &tmp->mutex_in_use : 0; } -static uint get_thread_lib(void) -{ -#ifdef _CS_GNU_LIBPTHREAD_VERSION - char buff[64]; - - confstr(_CS_GNU_LIBPTHREAD_VERSION, buff, sizeof(buff)); - - if (!strncasecmp(buff, "NPTL", 4)) - return THD_LIB_NPTL; - if (!strncasecmp(buff, "linuxthreads", 12)) - return THD_LIB_LT; -#endif - return THD_LIB_OTHER; -} - #ifdef _WIN32 /* In Visual Studio 2005 and later, default SIGABRT handler will overwrite diff --git a/mysys/my_winfile.c b/mysys/my_winfile.c index 0c76eb25560..9c8d747adc9 100644 --- a/mysys/my_winfile.c +++ b/mysys/my_winfile.c @@ -501,13 +501,12 @@ static File my_get_stdfile_descriptor(FILE *stream) } -File my_win_fileno(FILE *file) +File my_win_handle2File(HANDLE hFile) { - HANDLE hFile= (HANDLE)_get_osfhandle(fileno(file)); int retval= -1; uint i; - DBUG_ENTER("my_win_fileno"); + DBUG_ENTER("my_win_handle2File"); for(i= MY_FILE_MIN; i < my_file_limit; i++) { @@ -517,6 +516,14 @@ File my_win_fileno(FILE *file) break; } } + DBUG_RETURN(retval); +} + + +File my_win_fileno(FILE *file) +{ + DBUG_ENTER("my_win_fileno"); + int retval= my_win_handle2File((HANDLE) _get_osfhandle(fileno(file))); if(retval == -1) /* try std stream */ DBUG_RETURN(my_get_stdfile_descriptor(file)); diff --git a/mysys/my_wintoken.c b/mysys/my_wintoken.c new file mode 100644 index 00000000000..78c4bc9dad5 --- /dev/null +++ b/mysys/my_wintoken.c @@ -0,0 +1,42 @@ +/* Copyright (c) 2019, IBM. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +#include "mysys_priv.h" + +BOOL my_obtain_privilege(LPCSTR lpPrivilege) +{ + HANDLE hAccessToken; + TOKEN_PRIVILEGES token; + BOOL ret_value= FALSE; + + if (!OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES, &hAccessToken)) + { + return FALSE; + } + + if (!LookupPrivilegeValue(NULL, lpPrivilege, &token.Privileges[0].Luid)) + return FALSE; + + token.PrivilegeCount= 1; + token.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED; + + ret_value= AdjustTokenPrivileges(hAccessToken, FALSE, &token, 0, NULL, NULL); + + if (!ret_value || (GetLastError() != ERROR_SUCCESS)) + return FALSE; + + CloseHandle(hAccessToken); + return TRUE; +} diff --git a/mysys/mysys_priv.h b/mysys/mysys_priv.h index 0fa6a4c553f..adf2d39046a 100644 --- a/mysys/mysys_priv.h +++ b/mysys/mysys_priv.h @@ -64,6 +64,37 @@ extern PSI_rwlock_key key_SAFEHASH_mutex; extern PSI_stage_info stage_waiting_for_table_level_lock; +/* These keys are always defined. */ + +extern PSI_memory_key key_memory_DYNAMIC_STRING; +extern PSI_memory_key key_memory_IO_CACHE; +extern PSI_memory_key key_memory_KEY_CACHE; +extern PSI_memory_key key_memory_LIST; +extern PSI_memory_key key_memory_MY_BITMAP_bitmap; +extern PSI_memory_key key_memory_MY_DIR; +extern PSI_memory_key key_memory_MY_STAT; +extern PSI_memory_key key_memory_MY_TMPDIR_full_list; +extern PSI_memory_key key_memory_QUEUE; +extern PSI_memory_key key_memory_SAFE_HASH_ENTRY; +extern PSI_memory_key key_memory_TREE; +extern PSI_memory_key key_memory_charset_file; +extern PSI_memory_key key_memory_charset_loader; +extern PSI_memory_key key_memory_defaults; +extern PSI_memory_key key_memory_lf_dynarray; +extern PSI_memory_key key_memory_lf_node; +extern PSI_memory_key key_memory_lf_slist; +extern PSI_memory_key key_memory_my_compress_alloc; +extern PSI_memory_key key_memory_my_err_head; +extern PSI_memory_key key_memory_my_file_info; +extern PSI_memory_key key_memory_pack_frm; +extern PSI_memory_key key_memory_charsets; + +#ifdef _WIN32 +extern PSI_memory_key key_memory_win_SECURITY_ATTRIBUTES; +extern PSI_memory_key key_memory_win_PACL; +extern PSI_memory_key key_memory_win_IP_ADAPTER_ADDRESSES; +#endif + extern mysql_mutex_t THR_LOCK_malloc, THR_LOCK_open, THR_LOCK_keycache; extern mysql_mutex_t THR_LOCK_lock, THR_LOCK_net; extern mysql_mutex_t THR_LOCK_charset; @@ -71,9 +102,6 @@ extern mysql_mutex_t THR_LOCK_charset; #include <mysql/psi/mysql_file.h> #ifdef HAVE_PSI_INTERFACE -#ifdef HUGETLB_USE_PROC_MEMINFO -extern PSI_file_key key_file_proc_meminfo; -#endif /* HUGETLB_USE_PROC_MEMINFO */ extern PSI_file_key key_file_charset, key_file_cnf; #endif /* HAVE_PSI_INTERFACE */ diff --git a/mysys/psi_noop.c b/mysys/psi_noop.c index 9eec07a1cd8..09b839468ac 100644 --- a/mysys/psi_noop.c +++ b/mysys/psi_noop.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2011, 2015, Oracle and/or its affiliates. +/* Copyright (c) 2011, 2016, Oracle and/or its affiliates. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -168,7 +168,8 @@ rebind_table_noop(PSI_table_share *share NNN, return NULL; } -static void close_table_noop(PSI_table *table NNN) +static void close_table_noop(struct TABLE_SHARE *share NNN, + PSI_table *table NNN) { return; } @@ -199,6 +200,16 @@ static void set_thread_id_noop(PSI_thread *thread NNN, ulonglong id NNN) return; } +static void set_thread_THD_noop(PSI_thread *thread NNN, THD *thd NNN) +{ + return; +} + +static void set_thread_os_id_noop(PSI_thread *thread NNN) +{ + return; +} + static PSI_thread* get_thread_noop(void NNN) { @@ -226,6 +237,11 @@ static void set_thread_command_noop(int command NNN) return; } +static void set_connection_type_noop(opaque_vio_type conn_type NNN) +{ + return; +} + static void set_thread_start_time_noop(time_t start_time NNN) { return; @@ -373,7 +389,7 @@ static void end_cond_wait_noop(PSI_cond_locker* locker NNN, int rc NNN) } static struct PSI_table_locker* -start_table_io_wait_noop(struct PSI_table_locker_state_v1 *state NNN, +start_table_io_wait_noop(struct PSI_table_locker_state *state NNN, struct PSI_table *table NNN, enum PSI_table_io_operation op NNN, uint index NNN, @@ -382,13 +398,14 @@ start_table_io_wait_noop(struct PSI_table_locker_state_v1 *state NNN, return NULL; } -static void end_table_io_wait_noop(PSI_table_locker* locker NNN) +static void end_table_io_wait_noop(PSI_table_locker* locker NNN, + ulonglong numrows NNN) { return; } static struct PSI_table_locker* -start_table_lock_wait_noop(struct PSI_table_locker_state_v1 *state NNN, +start_table_lock_wait_noop(struct PSI_table_locker_state *state NNN, struct PSI_table *table NNN, enum PSI_table_lock_operation op NNN, ulong flags NNN, @@ -421,6 +438,12 @@ static void end_file_open_wait_and_bind_to_descriptor_noop return; } +static void end_temp_file_open_wait_and_bind_to_descriptor_noop + (PSI_file_locker *locker NNN, File file NNN, const char *filaneme NNN) +{ + return; +} + static void start_file_wait_noop(PSI_file_locker *locker NNN, size_t count NNN, const char *src_file NNN, @@ -448,12 +471,27 @@ static void end_file_close_wait_noop(PSI_file_locker *locker NNN, return; } -static void start_stage_noop(PSI_stage_key key NNN, - const char *src_file NNN, int src_line NNN) +static void end_file_rename_wait_noop(PSI_file_locker *locker NNN, + const char *old_name NNN, + const char *new_name NNN, + int result NNN) { return; } +static PSI_stage_progress* +start_stage_noop(PSI_stage_key key NNN, + const char *src_file NNN, int src_line NNN) +{ + return NULL; +} + +static PSI_stage_progress* +get_current_stage_progress_noop(void) +{ + return NULL; +} + static void end_stage_noop(void) { return; @@ -462,7 +500,8 @@ static void end_stage_noop(void) static PSI_statement_locker* get_thread_statement_locker_noop(PSI_statement_locker_state *state NNN, PSI_statement_key key NNN, - const void *charset NNN) + const void *charset NNN, + PSI_sp_share *sp_share NNN) { return NULL; } @@ -587,6 +626,73 @@ static void end_statement_noop(PSI_statement_locker *locker NNN, return; } +static PSI_transaction_locker* +get_thread_transaction_locker_noop(PSI_transaction_locker_state *state NNN, + const void *xid NNN, + ulonglong trxid NNN, + int isolation_level NNN, + my_bool read_only NNN, + my_bool autocommit NNN) +{ + return NULL; +} + +static void start_transaction_noop(PSI_transaction_locker *locker NNN, + const char *src_file NNN, uint src_line NNN) +{ + return; +} + +static void set_transaction_xid_noop(PSI_transaction_locker *locker NNN, + const void *xid NNN, + int xa_state NNN) +{ + return; +} + +static void set_transaction_xa_state_noop(PSI_transaction_locker *locker NNN, + int xa_state NNN) +{ + return; +} + +static void set_transaction_gtid_noop(PSI_transaction_locker *locker NNN, + const void *sid NNN, + const void *gtid_spec NNN) +{ + return; +} + +static void set_transaction_trxid_noop(PSI_transaction_locker *locker NNN, + const ulonglong *trxid NNN) +{ + return; +} + +static void inc_transaction_savepoints_noop(PSI_transaction_locker *locker NNN, + ulong count NNN) +{ + return; +} + +static void inc_transaction_rollback_to_savepoint_noop(PSI_transaction_locker *locker NNN, + ulong count NNN) +{ + return; +} + +static void inc_transaction_release_savepoint_noop(PSI_transaction_locker *locker NNN, + ulong count NNN) +{ + return; +} + +static void end_transaction_noop(PSI_transaction_locker *locker NNN, + my_bool commit NNN) +{ + return; +} + static PSI_socket_locker* start_socket_wait_noop(PSI_socket_locker_state *state NNN, PSI_socket *socket NNN, @@ -623,6 +729,39 @@ static void set_socket_thread_owner_noop(PSI_socket *socket NNN) return; } +static PSI_prepared_stmt* +create_prepare_stmt_noop(void *identity NNN, uint stmt_id NNN, + PSI_statement_locker *locker NNN, + const char *stmt_name NNN, size_t stmt_name_length NNN) +{ + return NULL; +} + +static void +execute_prepare_stmt_noop(PSI_statement_locker *locker NNN, + PSI_prepared_stmt *prepared_stmt NNN) +{ + return; +} + +static void set_prepared_stmt_text_noop(PSI_prepared_stmt *prepared_stmt NNN, + const char *text NNN, uint text_len NNN) +{ + return; +} + +void +destroy_prepared_stmt_noop(PSI_prepared_stmt *prepared_stmt NNN) +{ + return; +} + +void +reprepare_prepared_stmt_noop(PSI_prepared_stmt *prepared_stmt NNN) +{ + return; +} + static struct PSI_digest_locker* digest_start_noop(PSI_statement_locker *locker NNN) { @@ -644,6 +783,112 @@ set_thread_connect_attrs_noop(const char *buffer NNN, return 0; } +static PSI_sp_locker* +pfs_start_sp_noop(PSI_sp_locker_state *state NNN, PSI_sp_share *sp_share NNN) +{ + return NULL; +} + +static void pfs_end_sp_noop(PSI_sp_locker *locker NNN) +{ + return; +} + +static void +pfs_drop_sp_noop(uint object_type NNN, + const char *schema_name NNN, uint schema_name_length NNN, + const char *object_name NNN, uint object_name_length NNN) +{ + return; +} + +static PSI_sp_share* +pfs_get_sp_share_noop(uint object_type NNN, + const char *schema_name NNN, uint schema_name_length NNN, + const char *object_name NNN, uint object_name_length NNN) +{ + return NULL; +} + +static void +pfs_release_sp_share_noop(PSI_sp_share *sp_share NNN) +{ + return; +} + +static void register_memory_noop(const char *category NNN, + PSI_memory_info *info NNN, + int count NNN) +{ + return; +} + +static PSI_memory_key memory_alloc_noop(PSI_memory_key key NNN, size_t size NNN, struct PSI_thread ** owner NNN) +{ + *owner= NULL; + return PSI_NOT_INSTRUMENTED; +} + +static PSI_memory_key memory_realloc_noop(PSI_memory_key key NNN, size_t old_size NNN, size_t new_size NNN, struct PSI_thread ** owner NNN) +{ + *owner= NULL; + return PSI_NOT_INSTRUMENTED; +} + +static PSI_memory_key memory_claim_noop(PSI_memory_key key NNN, size_t size NNN, struct PSI_thread ** owner) +{ + *owner= NULL; + return PSI_NOT_INSTRUMENTED; +} + +static void memory_free_noop(PSI_memory_key key NNN, size_t size NNN, struct PSI_thread * owner NNN) +{ + return; +} + +static void unlock_table_noop(PSI_table *table NNN) +{ + return; +} + +static PSI_metadata_lock * +create_metadata_lock_noop(void *identity NNN, + const MDL_key *mdl_key NNN, + opaque_mdl_type mdl_type NNN, + opaque_mdl_duration mdl_duration NNN, + opaque_mdl_status mdl_status NNN, + const char *src_file NNN, + uint src_line NNN) +{ + return NULL; +} + +static void +set_metadata_lock_status_noop(PSI_metadata_lock* lock NNN, + opaque_mdl_status mdl_status NNN) +{ +} + +static void +destroy_metadata_lock_noop(PSI_metadata_lock* lock NNN) +{ +} + +static PSI_metadata_locker * +start_metadata_wait_noop(PSI_metadata_locker_state *state NNN, + PSI_metadata_lock *mdl NNN, + const char *src_file NNN, + uint src_line NNN) +{ + return NULL; +} + +static void +end_metadata_wait_noop(PSI_metadata_locker *locker NNN, + int rc NNN) +{ +} + static PSI PSI_noop= { register_mutex_noop, @@ -673,11 +918,14 @@ static PSI PSI_noop= spawn_thread_noop, new_thread_noop, set_thread_id_noop, + set_thread_THD_noop, + set_thread_os_id_noop, get_thread_noop, set_thread_user_noop, set_thread_user_host_noop, set_thread_db_noop, set_thread_command_noop, + set_connection_type_noop, set_thread_start_time_noop, set_thread_state_noop, set_thread_info_noop, @@ -708,11 +956,14 @@ static PSI PSI_noop= start_file_open_wait_noop, end_file_open_wait_noop, end_file_open_wait_and_bind_to_descriptor_noop, + end_temp_file_open_wait_and_bind_to_descriptor_noop, start_file_wait_noop, end_file_wait_noop, start_file_close_wait_noop, end_file_close_wait_noop, + end_file_rename_wait_noop, start_stage_noop, + get_current_stage_progress_noop, end_stage_noop, get_thread_statement_locker_noop, refine_statement_noop, @@ -735,14 +986,46 @@ static PSI PSI_noop= set_statement_no_index_used_noop, set_statement_no_good_index_used_noop, end_statement_noop, + get_thread_transaction_locker_noop, + start_transaction_noop, + set_transaction_xid_noop, + set_transaction_xa_state_noop, + set_transaction_gtid_noop, + set_transaction_trxid_noop, + inc_transaction_savepoints_noop, + inc_transaction_rollback_to_savepoint_noop, + inc_transaction_release_savepoint_noop, + end_transaction_noop, start_socket_wait_noop, end_socket_wait_noop, set_socket_state_noop, set_socket_info_noop, set_socket_thread_owner_noop, + create_prepare_stmt_noop, + destroy_prepared_stmt_noop, + reprepare_prepared_stmt_noop, + execute_prepare_stmt_noop, + set_prepared_stmt_text_noop, digest_start_noop, digest_end_noop, - set_thread_connect_attrs_noop + set_thread_connect_attrs_noop, + pfs_start_sp_noop, + pfs_end_sp_noop, + pfs_drop_sp_noop, + pfs_get_sp_share_noop, + pfs_release_sp_share_noop, + register_memory_noop, + memory_alloc_noop, + memory_realloc_noop, + memory_claim_noop, + memory_free_noop, + + unlock_table_noop, + create_metadata_lock_noop, + set_metadata_lock_status_noop, + destroy_metadata_lock_noop, + start_metadata_wait_noop, + end_metadata_wait_noop }; /** @@ -763,16 +1046,10 @@ struct PSI_bootstrap *PSI_hook= NULL; PSI *PSI_server= & PSI_noop; -/** - Global performance schema flag. - Indicate if the performance schema is enabled. - This flag is set at startup, and never changes. -*/ -my_bool pfs_enabled= FALSE; - void set_psi_server(PSI *psi) { PSI_server= psi; } C_MODE_END + diff --git a/mysys/queues.c b/mysys/queues.c index 7a0ef2017c2..0a1149bf9fc 100644 --- a/mysys/queues.c +++ b/mysys/queues.c @@ -75,7 +75,8 @@ int init_queue(QUEUE *queue, uint max_elements, uint offset_to_key, uint auto_extent) { DBUG_ENTER("init_queue"); - if ((queue->root= (uchar **) my_malloc((max_elements + 1) * sizeof(void*), + if ((queue->root= (uchar **) my_malloc(key_memory_QUEUE, + (max_elements + 1) * sizeof(void*), MYF(MY_WME))) == 0) DBUG_RETURN(1); queue->elements= 0; @@ -147,7 +148,7 @@ int resize_queue(QUEUE *queue, uint max_elements) DBUG_ENTER("resize_queue"); if (queue->max_elements == max_elements) DBUG_RETURN(0); - if ((new_root= (uchar **) my_realloc((void *)queue->root, + if ((new_root= (uchar **) my_realloc(key_memory_QUEUE, (void *)queue->root, (max_elements + 1)* sizeof(void*), MYF(MY_WME))) == 0) DBUG_RETURN(1); diff --git a/mysys/safemalloc.c b/mysys/safemalloc.c index d38c3bcfb66..e2d59f8e824 100644 --- a/mysys/safemalloc.c +++ b/mysys/safemalloc.c @@ -115,9 +115,10 @@ void *sf_malloc(size_t size, myf my_flags) init_done= 1; } - irem= (struct st_irem *) malloc (sizeof(struct st_irem) + size + 4); + if (size > SIZE_T_MAX - 1024L*1024L*16L) /* Wrong call */ + return 0; - if (!irem) + if (!(irem= (struct st_irem *) malloc (sizeof(struct st_irem) + size + 4))) return 0; /* we guarantee the alignment */ diff --git a/mysys/stacktrace.c b/mysys/stacktrace.c index f5e6352a72e..5718b2e762d 100644 --- a/mysys/stacktrace.c +++ b/mysys/stacktrace.c @@ -1,5 +1,6 @@ /* Copyright (c) 2001, 2011, Oracle and/or its affiliates + Copyright (c) 2020, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -14,12 +15,11 @@ along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ -#include <my_global.h> +#include "mysys_priv.h" #include <my_stacktrace.h> #ifndef __WIN__ #include <signal.h> -#include <my_pthread.h> #include <m_string.h> #ifdef HAVE_STACKTRACE #include <unistd.h> @@ -34,11 +34,49 @@ #include <execinfo.h> #endif +/** + Default handler for printing stacktrace +*/ + +static sig_handler default_handle_fatal_signal(int sig) +{ + my_safe_printf_stderr("%s: Got signal %d. Attempting backtrace\n", + my_progname_short, sig); + my_print_stacktrace(0,0,1); +#ifndef __WIN__ + signal(sig, SIG_DFL); + kill(getpid(), sig); +#endif /* __WIN__ */ + return; +} + + +/** + Initialize priting off stacktrace at signal +*/ + +void my_setup_stacktrace(void) +{ + struct sigaction sa; + sa.sa_flags = SA_RESETHAND | SA_NODEFER; + sigemptyset(&sa.sa_mask); + sa.sa_handler= default_handle_fatal_signal; + sigaction(SIGSEGV, &sa, NULL); + sigaction(SIGABRT, &sa, NULL); +#ifdef SIGBUS + sigaction(SIGBUS, &sa, NULL); +#endif + sigaction(SIGILL, &sa, NULL); + sigaction(SIGFPE, &sa, NULL); +} + + /* Attempt to print a char * pointer as a string. SYNOPSIS - Prints until max_len characters have been printed. + Prints either until the end of string ('\0'), or max_len characters have + been printed. RETURN VALUE 0 Pointer was within the heap address space. @@ -306,8 +344,8 @@ void my_print_stacktrace(uchar* stack_bottom, ulong thread_stack, :"r"(pc)); #endif /* __alpha__ */ - /* We are 1 frame above signal frame with NPTL and 2 frames above with LT */ - sigreturn_frame_count = thd_lib_detected == THD_LIB_LT ? 2 : 1; + /* We are 1 frame above signal frame with NPTL */ + sigreturn_frame_count = 1; while (fp < (uchar**) stack_bottom) { diff --git a/mysys/string.c b/mysys/string.c index 7c56b95af25..b346393d91e 100644 --- a/mysys/string.c +++ b/mysys/string.c @@ -36,7 +36,8 @@ my_bool init_dynamic_string(DYNAMIC_STRING *str, const char *init_str, if (!init_alloc) init_alloc=alloc_increment; - if (!(str->str=(char*) my_malloc(init_alloc,MYF(MY_WME)))) + if (!(str->str=(char*) my_malloc(key_memory_DYNAMIC_STRING, + init_alloc, MYF(MY_WME)))) DBUG_RETURN(TRUE); str->length=length-1; if (init_str) @@ -58,7 +59,8 @@ my_bool dynstr_set(DYNAMIC_STRING *str, const char *init_str) str->alloc_increment; if (!str->max_length) str->max_length=str->alloc_increment; - if (!(str->str=(char*) my_realloc(str->str,str->max_length,MYF(MY_WME)))) + if (!(str->str=(char*) my_realloc(key_memory_DYNAMIC_STRING, + str->str, str->max_length, MYF(MY_WME)))) DBUG_RETURN(TRUE); } if (init_str) @@ -81,7 +83,8 @@ my_bool dynstr_realloc(DYNAMIC_STRING *str, size_t additional_size) { str->max_length=((str->length + additional_size+str->alloc_increment-1)/ str->alloc_increment)*str->alloc_increment; - if (!(str->str=(char*) my_realloc(str->str,str->max_length,MYF(MY_WME)))) + if (!(str->str=(char*) my_realloc(key_memory_DYNAMIC_STRING, str->str, + str->max_length, MYF(MY_WME)))) DBUG_RETURN(TRUE); } DBUG_RETURN(FALSE); @@ -104,7 +107,8 @@ my_bool dynstr_append_mem(DYNAMIC_STRING *str, const char *append, size_t new_length=(str->length+length+str->alloc_increment)/ str->alloc_increment; new_length*=str->alloc_increment; - if (!(new_ptr=(char*) my_realloc(str->str,new_length,MYF(MY_WME)))) + if (!(new_ptr=(char*) my_realloc(key_memory_DYNAMIC_STRING, str->str, + new_length, MYF(MY_WME)))) DBUG_RETURN(TRUE); str->str=new_ptr; str->max_length=new_length; diff --git a/mysys/thr_alarm.c b/mysys/thr_alarm.c index 553dc55bfc3..92a471075c6 100644 --- a/mysys/thr_alarm.c +++ b/mysys/thr_alarm.c @@ -95,16 +95,8 @@ void init_thr_alarm(uint max_alarms) sigfillset(&full_signal_set); /* Neaded to block signals */ mysql_mutex_init(key_LOCK_alarm, &LOCK_alarm, MY_MUTEX_INIT_FAST); mysql_cond_init(key_COND_alarm, &COND_alarm, NULL); - if (thd_lib_detected == THD_LIB_LT) - thr_client_alarm= SIGALRM; - else - thr_client_alarm= SIGUSR1; -#ifndef USE_ALARM_THREAD - if (thd_lib_detected != THD_LIB_LT) -#endif - { - my_sigset(thr_client_alarm, thread_alarm); - } + thr_client_alarm= SIGUSR1; + my_sigset(thr_client_alarm, thread_alarm); sigemptyset(&s); sigaddset(&s, THR_SERVER_ALARM); alarm_thread=pthread_self(); @@ -121,11 +113,6 @@ void init_thr_alarm(uint max_alarms) } #elif defined(USE_ONE_SIGNAL_HAND) pthread_sigmask(SIG_BLOCK, &s, NULL); /* used with sigwait() */ - if (thd_lib_detected == THD_LIB_LT) - { - my_sigset(thr_client_alarm, process_alarm); /* Linuxthreads */ - pthread_sigmask(SIG_UNBLOCK, &s, NULL); - } #else my_sigset(THR_SERVER_ALARM, process_alarm); pthread_sigmask(SIG_UNBLOCK, &s, NULL); @@ -198,7 +185,8 @@ my_bool thr_alarm(thr_alarm_t *alrm, uint sec, ALARM *alarm_data) now= my_time(0); if (!alarm_data) { - if (!(alarm_data=(ALARM*) my_malloc(sizeof(ALARM),MYF(MY_WME)))) + if (!(alarm_data=(ALARM*) my_malloc(PSI_INSTRUMENT_ME, sizeof(ALARM), + MYF(MY_WME)))) goto abort_no_unlock; alarm_data->malloced= 1; } @@ -286,18 +274,6 @@ sig_handler process_alarm(int sig __attribute__((unused))) This must be first as we can't call DBUG inside an alarm for a normal thread */ - if (thd_lib_detected == THD_LIB_LT && - !pthread_equal(pthread_self(),alarm_thread)) - { -#if defined(MAIN) && !defined(__bsdi__) - printf("thread_alarm in process_alarm\n"); fflush(stdout); -#endif -#ifdef SIGNAL_HANDLER_RESET_ON_DELIVERY - my_sigset(thr_client_alarm, process_alarm); /* int. thread system calls */ -#endif - return; - } - /* We have to do do the handling of the alarm in a sub function, because otherwise we would get problems with two threads calling diff --git a/mysys/thr_lock.c b/mysys/thr_lock.c index 363e1a1ea26..168ec035e79 100644 --- a/mysys/thr_lock.c +++ b/mysys/thr_lock.c @@ -450,7 +450,7 @@ void thr_lock_delete(THR_LOCK *lock) void thr_lock_info_init(THR_LOCK_INFO *info, struct st_my_thread_var *tmp) { - if (tmp) + if (!tmp) tmp= my_thread_var; info->thread= tmp->pthread_self; info->thread_id= tmp->id; @@ -616,9 +616,10 @@ wait_for_lock(struct st_lock_list *wait, THR_LOCK_DATA *data, else { result= THR_LOCK_SUCCESS; - if (data->lock->get_status) - (*data->lock->get_status)(data->status_param, - data->type == TL_WRITE_CONCURRENT_INSERT); + if (data->lock->get_status && + (*data->lock->get_status)(data->status_param, + data->type == TL_WRITE_CONCURRENT_INSERT)) + result= THR_LOCK_ABORTED; check_locks(data->lock,"got wait_for_lock", data->type, 0); } mysql_mutex_unlock(&data->lock->mutex); @@ -703,8 +704,8 @@ thr_lock(THR_LOCK_DATA *data, THR_LOCK_INFO *owner, ulong lock_wait_timeout) if (lock_type == TL_READ_NO_INSERT) lock->read_no_write_count++; check_locks(lock,"read lock with old write lock", lock_type, 0); - if (lock->get_status) - (*lock->get_status)(data->status_param, 0); + if ((lock->get_status) && (*lock->get_status)(data->status_param, 0)) + result= THR_LOCK_ABORTED; statistic_increment(locks_immediate,&THR_LOCK_lock); goto end; } @@ -727,8 +728,8 @@ thr_lock(THR_LOCK_DATA *data, THR_LOCK_INFO *owner, ulong lock_wait_timeout) if (lock_type == TL_READ_NO_INSERT) lock->read_no_write_count++; check_locks(lock,"read lock with no write locks", lock_type, 0); - if (lock->get_status) - (*lock->get_status)(data->status_param, 0); + if ((lock->get_status) && (*lock->get_status)(data->status_param, 0)) + result= THR_LOCK_ABORTED; statistic_increment(locks_immediate,&THR_LOCK_lock); goto end; } @@ -836,9 +837,10 @@ thr_lock(THR_LOCK_DATA *data, THR_LOCK_INFO *owner, ulong lock_wait_timeout) data->prev=lock->write.last; lock->write.last= &data->next; check_locks(lock,"second write lock", lock_type, 0); - if (lock->get_status) - (*lock->get_status)(data->status_param, - lock_type == TL_WRITE_CONCURRENT_INSERT); + if ((lock->get_status) && + (*lock->get_status)(data->status_param, + lock_type == TL_WRITE_CONCURRENT_INSERT)) + result= THR_LOCK_ABORTED; statistic_increment(locks_immediate,&THR_LOCK_lock); goto end; } @@ -871,8 +873,9 @@ thr_lock(THR_LOCK_DATA *data, THR_LOCK_INFO *owner, ulong lock_wait_timeout) (*lock->write.last)=data; /* Add as current write lock */ data->prev=lock->write.last; lock->write.last= &data->next; - if (lock->get_status) - (*lock->get_status)(data->status_param, concurrent_insert); + if ((lock->get_status) && + (*lock->get_status)(data->status_param, concurrent_insert)) + result= THR_LOCK_ABORTED; check_locks(lock,"only write lock", lock_type, 0); statistic_increment(locks_immediate,&THR_LOCK_lock); goto end; @@ -1454,6 +1457,7 @@ my_bool thr_upgrade_write_delay_lock(THR_LOCK_DATA *data, { if (!lock->read.data) /* No read locks */ { /* We have the lock */ + /* For this function, get_status is not allowed to fail */ if (data->lock->get_status) (*data->lock->get_status)(data->status_param, 0); mysql_mutex_unlock(&lock->mutex); @@ -1654,9 +1658,10 @@ static ulong sum=0; /* The following functions is for WRITE_CONCURRENT_INSERT */ -static void test_get_status(void* param __attribute__((unused)), - my_bool concurrent_insert __attribute__((unused))) +static my_bool test_get_status(void* param __attribute__((unused)), + my_bool concurrent_insert __attribute__((unused))) { + return 0; } static void test_update_status(void* param __attribute__((unused))) diff --git a/mysys/thr_mutex.c b/mysys/thr_mutex.c index 568ed3ecb46..2a8e54621c0 100644 --- a/mysys/thr_mutex.c +++ b/mysys/thr_mutex.c @@ -156,7 +156,7 @@ static inline void remove_from_active_list(safe_mutex_t *mp) static int safe_mutex_lazy_init_deadlock_detection(safe_mutex_t *mp) { - if (!my_multi_malloc(MY_FAE | MY_WME, + if (!my_multi_malloc(PSI_NOT_INSTRUMENTED, MY_FAE | MY_WME, &mp->locked_mutex, sizeof(*mp->locked_mutex), &mp->used_mutex, sizeof(*mp->used_mutex), NullS)) { @@ -169,16 +169,12 @@ static int safe_mutex_lazy_init_deadlock_detection(safe_mutex_t *mp) pthread_mutex_lock(&THR_LOCK_mutex); mp->id= ++safe_mutex_id; pthread_mutex_unlock(&THR_LOCK_mutex); - my_hash_init2(mp->locked_mutex, 64, &my_charset_bin, - 128, - offsetof(safe_mutex_deadlock_t, id), - sizeof(mp->id), - 0, 0, 0, HASH_UNIQUE); - my_hash_init2(mp->used_mutex, 64, &my_charset_bin, - 128, - offsetof(safe_mutex_t, id), - sizeof(mp->id), - 0, 0, 0, HASH_UNIQUE); + my_hash_init2(PSI_NOT_INSTRUMENTED, mp->locked_mutex, 64, &my_charset_bin, 128, + offsetof(safe_mutex_deadlock_t, id), sizeof(mp->id), 0, 0, 0, + HASH_UNIQUE); + my_hash_init2(PSI_NOT_INSTRUMENTED, mp->used_mutex, 64, &my_charset_bin, 128, + offsetof(safe_mutex_t, id), sizeof(mp->id), 0, 0, 0, + HASH_UNIQUE); return 0; } @@ -237,6 +233,7 @@ int safe_mutex_lock(safe_mutex_t *mp, myf my_flags, const char *file, int error; DBUG_PRINT("mutex", ("%s (0x%lx) locking", mp->name ? mp->name : "Null", (ulong) mp)); + DBUG_PUSH_EMPTY; pthread_mutex_lock(&mp->global); if (!mp->file) @@ -287,7 +284,7 @@ int safe_mutex_lock(safe_mutex_t *mp, myf my_flags, const char *file, { error= pthread_mutex_trylock(&mp->mutex); if (error == EBUSY) - return error; + goto end; } else error= pthread_mutex_lock(&mp->mutex); @@ -341,7 +338,7 @@ int safe_mutex_lock(safe_mutex_t *mp, myf my_flags, const char *file, safe_mutex_t *mutex; /* Create object to store mutex info */ - if (!(deadlock= my_malloc(sizeof(*deadlock), + if (!(deadlock= my_malloc(PSI_NOT_INSTRUMENTED, sizeof(*deadlock), MYF(MY_ZEROFILL | MY_WME | MY_FAE)))) goto abort_loop; deadlock->name= mp->name; @@ -397,7 +394,10 @@ int safe_mutex_lock(safe_mutex_t *mp, myf my_flags, const char *file, } } - DBUG_PRINT("mutex", ("%s (0x%lx) locked", mp->name, (ulong) mp)); +end: + DBUG_POP_EMPTY; + if (!error) + DBUG_PRINT("mutex", ("%s (0x%lx) locked", mp->name, (ulong) mp)); return error; } diff --git a/mysys/thr_rwlock.c b/mysys/thr_rwlock.c index ea8b73bf8f5..a8711d517f9 100644 --- a/mysys/thr_rwlock.c +++ b/mysys/thr_rwlock.c @@ -21,116 +21,66 @@ #ifdef _WIN32 -static BOOL have_srwlock= FALSE; -/* Prototypes and function pointers for windows functions */ -typedef VOID (WINAPI* srw_func) (PSRWLOCK SRWLock); -typedef BOOLEAN (WINAPI* srw_bool_func) (PSRWLOCK SRWLock); - -static srw_func my_InitializeSRWLock; -static srw_func my_AcquireSRWLockExclusive; -static srw_func my_ReleaseSRWLockExclusive; -static srw_func my_AcquireSRWLockShared; -static srw_func my_ReleaseSRWLockShared; - -static srw_bool_func my_TryAcquireSRWLockExclusive; -static srw_bool_func my_TryAcquireSRWLockShared; - -/** - Check for presence of Windows slim reader writer lock function. - Load function pointers. -*/ - -static void check_srwlock_availability(void) -{ - HMODULE module= GetModuleHandle("kernel32"); - - my_InitializeSRWLock= (srw_func) GetProcAddress(module, - "InitializeSRWLock"); - my_AcquireSRWLockExclusive= (srw_func) GetProcAddress(module, - "AcquireSRWLockExclusive"); - my_AcquireSRWLockShared= (srw_func) GetProcAddress(module, - "AcquireSRWLockShared"); - my_ReleaseSRWLockExclusive= (srw_func) GetProcAddress(module, - "ReleaseSRWLockExclusive"); - my_ReleaseSRWLockShared= (srw_func) GetProcAddress(module, - "ReleaseSRWLockShared"); - my_TryAcquireSRWLockExclusive= (srw_bool_func) GetProcAddress(module, - "TryAcquireSRWLockExclusive"); - my_TryAcquireSRWLockShared= (srw_bool_func) GetProcAddress(module, - "TryAcquireSRWLockShared"); - - /* - We currently require TryAcquireSRWLockExclusive. This API is missing on - Vista, this means SRWLock are only used starting with Win7. - - If "trylock" usage for rwlocks is eliminated from server codebase (it is used - in a single place currently, in query cache), then SRWLock can be enabled on - Vista too. In this case condition below needs to be changed to e.g check - for my_InitializeSRWLock. - */ - - if (my_TryAcquireSRWLockExclusive) - have_srwlock= TRUE; - -} - - -static int srw_init(my_rw_lock_t *rwp) +int my_rw_init(my_rw_lock_t *rwp) { - my_InitializeSRWLock(&rwp->srwlock); + InitializeSRWLock(&rwp->srwlock); rwp->have_exclusive_srwlock = FALSE; return 0; } -static int srw_rdlock(my_rw_lock_t *rwp) +int my_rw_rdlock(my_rw_lock_t *rwp) { - my_AcquireSRWLockShared(&rwp->srwlock); + AcquireSRWLockShared(&rwp->srwlock); return 0; } -static int srw_tryrdlock(my_rw_lock_t *rwp) +int my_rw_tryrdlock(my_rw_lock_t *rwp) { - - if (!my_TryAcquireSRWLockShared(&rwp->srwlock)) + if (!TryAcquireSRWLockShared(&rwp->srwlock)) return EBUSY; return 0; } -static int srw_wrlock(my_rw_lock_t *rwp) +int my_rw_wrlock(my_rw_lock_t *rwp) { - my_AcquireSRWLockExclusive(&rwp->srwlock); + AcquireSRWLockExclusive(&rwp->srwlock); rwp->have_exclusive_srwlock= TRUE; return 0; } - -static int srw_trywrlock(my_rw_lock_t *rwp) +int my_rw_trywrlock(my_rw_lock_t *rwp) { - if (!my_TryAcquireSRWLockExclusive(&rwp->srwlock)) + if (!TryAcquireSRWLockExclusive(&rwp->srwlock)) return EBUSY; rwp->have_exclusive_srwlock= TRUE; return 0; } -static int srw_unlock(my_rw_lock_t *rwp) +int my_rw_unlock(my_rw_lock_t *rwp) { if (rwp->have_exclusive_srwlock) { rwp->have_exclusive_srwlock= FALSE; - my_ReleaseSRWLockExclusive(&rwp->srwlock); + ReleaseSRWLockExclusive(&rwp->srwlock); } else { - my_ReleaseSRWLockShared(&rwp->srwlock); + ReleaseSRWLockShared(&rwp->srwlock); } return 0; } -#endif /*_WIN32 */ +int my_rw_destroy(my_rw_lock_t* rwp) +{ + DBUG_ASSERT(!rwp->have_exclusive_srwlock); + return 0; +} + +#else /* Source base from Sun Microsystems SPILT, simplified for MySQL use @@ -175,22 +125,6 @@ int my_rw_init(my_rw_lock_t *rwp) { pthread_condattr_t cond_attr; -#ifdef _WIN32 - /* - Once initialization is used here rather than in my_init(), in order to - - avoid my_init() pitfalls- (undefined order in which initialization should - run) - - be potentially useful C++ (static constructors) - - just to simplify the API. - Also, the overhead is of my_pthread_once is very small. - */ - static my_pthread_once_t once_control= MY_PTHREAD_ONCE_INIT; - my_pthread_once(&once_control, check_srwlock_availability); - - if (have_srwlock) - return srw_init(rwp); -#endif - pthread_mutex_init( &rwp->lock, MY_MUTEX_INIT_FAST); pthread_condattr_init( &cond_attr ); pthread_cond_init( &rwp->readers, &cond_attr ); @@ -209,10 +143,6 @@ int my_rw_init(my_rw_lock_t *rwp) int my_rw_destroy(my_rw_lock_t *rwp) { -#ifdef _WIN32 - if (have_srwlock) - return 0; /* no destroy function */ -#endif DBUG_ASSERT(rwp->state == 0); pthread_mutex_destroy( &rwp->lock ); pthread_cond_destroy( &rwp->readers ); @@ -223,11 +153,6 @@ int my_rw_destroy(my_rw_lock_t *rwp) int my_rw_rdlock(my_rw_lock_t *rwp) { -#ifdef _WIN32 - if (have_srwlock) - return srw_rdlock(rwp); -#endif - pthread_mutex_lock(&rwp->lock); /* active or queued writers */ @@ -242,12 +167,6 @@ int my_rw_rdlock(my_rw_lock_t *rwp) int my_rw_tryrdlock(my_rw_lock_t *rwp) { int res; - -#ifdef _WIN32 - if (have_srwlock) - return srw_tryrdlock(rwp); -#endif - pthread_mutex_lock(&rwp->lock); if ((rwp->state < 0 ) || rwp->waiters) res= EBUSY; /* Can't get lock */ @@ -263,11 +182,6 @@ int my_rw_tryrdlock(my_rw_lock_t *rwp) int my_rw_wrlock(my_rw_lock_t *rwp) { -#ifdef _WIN32 - if (have_srwlock) - return srw_wrlock(rwp); -#endif - pthread_mutex_lock(&rwp->lock); rwp->waiters++; /* another writer queued */ @@ -289,11 +203,6 @@ int my_rw_trywrlock(my_rw_lock_t *rwp) { int res; -#ifdef _WIN32 - if (have_srwlock) - return srw_trywrlock(rwp); -#endif - pthread_mutex_lock(&rwp->lock); if (rwp->state) res= EBUSY; /* Can't get lock */ @@ -312,11 +221,6 @@ int my_rw_trywrlock(my_rw_lock_t *rwp) int my_rw_unlock(my_rw_lock_t *rwp) { -#ifdef _WIN32 - if (have_srwlock) - return srw_unlock(rwp); -#endif - DBUG_PRINT("rw_unlock", ("state: %d waiters: %d", rwp->state, rwp->waiters)); pthread_mutex_lock(&rwp->lock); @@ -347,7 +251,8 @@ int my_rw_unlock(my_rw_lock_t *rwp) return(0); } -#endif /* defined(NEED_MY_RW_LOCK) */ +#endif /* !defined _WIN32 */ +#endif /* NEED_MY_RW_LOCK*/ int rw_pr_init(rw_pr_lock_t *rwlock) diff --git a/mysys/thr_timer.c b/mysys/thr_timer.c index 1532875d7f3..f87c1f75555 100644 --- a/mysys/thr_timer.c +++ b/mysys/thr_timer.c @@ -85,7 +85,7 @@ my_bool init_thr_timer(uint alloc_timers) /* Create a thread to handle timers */ pthread_attr_init(&thr_attr); pthread_attr_setscope(&thr_attr,PTHREAD_SCOPE_PROCESS); - pthread_attr_setstacksize(&thr_attr,8196); + pthread_attr_setstacksize(&thr_attr,64*1024); thr_timer_inited= 1; if (mysql_thread_create(key_thread_timer, &timer_thread, &thr_attr, timer_handler, NULL)) @@ -141,6 +141,18 @@ void thr_timer_init(thr_timer_t *timer_data, void(*function)(void*), DBUG_VOID_RETURN; } +/* + Make timer periodic + + @param timer_data Timer structure + @param micro_seconds Period +*/ +void thr_timer_set_period(thr_timer_t* timer_data, ulonglong micro_seconds) +{ + DBUG_ENTER("thr_timer_set_period"); + timer_data->period= micro_seconds; + DBUG_VOID_RETURN; +} /* Request timer after X milliseconds @@ -240,18 +252,35 @@ static sig_handler process_timers(struct timespec *now) { void (*function)(void*); void *func_arg; + my_bool is_periodic; timer_data= (thr_timer_t*) queue_top(&timer_queue); function= timer_data->func; func_arg= timer_data->func_arg; + is_periodic= timer_data->period != 0; timer_data->expired= 1; /* Mark expired */ /* We remove timer before calling timer function to allow thread to delete it's timer data any time. + + Deleting timer inside the callback would not work + for periodic timers, they need to be removed from + queue prior to destroying timer_data. */ queue_remove_top(&timer_queue); /* Remove timer */ (*function)(func_arg); /* Inform thread of timeout */ + /* + If timer is periodic, recalculate next expiration time, and + reinsert it into the queue. + */ + if (is_periodic && timer_data->period) + { + set_timespec_nsec(timer_data->expire_time, timer_data->period * 1000); + timer_data->expired= 0; + queue_insert(&timer_queue, (uchar*)timer_data); + } + /* Check if next one has also expired */ timer_data= (thr_timer_t*) queue_top(&timer_queue); if (cmp_timespec(timer_data->expire_time, (*now)) > 0) diff --git a/mysys/tree.c b/mysys/tree.c index 8cae30e2d3e..cd44f779e6f 100644 --- a/mysys/tree.c +++ b/mysys/tree.c @@ -128,7 +128,7 @@ void init_tree(TREE *tree, size_t default_alloc_size, size_t memory_limit, } if (!(tree->with_delete= MY_TEST(my_flags & MY_TREE_WITH_DELETE))) { - init_alloc_root(&tree->mem_root, "tree", default_alloc_size, 0, + init_alloc_root(key_memory_TREE, &tree->mem_root, default_alloc_size, 0, MYF(my_flags)); tree->mem_root.min_malloc= sizeof(TREE_ELEMENT)+tree->size_of_element; } @@ -275,7 +275,7 @@ TREE_ELEMENT *tree_insert(TREE *tree, void *key, uint key_size, key_size+=tree->size_of_element; if (tree->with_delete) - element=(TREE_ELEMENT *) my_malloc(alloc_size, + element=(TREE_ELEMENT *) my_malloc(key_memory_TREE, alloc_size, MYF(tree->my_flags | MY_WME)); else element=(TREE_ELEMENT *) alloc_root(&tree->mem_root,alloc_size); diff --git a/mysys/typelib.c b/mysys/typelib.c index f0037921a87..03f3609b5e8 100644 --- a/mysys/typelib.c +++ b/mysys/typelib.c @@ -182,11 +182,11 @@ const char *get_type(TYPELIB *typelib, uint nr) a integer representation of the supplied string */ -my_ulonglong find_typeset(char *x, TYPELIB *lib, int *err) +my_ulonglong find_typeset(const char *x, TYPELIB *lib, int *err) { my_ulonglong result; int find; - char *i; + const char *i; DBUG_ENTER("find_set"); DBUG_PRINT("enter",("x: '%s' lib: %p", x, lib)); @@ -226,7 +226,7 @@ my_ulonglong find_typeset(char *x, TYPELIB *lib, int *err) NULL otherwise */ -TYPELIB *copy_typelib(MEM_ROOT *root, TYPELIB *from) +TYPELIB *copy_typelib(MEM_ROOT *root, const TYPELIB *from) { TYPELIB *to; uint i; diff --git a/mysys/waiting_threads.c b/mysys/waiting_threads.c index 1441c25cc59..dd60088d534 100644 --- a/mysys/waiting_threads.c +++ b/mysys/waiting_threads.c @@ -395,7 +395,8 @@ static void wt_resource_create(uchar *arg) bzero(rc, sizeof(*rc)); rc_rwlock_init(rc); mysql_cond_init(key_WT_RESOURCE_cond, &rc->cond, 0); - my_init_dynamic_array(&rc->owners, sizeof(WT_THD *), 0, 5, MYF(0)); + my_init_dynamic_array(PSI_INSTRUMENT_ME, &rc->owners, + sizeof(WT_THD *), 0, 5, MYF(0)); DBUG_VOID_RETURN; } @@ -508,7 +509,8 @@ void wt_thd_lazy_init(WT_THD *thd, const ulong *ds, const ulong *ts, thd->deadlock_search_depth_long= dl; thd->timeout_long= tl; /* dynamic array is also initialized lazily - without memory allocations */ - my_init_dynamic_array(&thd->my_resources, sizeof(WT_RESOURCE *), 0, 5, MYF(0)); + my_init_dynamic_array(PSI_INSTRUMENT_ME, &thd->my_resources, + sizeof(WT_RESOURCE *), 0, 5, MYF(0)); #ifndef DBUG_OFF thd->name= my_thread_name(); #endif |