From 37946731110246b89607b85b391f32b0f390524e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 21 Jun 2022 16:59:49 +0300 Subject: MDEV-28836: Memory alignment cleanup Table_cache_instance: Define the structure aligned at the CPU cache line, and remove a pad[] data member. Krunal Bauskar reported this to improve performance on ARMv8. aligned_malloc(): Wrapper for the Microsoft _aligned_malloc() and the ISO/IEC 9899:2011 aligned_alloc(). Note: The parameters are in the Microsoft order (size, alignment), opposite of aligned_alloc(alignment, size). Note: The standard defines that size must be an integer multiple of alignment. It is enforced by AddressSanitizer but not by GNU libc on Linux. aligned_free(): Wrapper for the Microsoft _aligned_free() and the standard free(). HAVE_ALIGNED_ALLOC: A new test. Unfortunately, support for aligned_alloc() may still be missing on some platforms. We will fall back to posix_memalign() for those cases. HAVE_MEMALIGN: Remove, along with any use of the nonstandard memalign(). PFS_ALIGNEMENT (sic): Removed; we will use CPU_LEVEL1_DCACHE_LINESIZE. PFS_ALIGNED: Defined using the C++11 keyword alignas. buf_pool_t::page_hash_table::create(), lock_sys_t::hash_table::create(): lock_sys_t::hash_table::resize(): Pad the allocation size to an integer multiple of the alignment. Reviewed by: Vladislav Vaintroub --- cmake/os/WindowsCache.cmake | 2 +- config.h.cmake | 2 +- configure.cmake | 2 +- include/aligned.h | 34 ++++++++++++++++ include/my_global.h | 2 +- mysys/my_lockmem.c | 6 ++- sql/table_cache.cc | 11 ++++-- storage/innobase/buf/buf0buf.cc | 7 ++-- storage/innobase/include/buf0buf.h | 21 ---------- storage/innobase/include/univ.i | 1 + storage/innobase/include/ut0pool.h | 9 +---- storage/innobase/lock/lock0lock.cc | 14 ++++--- storage/perfschema/pfs_digest.cc | 3 +- storage/perfschema/pfs_events_stages.h | 3 +- storage/perfschema/pfs_events_statements.h | 3 +- storage/perfschema/pfs_events_waits.h | 4 +- storage/perfschema/pfs_global.cc | 56 ++++----------------------- storage/perfschema/pfs_global.h | 23 ++--------- storage/perfschema/pfs_host.cc | 3 +- storage/perfschema/pfs_instr.cc | 3 +- storage/perfschema/unittest/stub_pfs_global.h | 16 +------- tpool/tpool_generic.cc | 19 +++------ 22 files changed, 94 insertions(+), 150 deletions(-) create mode 100644 include/aligned.h diff --git a/cmake/os/WindowsCache.cmake b/cmake/os/WindowsCache.cmake index 9b855dbb42b..316b42af796 100644 --- a/cmake/os/WindowsCache.cmake +++ b/cmake/os/WindowsCache.cmake @@ -23,6 +23,7 @@ IF(MSVC) SET(BFD_H_EXISTS 0 CACHE INTERNAL "") SET(HAVE_ACCESS 1 CACHE INTERNAL "") SET(HAVE_ALARM CACHE INTERNAL "") +SET(HAVE_ALIGNED_ALLOC CACHE INTERNAL "") SET(HAVE_ALLOCA_H CACHE INTERNAL "") SET(HAVE_ARPA_INET_H CACHE INTERNAL "") SET(HAVE_BACKTRACE CACHE INTERNAL "") @@ -93,7 +94,6 @@ SET(HAVE_MALLINFO CACHE INTERNAL "") SET(HAVE_MALLINFO2 CACHE INTERNAL "") SET(HAVE_MALLOC_H 1 CACHE INTERNAL "") SET(HAVE_MALLOC_ZONE CACHE INTERNAL "") -SET(HAVE_MEMALIGN CACHE INTERNAL "") SET(HAVE_MEMCPY 1 CACHE INTERNAL "") SET(HAVE_MEMMOVE 1 CACHE INTERNAL "") SET(HAVE_MEMORY_H 1 CACHE INTERNAL "") diff --git a/config.h.cmake b/config.h.cmake index 828f1fbb0d2..4ff2f2fbf12 100644 --- a/config.h.cmake +++ b/config.h.cmake @@ -19,6 +19,7 @@ /* Headers we may want to use. */ #cmakedefine STDC_HEADERS 1 #cmakedefine _GNU_SOURCE 1 +#cmakedefine HAVE_ALIGNED_ALLOC 1 #cmakedefine HAVE_ALLOCA_H 1 #cmakedefine HAVE_ARPA_INET_H 1 #cmakedefine HAVE_ASM_TERMBITS_H 1 @@ -163,7 +164,6 @@ #cmakedefine HAVE_LRAND48 1 #cmakedefine HAVE_LOCALTIME_R 1 #cmakedefine HAVE_LSTAT 1 -#cmakedefine HAVE_MEMALIGN 1 /* #cmakedefine HAVE_MLOCK 1 see Bug#54662 */ #cmakedefine HAVE_NL_LANGINFO 1 #cmakedefine HAVE_MADVISE 1 diff --git a/configure.cmake b/configure.cmake index e940bec5db0..19ceaa06220 100644 --- a/configure.cmake +++ b/configure.cmake @@ -324,6 +324,7 @@ ENDIF() CHECK_FUNCTION_EXISTS (accept4 HAVE_ACCEPT4) CHECK_FUNCTION_EXISTS (access HAVE_ACCESS) CHECK_FUNCTION_EXISTS (alarm HAVE_ALARM) +CHECK_FUNCTION_EXISTS (aligned_alloc HAVE_ALIGNED_ALLOC) SET(HAVE_ALLOCA 1) CHECK_FUNCTION_EXISTS (backtrace HAVE_BACKTRACE) CHECK_FUNCTION_EXISTS (backtrace_symbols HAVE_BACKTRACE_SYMBOLS) @@ -420,7 +421,6 @@ CHECK_FUNCTION_EXISTS (thr_setconcurrency HAVE_THR_SETCONCURRENCY) CHECK_FUNCTION_EXISTS (thr_yield HAVE_THR_YIELD) CHECK_FUNCTION_EXISTS (vasprintf HAVE_VASPRINTF) CHECK_FUNCTION_EXISTS (vsnprintf HAVE_VSNPRINTF) -CHECK_FUNCTION_EXISTS (memalign HAVE_MEMALIGN) CHECK_FUNCTION_EXISTS (nl_langinfo HAVE_NL_LANGINFO) IF(HAVE_SYS_EVENT_H) diff --git a/include/aligned.h b/include/aligned.h new file mode 100644 index 00000000000..ad2c7532b26 --- /dev/null +++ b/include/aligned.h @@ -0,0 +1,34 @@ +/* + Copyright (c) 2022, MariaDB Corporation. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */ + +inline void *aligned_malloc(size_t size, size_t alignment) +{ +#ifdef _WIN32 + return _aligned_malloc(size, alignment); +#elif defined HAVE_ALIGNED_ALLOC + return aligned_alloc(alignment, size); +#else + void *result; + if (posix_memalign(&result, alignment, size)) + result= NULL; + return result; +#endif +} + +inline void aligned_free(void *ptr) +{ + IF_WIN(_aligned_free,free)(ptr); +} diff --git a/include/my_global.h b/include/my_global.h index 224909116dd..2eed45e8407 100644 --- a/include/my_global.h +++ b/include/my_global.h @@ -1,6 +1,6 @@ /* Copyright (c) 2001, 2013, Oracle and/or its affiliates. - Copyright (c) 2009, 2021, MariaDB Corporation. + Copyright (c) 2009, 2022, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/mysys/my_lockmem.c b/mysys/my_lockmem.c index e159502a278..732aef1fdac 100644 --- a/mysys/my_lockmem.c +++ b/mysys/my_lockmem.c @@ -1,4 +1,5 @@ /* Copyright (c) 2000, 2013, Oracle and/or its affiliates. All rights reserved. + Copyright (c) 2022, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -17,6 +18,7 @@ #include "mysys_priv.h" #include "mysys_err.h" +#include "aligned.h" #include #ifdef HAVE_MLOCK @@ -39,7 +41,7 @@ uchar *my_malloc_lock(uint size,myf MyFlags) DBUG_ENTER("my_malloc_lock"); size=((size-1) & ~(pagesize-1))+pagesize; - if (!(ptr=memalign(pagesize,size))) + if (!(ptr=aligned_malloc(size,pagesize))) { if (MyFlags & (MY_FAE+MY_WME)) my_error(EE_OUTOFMEMORY, MYF(ME_BELL+ME_FATAL), size); @@ -91,7 +93,7 @@ void my_free_lock(uchar *ptr) } mysql_mutex_unlock(&THR_LOCK_malloc); my_free(element); - free(ptr); /* Free even if not locked */ + aligned_free(ptr); /* Free even if not locked */ } #endif /* HAVE_MLOCK */ diff --git a/sql/table_cache.cc b/sql/table_cache.cc index 2e9fb34c17e..8fb3a559d86 100644 --- a/sql/table_cache.cc +++ b/sql/table_cache.cc @@ -1,5 +1,5 @@ /* Copyright (c) 2000, 2012, Oracle and/or its affiliates. - Copyright (c) 2010, 2011 Monty Program Ab + Copyright (c) 2010, 2022, MariaDB Corporation. Copyright (C) 2013 Sergey Vojtovich and MariaDB Foundation This program is free software; you can redistribute it and/or modify @@ -50,6 +50,7 @@ #include "lf.h" #include "table.h" #include "sql_base.h" +#include "aligned.h" /** Configuration. */ @@ -122,6 +123,7 @@ struct Table_cache_instance records, Share_free_tables::List (TABLE::prev and TABLE::next), TABLE::in_use. */ + alignas(CPU_LEVEL1_DCACHE_LINESIZE) mysql_mutex_t LOCK_table_cache; I_P_List , @@ -130,11 +132,10 @@ struct Table_cache_instance ulong records; uint mutex_waits; uint mutex_nowaits; - /** Avoid false sharing between instances */ - char pad[CPU_LEVEL1_DCACHE_LINESIZE]; Table_cache_instance(): records(0), mutex_waits(0), mutex_nowaits(0) { + static_assert(!(sizeof(*this) % CPU_LEVEL1_DCACHE_LINESIZE), "alignment"); mysql_mutex_init(key_LOCK_table_cache, &LOCK_table_cache, MY_MUTEX_INIT_FAST); } @@ -146,6 +147,10 @@ struct Table_cache_instance DBUG_ASSERT(records == 0); } + static void *operator new[](size_t size) + { return aligned_malloc(size, CPU_LEVEL1_DCACHE_LINESIZE); } + static void operator delete[](void *ptr) { aligned_free(ptr); } + /** Lock table cache mutex and check contention. diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 706e36dd8e6..baee1e47159 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -1085,9 +1085,10 @@ inline const buf_block_t *buf_pool_t::chunk_t::not_freed() const void buf_pool_t::page_hash_table::create(ulint n) { n_cells= ut_find_prime(n); - const size_t size= pad(n_cells) * sizeof *array; - void* v= aligned_malloc(size, CPU_LEVEL1_DCACHE_LINESIZE); - memset(v, 0, size); + const size_t size= MY_ALIGN(pad(n_cells) * sizeof *array, + CPU_LEVEL1_DCACHE_LINESIZE); + void *v= aligned_malloc(size, CPU_LEVEL1_DCACHE_LINESIZE); + memset_aligned(v, 0, size); array= static_cast(v); } diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index 6f1bfa4cfcd..87236415150 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -332,27 +332,6 @@ buf_page_is_corrupted( ulint fsp_flags) MY_ATTRIBUTE((warn_unused_result)); -inline void *aligned_malloc(size_t size, size_t align) -{ -#ifdef _MSC_VER - return _aligned_malloc(size, align); -#else - void *result; - if (posix_memalign(&result, align, size)) - result= NULL; - return result; -#endif -} - -inline void aligned_free(void *ptr) -{ -#ifdef _MSC_VER - _aligned_free(ptr); -#else - free(ptr); -#endif -} - /** Read the key version from the page. In full crc32 format, key version is stored at {0-3th} bytes. In other format, it is stored in 26th position. diff --git a/storage/innobase/include/univ.i b/storage/innobase/include/univ.i index c2b60a9ea41..da9273c041e 100644 --- a/storage/innobase/include/univ.i +++ b/storage/innobase/include/univ.i @@ -67,6 +67,7 @@ support cross-platform development and expose comonly used SQL names. */ #include #include "my_counter.h" +#include "aligned.h" #include #include diff --git a/storage/innobase/include/ut0pool.h b/storage/innobase/include/ut0pool.h index ccea69fe549..63628cc169f 100644 --- a/storage/innobase/include/ut0pool.h +++ b/storage/innobase/include/ut0pool.h @@ -64,15 +64,8 @@ struct Pool { ut_a(m_start == 0); -#ifdef _MSC_VER m_start = static_cast( - _aligned_malloc(m_size, CPU_LEVEL1_DCACHE_LINESIZE)); -#else - void* start; - ut_a(!posix_memalign(&start, CPU_LEVEL1_DCACHE_LINESIZE, - m_size)); - m_start = static_cast(start); -#endif + aligned_malloc(m_size, CPU_LEVEL1_DCACHE_LINESIZE)); memset_aligned( m_start, 0, m_size); diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc index 91791fb8a3a..9323117eff5 100644 --- a/storage/innobase/lock/lock0lock.cc +++ b/storage/innobase/lock/lock0lock.cc @@ -75,9 +75,10 @@ struct TableLockGetNode void lock_sys_t::hash_table::create(ulint n) { n_cells= ut_find_prime(n); - const size_t size= pad(n_cells) * sizeof *array; - void* v= aligned_malloc(size, CPU_LEVEL1_DCACHE_LINESIZE); - memset(v, 0, size); + const size_t size= MY_ALIGN(pad(n_cells) * sizeof *array, + CPU_LEVEL1_DCACHE_LINESIZE); + void *v= aligned_malloc(size, CPU_LEVEL1_DCACHE_LINESIZE); + memset_aligned(v, 0, size); array= static_cast(v); } @@ -87,9 +88,10 @@ void lock_sys_t::hash_table::resize(ulint n) { ut_ad(lock_sys.is_writer()); ulint new_n_cells= ut_find_prime(n); - const size_t size= pad(new_n_cells) * sizeof *array; - void* v= aligned_malloc(size, CPU_LEVEL1_DCACHE_LINESIZE); - memset(v, 0, size); + const size_t size= MY_ALIGN(pad(new_n_cells) * sizeof *array, + CPU_LEVEL1_DCACHE_LINESIZE); + void *v= aligned_malloc(size, CPU_LEVEL1_DCACHE_LINESIZE); + memset_aligned(v, 0, size); hash_cell_t *new_array= static_cast(v); for (auto i= pad(n_cells); i--; ) diff --git a/storage/perfschema/pfs_digest.cc b/storage/perfschema/pfs_digest.cc index 43375c753dd..0dff583fa6d 100644 --- a/storage/perfschema/pfs_digest.cc +++ b/storage/perfschema/pfs_digest.cc @@ -1,4 +1,5 @@ /* Copyright (c) 2008, 2021, Oracle and/or its affiliates. + Copyright (c) 2022, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License, version 2.0, @@ -56,7 +57,7 @@ bool flag_statements_digest= true; Current index in Stat array where new record is to be inserted. index 0 is reserved for "all else" case when entire array is full. */ -static PFS_ALIGNED PFS_cacheline_uint32 digest_monotonic_index; +PFS_ALIGNED static PFS_cacheline_uint32 digest_monotonic_index; bool digest_full= false; diff --git a/storage/perfschema/pfs_events_stages.h b/storage/perfschema/pfs_events_stages.h index eeae350ddf4..0fa7bc8de51 100644 --- a/storage/perfschema/pfs_events_stages.h +++ b/storage/perfschema/pfs_events_stages.h @@ -1,4 +1,5 @@ /* Copyright (c) 2010, 2021, Oracle and/or its affiliates. + Copyright (c) 2022, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License, version 2.0, @@ -49,7 +50,7 @@ extern bool flag_events_stages_history; extern bool flag_events_stages_history_long; extern bool events_stages_history_long_full; -extern PFS_ALIGNED PFS_cacheline_uint32 events_stages_history_long_index; +PFS_ALIGNED extern PFS_cacheline_uint32 events_stages_history_long_index; extern PFS_events_stages *events_stages_history_long_array; extern ulong events_stages_history_long_size; diff --git a/storage/perfschema/pfs_events_statements.h b/storage/perfschema/pfs_events_statements.h index 2b2426ef7d1..b03afb8a768 100644 --- a/storage/perfschema/pfs_events_statements.h +++ b/storage/perfschema/pfs_events_statements.h @@ -1,4 +1,5 @@ /* Copyright (c) 2010, 2021, Oracle and/or its affiliates. + Copyright (c) 2022, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License, version 2.0, @@ -132,7 +133,7 @@ extern bool flag_events_statements_history; extern bool flag_events_statements_history_long; extern bool events_statements_history_long_full; -extern PFS_ALIGNED PFS_cacheline_uint32 events_statements_history_long_index; +PFS_ALIGNED extern PFS_cacheline_uint32 events_statements_history_long_index; extern PFS_events_statements *events_statements_history_long_array; extern size_t events_statements_history_long_size; diff --git a/storage/perfschema/pfs_events_waits.h b/storage/perfschema/pfs_events_waits.h index 8a7355f2d32..ad992b4b0bd 100644 --- a/storage/perfschema/pfs_events_waits.h +++ b/storage/perfschema/pfs_events_waits.h @@ -1,5 +1,5 @@ /* Copyright (c) 2008, 2021, Oracle and/or its affiliates.. - Copyright (c) 2017, 2019, MariaDB Corporation. + Copyright (c) 2017, 2012, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License, version 2.0, @@ -127,7 +127,7 @@ extern bool flag_global_instrumentation; extern bool flag_thread_instrumentation; extern bool events_waits_history_long_full; -extern PFS_ALIGNED PFS_cacheline_uint32 events_waits_history_long_index; +PFS_ALIGNED extern PFS_cacheline_uint32 events_waits_history_long_index; extern PFS_events_waits *events_waits_history_long_array; extern ulong events_waits_history_long_size; diff --git a/storage/perfschema/pfs_global.cc b/storage/perfschema/pfs_global.cc index 1f9d24eea4e..e87941448a6 100644 --- a/storage/perfschema/pfs_global.cc +++ b/storage/perfschema/pfs_global.cc @@ -1,5 +1,6 @@ /* Copyright (c) 2008, 2021, Oracle and/or its affiliates. All rights reserved. + Copyright (c) 2022, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License, version 2.0, @@ -30,6 +31,8 @@ #include "pfs_global.h" #include "pfs_builtin_memory.h" #include "log.h" +#include "aligned.h" +#include "assume_aligned.h" #include #include @@ -46,9 +49,6 @@ #ifdef HAVE_NETINET_IN_H #include #endif -#ifdef HAVE_MALLOC_H -#include -#endif bool pfs_initialized= false; @@ -62,41 +62,16 @@ void *pfs_malloc(PFS_builtin_memory_class *klass, size_t size, myf flags) assert(klass != NULL); assert(size > 0); - void *ptr= NULL; + const size_t aligned_size= MY_ALIGN(size, CPU_LEVEL1_DCACHE_LINESIZE); -#ifdef PFS_ALIGNEMENT -#ifdef HAVE_POSIX_MEMALIGN - /* Linux */ - if (unlikely(posix_memalign(& ptr, PFS_ALIGNEMENT, size))) - return NULL; -#else -#ifdef HAVE_MEMALIGN - /* Solaris */ - ptr= memalign(PFS_ALIGNEMENT, size); - if (unlikely(ptr == NULL)) - return NULL; -#else -#ifdef HAVE_ALIGNED_MALLOC - /* Windows */ - ptr= _aligned_malloc(size, PFS_ALIGNEMENT); + void *ptr= aligned_malloc(aligned_size, CPU_LEVEL1_DCACHE_LINESIZE); if (unlikely(ptr == NULL)) return NULL; -#else -#error "Missing implementation for PFS_ALIGNENT" -#endif /* HAVE_ALIGNED_MALLOC */ -#endif /* HAVE_MEMALIGN */ -#endif /* HAVE_POSIX_MEMALIGN */ -#else /* PFS_ALIGNMENT */ - /* Everything else */ - ptr= malloc(size); - if (unlikely(ptr == NULL)) - return NULL; -#endif klass->count_alloc(size); if (flags & MY_ZEROFILL) - memset(ptr, 0, size); + memset_aligned(ptr, 0, aligned_size); return ptr; } @@ -105,24 +80,7 @@ void pfs_free(PFS_builtin_memory_class *klass, size_t size, void *ptr) if (ptr == NULL) return; -#ifdef HAVE_POSIX_MEMALIGN - /* Allocated with posix_memalign() */ - free(ptr); -#else -#ifdef HAVE_MEMALIGN - /* Allocated with memalign() */ - free(ptr); -#else -#ifdef HAVE_ALIGNED_MALLOC - /* Allocated with _aligned_malloc() */ - _aligned_free(ptr); -#else - /* Allocated with malloc() */ - free(ptr); -#endif /* HAVE_ALIGNED_MALLOC */ -#endif /* HAVE_MEMALIGN */ -#endif /* HAVE_POSIX_MEMALIGN */ - + aligned_free(ptr); klass->count_free(size); } diff --git a/storage/perfschema/pfs_global.h b/storage/perfschema/pfs_global.h index 48a5587839e..6bce7c347cf 100644 --- a/storage/perfschema/pfs_global.h +++ b/storage/perfschema/pfs_global.h @@ -1,4 +1,5 @@ /* Copyright (c) 2008, 2021, Oracle and/or its affiliates. + Copyright (c) 2022, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License, version 2.0, @@ -35,23 +36,7 @@ extern bool pfs_initialized; /** Total memory allocated by the performance schema, in bytes. */ extern size_t pfs_allocated_memory; -#if defined(HAVE_POSIX_MEMALIGN) || defined(HAVE_MEMALIGN) || defined(HAVE_ALIGNED_MALLOC) -#define PFS_ALIGNEMENT CPU_LEVEL1_DCACHE_LINESIZE -#define PFS_ALIGNED MY_ALIGNED(PFS_ALIGNEMENT) -#else -/* - Known platforms that do not provide aligned memory: - - MacOSX Darwin (osx10.5) - For these platforms, compile without the alignment optimization. -*/ -#define PFS_ALIGNED -#endif /* HAVE_POSIX_MEMALIGN || HAVE_MEMALIGN || HAVE_ALIGNED_MALLOC */ - -#ifdef CPU_LEVEL1_DCACHE_LINESIZE -#define PFS_CACHE_LINE_SIZE CPU_LEVEL1_DCACHE_LINESIZE -#else -#define PFS_CACHE_LINE_SIZE 128 -#endif +#define PFS_ALIGNED alignas(CPU_LEVEL1_DCACHE_LINESIZE) /** A uint32 variable, guaranteed to be alone in a CPU cache line. @@ -60,7 +45,7 @@ extern size_t pfs_allocated_memory; struct PFS_cacheline_uint32 { uint32 m_u32; - char m_full_cache_line[PFS_CACHE_LINE_SIZE - sizeof(uint32)]; + char m_full_cache_line[CPU_LEVEL1_DCACHE_LINESIZE - sizeof(uint32)]; PFS_cacheline_uint32() : m_u32(0) @@ -74,7 +59,7 @@ struct PFS_cacheline_uint32 struct PFS_cacheline_uint64 { uint64 m_u64; - char m_full_cache_line[PFS_CACHE_LINE_SIZE - sizeof(uint64)]; + char m_full_cache_line[CPU_LEVEL1_DCACHE_LINESIZE - sizeof(uint64)]; PFS_cacheline_uint64() : m_u64(0) diff --git a/storage/perfschema/pfs_host.cc b/storage/perfschema/pfs_host.cc index 7d7af7700c2..cbd97e69041 100644 --- a/storage/perfschema/pfs_host.cc +++ b/storage/perfschema/pfs_host.cc @@ -1,4 +1,5 @@ /* Copyright (c) 2010, 2021, Oracle and/or its affiliates. + Copyright (c) 2022, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License, version 2.0, @@ -135,7 +136,7 @@ static void set_host_key(PFS_host_key *key, PFS_host *find_or_create_host(PFS_thread *thread, const char *hostname, uint hostname_length) { - static PFS_ALIGNED PFS_cacheline_uint32 monotonic; + PFS_ALIGNED static PFS_cacheline_uint32 monotonic; LF_PINS *pins= get_host_hash_pins(thread); if (unlikely(pins == NULL)) diff --git a/storage/perfschema/pfs_instr.cc b/storage/perfschema/pfs_instr.cc index 41b1b24295a..a2fcaa9222c 100644 --- a/storage/perfschema/pfs_instr.cc +++ b/storage/perfschema/pfs_instr.cc @@ -1,4 +1,5 @@ /* Copyright (c) 2008, 2021, Oracle and/or its affiliates. + Copyright (c) 2022, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License, version 2.0, @@ -88,7 +89,7 @@ PFS_stage_stat *global_instr_class_stages_array= NULL; PFS_statement_stat *global_instr_class_statements_array= NULL; PFS_memory_stat *global_instr_class_memory_array= NULL; -static PFS_ALIGNED PFS_cacheline_uint64 thread_internal_id_counter; +PFS_ALIGNED static PFS_cacheline_uint64 thread_internal_id_counter; /** Hash table for instrumented files. */ LF_HASH pfs_filename_hash; diff --git a/storage/perfschema/unittest/stub_pfs_global.h b/storage/perfschema/unittest/stub_pfs_global.h index 6d10e29161d..3a3aedb537e 100644 --- a/storage/perfschema/unittest/stub_pfs_global.h +++ b/storage/perfschema/unittest/stub_pfs_global.h @@ -25,9 +25,7 @@ #include #include #include -#ifdef HAVE_MEMALIGN -# include -#endif +#include "aligned.h" bool pfs_initialized= false; size_t pfs_allocated_memory_size= 0; @@ -49,17 +47,7 @@ void *pfs_malloc(PFS_builtin_memory_class *klass, size_t size, myf) if (--stub_alloc_fails_after_count <= 0) return NULL; -#ifndef PFS_ALIGNEMENT - void *ptr= malloc(size); -#elif defined HAVE_MEMALIGN - void *ptr= memalign(PFS_ALIGNEMENT, size); -#elif defined HAVE_ALIGNED_MALLOC - void *ptr= _aligned_malloc(size, PFS_ALIGNEMENT); -#else - void *ptr; - if (posix_memalign(&ptr, PFS_ALIGNEMENT, size)) - ptr= NULL; -#endif + void *ptr= aligned_malloc(size, CPU_LEVEL1_DCACHE_LINESIZE); if (ptr != NULL) memset(ptr, 0, size); return ptr; diff --git a/tpool/tpool_generic.cc b/tpool/tpool_generic.cc index 22d29d6400b..dfd780e7ea3 100644 --- a/tpool/tpool_generic.cc +++ b/tpool/tpool_generic.cc @@ -33,6 +33,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111 - 1301 USA*/ #include #include #include +#include "aligned.h" namespace tpool { @@ -180,23 +181,13 @@ struct alignas(CPU_LEVEL1_DCACHE_LINESIZE) worker_data {} /*Define custom new/delete because of overaligned structure. */ - void* operator new(size_t size) + static void *operator new(size_t size) { -#ifdef _WIN32 - return _aligned_malloc(size, CPU_LEVEL1_DCACHE_LINESIZE); -#else - void* ptr; - int ret = posix_memalign(&ptr, CPU_LEVEL1_DCACHE_LINESIZE, size); - return ret ? 0 : ptr; -#endif + return aligned_malloc(size, CPU_LEVEL1_DCACHE_LINESIZE); } - void operator delete(void* p) + static void operator delete(void* p) { -#ifdef _WIN32 - _aligned_free(p); -#else - free(p); -#endif + aligned_free(p); } }; -- cgit v1.2.1