diff options
author | Marko Mäkelä <marko.makela@mariadb.com> | 2022-03-14 11:30:32 +0200 |
---|---|---|
committer | Marko Mäkelä <marko.makela@mariadb.com> | 2022-03-14 11:30:32 +0200 |
commit | e67d46e4a16a6550734dbff8d1a28c578ab3b23d (patch) | |
tree | 42746de58b0e919847c304c523047fa7032891d4 | |
parent | 810ed88c65c6b8984de919ac7763e7e6c115048e (diff) | |
parent | 572e34304ef375651ca9d621d77f7eb61c66b8e0 (diff) | |
download | mariadb-git-e67d46e4a16a6550734dbff8d1a28c578ab3b23d.tar.gz |
Merge 10.6 into 10.7
-rw-r--r-- | include/my_valgrind.h | 10 | ||||
-rw-r--r-- | mysys/my_lib.c | 20 | ||||
-rw-r--r-- | mysys/my_symlink.c | 7 | ||||
-rw-r--r-- | sql/datadict.cc | 2 | ||||
-rw-r--r-- | sql/ddl_log.cc | 8 | ||||
-rw-r--r-- | sql/discover.cc | 3 | ||||
-rw-r--r-- | sql/parse_file.cc | 2 | ||||
-rw-r--r-- | sql/sql_parse.cc | 2 | ||||
-rw-r--r-- | storage/innobase/buf/buf0rea.cc | 2 | ||||
-rw-r--r-- | storage/innobase/os/os0file.cc | 18 | ||||
-rw-r--r-- | storage/innobase/trx/trx0purge.cc | 4 | ||||
-rw-r--r-- | storage/maria/s3_func.h | 1 | ||||
-rw-r--r-- | storage/myisam/mi_info.c | 3 | ||||
-rw-r--r-- | strings/my_vsnprintf.c | 3 | ||||
-rw-r--r-- | tpool/aio_liburing.cc | 2 | ||||
-rw-r--r-- | tpool/aio_linux.cc | 2 | ||||
-rw-r--r-- | tpool/aio_simulated.cc | 27 | ||||
-rw-r--r-- | tpool/tpool.h | 6 | ||||
-rw-r--r-- | tpool/tpool_generic.cc | 52 |
19 files changed, 124 insertions, 50 deletions
diff --git a/include/my_valgrind.h b/include/my_valgrind.h index 260521d4d4b..a24ad597d36 100644 --- a/include/my_valgrind.h +++ b/include/my_valgrind.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2010, 2020, MariaDB Corporation. +/* Copyright (C) 2010, 2022, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -37,6 +37,11 @@ # define MEM_GET_VBITS(a,b,len) __msan_copy_shadow(b,a,len) # define MEM_SET_VBITS(a,b,len) __msan_copy_shadow(a,b,len) # define REDZONE_SIZE 8 +# ifdef __linux__ +# define MSAN_STAT_WORKAROUND(st) MEM_MAKE_DEFINED(st, sizeof(*st)) +# else +# define MSAN_STAT_WORKAROUND(st) ((void) 0) +# endif #elif defined(HAVE_VALGRIND_MEMCHECK_H) && defined(HAVE_valgrind) # include <valgrind/memcheck.h> # define HAVE_MEM_CHECK @@ -49,6 +54,7 @@ # define MEM_GET_VBITS(a,b,len) VALGRIND_GET_VBITS(a,b,len) # define MEM_SET_VBITS(a,b,len) VALGRIND_SET_VBITS(a,b,len) # define REDZONE_SIZE 8 +# define MSAN_STAT_WORKAROUND(st) ((void) 0) #elif defined(__SANITIZE_ADDRESS__) && (!defined(_MSC_VER) || defined (__clang__)) # include <sanitizer/asan_interface.h> /* How to do manual poisoning: @@ -62,6 +68,7 @@ https://github.com/google/sanitizers/wiki/AddressSanitizerManualPoisoning */ # define MEM_CHECK_DEFINED(a,len) ((void) 0) # define MEM_GET_VBITS(a,b,len) ((void) 0) # define MEM_SET_VBITS(a,b,len) ((void) 0) +# define MSAN_STAT_WORKAROUND(st) ((void) 0) # define REDZONE_SIZE 8 #else # define MEM_UNDEFINED(a,len) ((void) 0) @@ -73,6 +80,7 @@ https://github.com/google/sanitizers/wiki/AddressSanitizerManualPoisoning */ # define MEM_GET_VBITS(a,b,len) ((void) 0) # define MEM_SET_VBITS(a,b,len) ((void) 0) # define REDZONE_SIZE 0 +# define MSAN_STAT_WORKAROUND(st) ((void) 0) #endif /* __has_feature(memory_sanitizer) */ #ifdef HAVE_valgrind diff --git a/mysys/my_lib.c b/mysys/my_lib.c index ca50699b4c3..fb03f0aa5c2 100644 --- a/mysys/my_lib.c +++ b/mysys/my_lib.c @@ -1,5 +1,5 @@ /* Copyright (c) 2000, 2013, Oracle and/or its affiliates. All rights reserved. - Copyright (c) 2008, 2020, MariaDB Corporation. + Copyright (c) 2008, 2022, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -332,6 +332,13 @@ int my_fstat(File Filedes, MY_STAT *stat_area, DBUG_PRINT("my",("fd: %d MyFlags: %lu", Filedes, MyFlags)); #ifdef _WIN32 DBUG_RETURN(my_win_fstat(Filedes, stat_area)); +#elif defined HAVE_valgrind + { + int s= fstat(Filedes, stat_area); + if (!s) + MSAN_STAT_WORKAROUND(stat_area); + DBUG_RETURN(s); + } #else DBUG_RETURN(fstat(Filedes, (struct stat *) stat_area)); #endif @@ -350,11 +357,14 @@ MY_STAT *my_stat(const char *path, MY_STAT *stat_area, myf my_flags) my_flags))) goto error; #ifndef _WIN32 - if (! stat((char *) path, (struct stat *) stat_area) ) - DBUG_RETURN(stat_area); + if (!stat((char *) path, (struct stat *) stat_area)) + { + MSAN_STAT_WORKAROUND(stat_area); + DBUG_RETURN(stat_area); + } #else - if (! my_win_stat(path, stat_area) ) - DBUG_RETURN(stat_area); + if (!my_win_stat(path, stat_area)) + DBUG_RETURN(stat_area); #endif DBUG_PRINT("error",("Got errno: %d from stat", errno)); my_errno= errno; diff --git a/mysys/my_symlink.c b/mysys/my_symlink.c index 323ae69a39c..8238e501e7f 100644 --- a/mysys/my_symlink.c +++ b/mysys/my_symlink.c @@ -1,6 +1,6 @@ /* Copyright (c) 2001, 2011, Oracle and/or its affiliates - Copyright (c) 2010, 2017, MariaDB + Copyright (c) 2010, 2022, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -113,7 +113,10 @@ int my_is_symlink(const char *filename __attribute__((unused))) { #if defined (HAVE_LSTAT) && defined (S_ISLNK) struct stat stat_buff; - return !lstat(filename, &stat_buff) && S_ISLNK(stat_buff.st_mode); + if (lstat(filename, &stat_buff)) + return 0; + MSAN_STAT_WORKAROUND(&stat_buff); + return !!S_ISLNK(stat_buff.st_mode); #elif defined (_WIN32) DWORD dwAttr = GetFileAttributes(filename); return (dwAttr != INVALID_FILE_ATTRIBUTES) && diff --git a/sql/datadict.cc b/sql/datadict.cc index 783229a47a6..b2c4b615c4d 100644 --- a/sql/datadict.cc +++ b/sql/datadict.cc @@ -163,6 +163,8 @@ cont: if (mysql_file_fstat(file, &state, MYF(MY_WME))) goto err; + MSAN_STAT_WORKAROUND(&state); + if (mysql_file_seek(file, 0, SEEK_SET, MYF(MY_WME))) goto err; diff --git a/sql/ddl_log.cc b/sql/ddl_log.cc index 418b5f3b5b0..2ac4e256112 100644 --- a/sql/ddl_log.cc +++ b/sql/ddl_log.cc @@ -2491,11 +2491,11 @@ bool ddl_log_write_entry(DDL_LOG_ENTRY *ddl_log_entry, if (unlikely(write_ddl_log_file_entry((*active_entry)->entry_pos))) { + sql_print_error("DDL_LOG: Failed to write entry %u", + (*active_entry)->entry_pos); ddl_log_release_memory_entry(*active_entry); *active_entry= 0; error= TRUE; - sql_print_error("DDL_LOG: Failed to write entry %u", - (*active_entry)->entry_pos); } DBUG_RETURN(error); } @@ -2558,13 +2558,13 @@ bool ddl_log_write_execute_entry(uint first_entry, (*active_entry)->entry_pos, first_entry)); if (write_ddl_log_file_entry((*active_entry)->entry_pos)) { + sql_print_error("DDL_LOG: Error writing execute entry %u", + (*active_entry)->entry_pos); if (got_free_entry) { ddl_log_release_memory_entry(*active_entry); *active_entry= 0; } - sql_print_error("DDL_LOG: Error writing execute entry %u", - (*active_entry)->entry_pos); DBUG_RETURN(TRUE); } (void) ddl_log_sync_no_lock(); diff --git a/sql/discover.cc b/sql/discover.cc index 9b524760b63..201169357a2 100644 --- a/sql/discover.cc +++ b/sql/discover.cc @@ -1,5 +1,5 @@ /* Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved. - Copyright (c) 2009, 2020, MariaDB Corporation. + Copyright (c) 2009, 2022, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -72,6 +72,7 @@ int readfrm(const char *name, const uchar **frmdata, size_t *len) error= 2; if (mysql_file_fstat(file, &state, MYF(0))) goto err; + MSAN_STAT_WORKAROUND(&state); read_len= (size_t)MY_MIN(FRM_MAX_SIZE, state.st_size); // safety // Read whole frm file diff --git a/sql/parse_file.cc b/sql/parse_file.cc index 8911c683901..57140019341 100644 --- a/sql/parse_file.cc +++ b/sql/parse_file.cc @@ -464,6 +464,8 @@ sql_parse_prepare(const LEX_CSTRING *file_name, MEM_ROOT *mem_root, DBUG_RETURN(0); } + MSAN_STAT_WORKAROUND(&stat_info); + if (stat_info.st_size > INT_MAX-1) { my_error(ER_FPARSER_TOO_BIG_FILE, MYF(0), file_name->str); diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index 18629f4bd22..fe2307a2e91 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -1188,7 +1188,7 @@ static enum enum_server_command fetch_command(THD *thd, char *packet) DISPATCH_COMMAND_CLOSE_CONNECTION request of THD shutdown (s. dispatch_command() description) @retval - DISPATCH_COMMAND_WOULDBLOCK - need to wait for asyncronous operations + DISPATCH_COMMAND_WOULDBLOCK - need to wait for asynchronous operations to finish. Only returned if parameter 'blocking' is false. */ diff --git a/storage/innobase/buf/buf0rea.cc b/storage/innobase/buf/buf0rea.cc index 2dc779e81d0..f1cb34dd5f3 100644 --- a/storage/innobase/buf/buf0rea.cc +++ b/storage/innobase/buf/buf0rea.cc @@ -290,7 +290,7 @@ nothing_read: /* Trx sys header is so low in the latching order that we play safe and do not leave the i/o-completion to an asynchronous i/o-thread. Change buffer pages must always be read with - syncronous i/o, to make sure they do not get involved in + synchronous i/o, to make sure they do not get involved in thread deadlocks. */ sync = true; } diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index 0405b9fd5a0..b66e6b9a2d5 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -942,7 +942,7 @@ os_file_status_posix( if (!ret) { /* file exists, everything OK */ - + MSAN_STAT_WORKAROUND(&statinfo); } else if (errno == ENOENT || errno == ENOTDIR || errno == ENAMETOOLONG) { /* file does not exist */ return(true); @@ -1531,8 +1531,10 @@ bool os_file_close_func(os_file_t file) os_offset_t os_file_get_size(os_file_t file) { - struct stat statbuf; - return fstat(file, &statbuf) ? os_offset_t(-1) : statbuf.st_size; + struct stat statbuf; + if (fstat(file, &statbuf)) return os_offset_t(-1); + MSAN_STAT_WORKAROUND(&statbuf); + return statbuf.st_size; } /** Gets a file size. @@ -1549,6 +1551,7 @@ os_file_get_size( int ret = stat(filename, &s); if (ret == 0) { + MSAN_STAT_WORKAROUND(&s); file_size.m_total_size = s.st_size; /* st_blocks is in 512 byte sized blocks */ file_size.m_alloc_size = s.st_blocks * 512; @@ -1593,6 +1596,8 @@ os_file_get_status_posix( return(DB_FAIL); } + MSAN_STAT_WORKAROUND(statinfo); + switch (statinfo->st_mode & S_IFMT) { case S_IFDIR: stat_info->type = OS_FILE_TYPE_DIR; @@ -2785,7 +2790,7 @@ os_file_set_eof( #endif /* !_WIN32*/ -/** Does a syncronous read or write depending upon the type specified +/** Does a synchronous read or write depending upon the type specified In case of partial reads/writes the function tries NUM_RETRIES_ON_PARTIAL_IO times to read/write the complete data. @param[in] type, IO flags @@ -3275,6 +3280,7 @@ fallback: if (fstat(file, &statbuf)) { err = errno; } else { + MSAN_STAT_WORKAROUND(&statbuf); os_offset_t current_size = statbuf.st_size; if (current_size >= size) { return true; @@ -4158,7 +4164,10 @@ void fil_node_t::find_metadata(os_file_t file #else struct stat sbuf; if (!statbuf && !fstat(file, &sbuf)) + { + MSAN_STAT_WORKAROUND(&sbuf); statbuf= &sbuf; + } if (statbuf) block_size= statbuf->st_blksize; # ifdef UNIV_LINUX @@ -4191,6 +4200,7 @@ bool fil_node_t::read_page0() struct stat statbuf; if (fstat(handle, &statbuf)) return false; + MSAN_STAT_WORKAROUND(&statbuf); os_offset_t size_bytes= statbuf.st_size; #else os_offset_t size_bytes= os_file_get_size(handle); diff --git a/storage/innobase/trx/trx0purge.cc b/storage/innobase/trx/trx0purge.cc index 2b7fc71d716..cc1ba3382af 100644 --- a/storage/innobase/trx/trx0purge.cc +++ b/storage/innobase/trx/trx0purge.cc @@ -119,7 +119,7 @@ TRANSACTIONAL_INLINE inline bool TrxUndoRsegsIterator::set_next() trx_id_t last_trx_no, tail_trx_no; { #ifdef SUX_LOCK_GENERIC - purge_sys.rseg->latch.rd_lock(); + purge_sys.rseg->latch.rd_lock(SRW_LOCK_CALL); #else transactional_shared_lock_guard<srw_spin_lock> rg {purge_sys.rseg->latch}; @@ -636,7 +636,7 @@ TRANSACTIONAL_TARGET static void trx_purge_truncate_history() if (rseg.space != &space) continue; #ifdef SUX_LOCK_GENERIC - rseg.latch.rd_lock(); + rseg.latch.rd_lock(SRW_LOCK_CALL); #else transactional_shared_lock_guard<srw_spin_lock> g{rseg.latch}; #endif diff --git a/storage/maria/s3_func.h b/storage/maria/s3_func.h index a062131d5a5..f73a95dea24 100644 --- a/storage/maria/s3_func.h +++ b/storage/maria/s3_func.h @@ -141,7 +141,6 @@ C_MODE_END C_MODE_START /* Dummy structures and interfaces to be used when compiling without S3 */ struct s3_info; -typedef struct s3_info S3_INFO; struct ms3_st; C_MODE_END #endif /* WITH_S3_STORAGE_ENGINE */ diff --git a/storage/myisam/mi_info.c b/storage/myisam/mi_info.c index 9e1a5e416de..b754dde2a7d 100644 --- a/storage/myisam/mi_info.c +++ b/storage/myisam/mi_info.c @@ -87,7 +87,10 @@ int mi_status(MI_INFO *info, register MI_ISAMINFO *x, uint flag) x->index_file_name = share->index_file_name; } if ((flag & HA_STATUS_TIME) && !mysql_file_fstat(info->dfile, &state, MYF(0))) + { + MSAN_STAT_WORKAROUND(&state); x->update_time=state.st_mtime; + } else x->update_time=0; if (flag & HA_STATUS_AUTO) diff --git a/strings/my_vsnprintf.c b/strings/my_vsnprintf.c index 8417e07a6b0..95ac1d50e81 100644 --- a/strings/my_vsnprintf.c +++ b/strings/my_vsnprintf.c @@ -738,10 +738,11 @@ size_t my_vsnprintf_ex(CHARSET_INFO *cs, char *to, size_t n, } else if (*fmt == 'f' || *fmt == 'g') { + double d; #if __has_feature(memory_sanitizer) /* QQ: MSAN has double trouble? */ __msan_check_mem_is_initialized(ap, sizeof(double)); #endif - double d= va_arg(ap, double); + d= va_arg(ap, double); #if __has_feature(memory_sanitizer) /* QQ: MSAN has double trouble? */ __msan_unpoison(&d, sizeof(double)); #endif diff --git a/tpool/aio_liburing.cc b/tpool/aio_liburing.cc index cca95bb6d37..b8666482193 100644 --- a/tpool/aio_liburing.cc +++ b/tpool/aio_liburing.cc @@ -161,6 +161,8 @@ private: } io_uring_cqe_seen(&aio->uring_, cqe); + if (iocb->m_ret_len != iocb->m_len && !iocb->m_err) + finish_synchronous(iocb); // If we need to resubmit the IO operation, but the ring is full, // we will follow the same path as for any other error codes. diff --git a/tpool/aio_linux.cc b/tpool/aio_linux.cc index 4abc2139881..fc6e5b53e1a 100644 --- a/tpool/aio_linux.cc +++ b/tpool/aio_linux.cc @@ -128,6 +128,8 @@ class aio_linux final : public aio { iocb->m_ret_len= event.res; iocb->m_err= 0; + if (iocb->m_ret_len != iocb->m_len) + finish_synchronous(iocb); } iocb->m_internal_task.m_func= iocb->m_callback; iocb->m_internal_task.m_arg= iocb; diff --git a/tpool/aio_simulated.cc b/tpool/aio_simulated.cc index 6b6fe71c8ab..4bc58c2930c 100644 --- a/tpool/aio_simulated.cc +++ b/tpool/aio_simulated.cc @@ -136,32 +136,7 @@ public: static void simulated_aio_callback(void *param) { aiocb *cb= (aiocb *) param; -#ifdef _WIN32 - size_t ret_len; -#else - ssize_t ret_len; -#endif - int err= 0; - switch (cb->m_opcode) - { - case aio_opcode::AIO_PREAD: - ret_len= pread(cb->m_fh, cb->m_buffer, cb->m_len, cb->m_offset); - break; - case aio_opcode::AIO_PWRITE: - ret_len= pwrite(cb->m_fh, cb->m_buffer, cb->m_len, cb->m_offset); - break; - default: - abort(); - } -#ifdef _WIN32 - if (static_cast<int>(ret_len) < 0) - err= GetLastError(); -#else - if (ret_len < 0) - err= errno; -#endif - cb->m_ret_len = ret_len; - cb->m_err = err; + synchronous(cb); cb->m_internal_task.m_func= cb->m_callback; thread_pool *pool= (thread_pool *)cb->m_internal; pool->submit_task(&cb->m_internal_task); diff --git a/tpool/tpool.h b/tpool/tpool.h index f857dddd57a..2c61c2d62b2 100644 --- a/tpool/tpool.h +++ b/tpool/tpool.h @@ -161,7 +161,7 @@ class aio { public: /** - Submit asyncronous IO. + Submit asynchronous IO. On completion, cb->m_callback is executed. */ virtual int submit_io(aiocb *cb)= 0; @@ -170,6 +170,10 @@ public: /** "Unind" file to AIO handler (used on Windows only) */ virtual int unbind(const native_file_handle &fd)= 0; virtual ~aio(){}; +protected: + static void synchronous(aiocb *cb); + /** finish a partial read/write callback synchronously */ + static void finish_synchronous(aiocb *cb); }; class timer diff --git a/tpool/tpool_generic.cc b/tpool/tpool_generic.cc index ecc489c3357..a1b9a3ce945 100644 --- a/tpool/tpool_generic.cc +++ b/tpool/tpool_generic.cc @@ -52,6 +52,58 @@ namespace tpool static const int OVERSUBSCRIBE_FACTOR = 2; /** + Process the cb synchronously +*/ +void aio::synchronous(aiocb *cb) +{ +#ifdef _WIN32 + size_t ret_len; +#else + ssize_t ret_len; +#endif + int err= 0; + switch (cb->m_opcode) + { + case aio_opcode::AIO_PREAD: + ret_len= pread(cb->m_fh, cb->m_buffer, cb->m_len, cb->m_offset); + break; + case aio_opcode::AIO_PWRITE: + ret_len= pwrite(cb->m_fh, cb->m_buffer, cb->m_len, cb->m_offset); + break; + default: + abort(); + } +#ifdef _WIN32 + if (static_cast<int>(ret_len) < 0) + err= GetLastError(); +#else + if (ret_len < 0) + { + err= errno; + ret_len= 0; + } +#endif + cb->m_ret_len = ret_len; + cb->m_err = err; + if (!err && cb->m_ret_len != cb->m_len) + finish_synchronous(cb); +} + + +/** + A partial read/write has occured, continue synchronously. +*/ +void aio::finish_synchronous(aiocb *cb) +{ + assert(cb->m_ret_len != (unsigned int) cb->m_len && !cb->m_err); + /* partial read/write */ + cb->m_buffer= (char *) cb->m_buffer + cb->m_ret_len; + cb->m_len-= (unsigned int) cb->m_ret_len; + cb->m_offset+= cb->m_ret_len; + synchronous(cb); +} + +/** Implementation of generic threadpool. This threadpool consists of the following components |