diff options
| author | Sergei Petrunia <psergey@askmonty.org> | 2016-12-31 23:30:09 +0300 |
|---|---|---|
| committer | Sergei Petrunia <psergey@askmonty.org> | 2016-12-31 23:30:09 +0300 |
| commit | cfb59f3196aac1b41cdda79952031dcc64042914 (patch) | |
| tree | 3833cd1f2f4f2360eb3a16bd7ae4606e8493dea8 | |
| parent | ebfc4e6ad02b0cef34ec3f446007b98d85af9296 (diff) | |
| download | mariadb-git-cfb59f3196aac1b41cdda79952031dcc64042914.tar.gz | |
Copy of
commit f6ed777697db4ad7aee1e98c53243dced2b5891c
Author: Chenyu Yan <seayan@outlook.com>
Date: Thu Dec 29 16:10:25 2016 -0800
Fix BIG_TIMEOUT constant
Summary: Update the constant to reflect what the comments indicates.
Reviewed By: gunnarku
Differential Revision: D4374476
fbshipit-source-id: dd7f484
132 files changed, 5995 insertions, 3220 deletions
diff --git a/storage/rocksdb/CMakeLists.txt b/storage/rocksdb/CMakeLists.txt index f2c52ce84c4..a5d07a12d96 100644 --- a/storage/rocksdb/CMakeLists.txt +++ b/storage/rocksdb/CMakeLists.txt @@ -4,6 +4,11 @@ IF (NOT EXISTS "${CMAKE_SOURCE_DIR}/rocksdb/Makefile") MESSAGE(SEND_ERROR "Missing Makefile in rocksdb directory. Try \"git submodule update\".") ENDIF() +CHECK_FUNCTION_EXISTS(sched_getcpu HAVE_SCHED_GETCPU) +IF(HAVE_SCHED_GETCPU) + ADD_DEFINITIONS(-DHAVE_SCHED_GETCPU=1) +ENDIF() + # get a list of rocksdb library source files # run with env -i to avoid passing variables EXECUTE_PROCESS( @@ -51,32 +56,26 @@ ENDIF() SET(rocksdb_static_libs ) IF (NOT "$ENV{WITH_SNAPPY}" STREQUAL "") SET(rocksdb_static_libs ${rocksdb_static_libs} - $ENV{WITH_SNAPPY}/lib/libsnappy${PIC_EXT}.a) + $ENV{WITH_SNAPPY}/libsnappy${PIC_EXT}.a) ADD_DEFINITIONS(-DSNAPPY) -ELSE() - SET(rocksdb_static_libs ${rocksdb_static_libs} snappy) ENDIF() IF (NOT "$ENV{WITH_LZ4}" STREQUAL "") SET(rocksdb_static_libs ${rocksdb_static_libs} - $ENV{WITH_LZ4}/lib/liblz4${PIC_EXT}.a) + $ENV{WITH_LZ4}/liblz4${PIC_EXT}.a) ADD_DEFINITIONS(-DLZ4) -ELSE() - SET(rocksdb_static_libs ${rocksdb_static_libs} lz4) ENDIF() IF (NOT "$ENV{WITH_BZ2}" STREQUAL "") SET(rocksdb_static_libs ${rocksdb_static_libs} - $ENV{WITH_BZ2}/lib/libbz2${PIC_EXT}.a) + $ENV{WITH_BZ2}/libbz2${PIC_EXT}.a) ADD_DEFINITIONS(-DBZIP2) -ELSE() - SET(rocksdb_static_libs ${rocksdb_static_libs} bz2) ENDIF() # link ZSTD only if instructed IF (NOT "$ENV{WITH_ZSTD}" STREQUAL "") SET(rocksdb_static_libs ${rocksdb_static_libs} - $ENV{WITH_ZSTD}/lib/libzstd${PIC_EXT}.a) + $ENV{WITH_ZSTD}/libzstd${PIC_EXT}.a) ADD_DEFINITIONS(-DZSTD) ENDIF() diff --git a/storage/rocksdb/event_listener.cc b/storage/rocksdb/event_listener.cc index 851e4782416..e4338f07986 100644 --- a/storage/rocksdb/event_listener.cc +++ b/storage/rocksdb/event_listener.cc @@ -39,7 +39,7 @@ extract_index_stats( ) { std::vector<Rdb_index_stats> ret; for (auto fn : files) { - auto it = props.find(fn); + const auto it = props.find(fn); DBUG_ASSERT(it != props.end()); std::vector<Rdb_index_stats> stats; Rdb_tbl_prop_coll::read_stats_from_tbl_props(it->second, &stats); @@ -48,6 +48,19 @@ extract_index_stats( return ret; } +void Rdb_event_listener::update_index_stats( + const rocksdb::TableProperties& props +) { + DBUG_ASSERT(m_ddl_manager != nullptr); + const auto tbl_props = + std::make_shared<const rocksdb::TableProperties>(props); + + std::vector<Rdb_index_stats> stats; + Rdb_tbl_prop_coll::read_stats_from_tbl_props(tbl_props, &stats); + + m_ddl_manager->adjust_stats(stats); +} + void Rdb_event_listener::OnCompactionCompleted( rocksdb::DB *db, const rocksdb::CompactionJobInfo& ci @@ -67,14 +80,14 @@ void Rdb_event_listener::OnFlushCompleted( const rocksdb::FlushJobInfo& flush_job_info ) { DBUG_ASSERT(db != nullptr); - DBUG_ASSERT(m_ddl_manager != nullptr); - - auto tbl_props = std::make_shared<const rocksdb::TableProperties>( - flush_job_info.table_properties); - - std::vector<Rdb_index_stats> stats; - Rdb_tbl_prop_coll::read_stats_from_tbl_props(tbl_props, &stats); - m_ddl_manager->adjust_stats(stats); + update_index_stats(flush_job_info.table_properties); } +void Rdb_event_listener::OnExternalFileIngested( + rocksdb::DB* db, + const rocksdb::ExternalFileIngestionInfo& info +) { + DBUG_ASSERT(db != nullptr); + update_index_stats(info.table_properties); +} } // namespace myrocks diff --git a/storage/rocksdb/event_listener.h b/storage/rocksdb/event_listener.h index f92c6d0dea1..8895da45c7b 100644 --- a/storage/rocksdb/event_listener.h +++ b/storage/rocksdb/event_listener.h @@ -24,17 +24,26 @@ class Rdb_ddl_manager; class Rdb_event_listener : public rocksdb::EventListener { public: - explicit Rdb_event_listener(Rdb_ddl_manager* ddl_manager) : + Rdb_event_listener(const Rdb_event_listener&) = delete; + Rdb_event_listener& operator=(const Rdb_event_listener&) = delete; + + explicit Rdb_event_listener(Rdb_ddl_manager* const ddl_manager) : m_ddl_manager(ddl_manager) { } void OnCompactionCompleted( - rocksdb::DB *db, const rocksdb::CompactionJobInfo& ci) override; + rocksdb::DB* db, const rocksdb::CompactionJobInfo& ci) override; void OnFlushCompleted( rocksdb::DB* db, const rocksdb::FlushJobInfo& flush_job_info) override; + void OnExternalFileIngested( + rocksdb::DB* db, const rocksdb::ExternalFileIngestionInfo& ingestion_info) + override; private: Rdb_ddl_manager* m_ddl_manager; + + void update_index_stats( + const rocksdb::TableProperties& props); }; } // namespace myrocks diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc index 4fbaf01f154..a557f96b92f 100644 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@ -28,11 +28,13 @@ #include <queue> #include <string> #include <vector> +#include <algorithm> /* MySQL includes */ #include "./debug_sync.h" #include "./my_bit.h" #include "./my_stacktrace.h" +#include "./sql_audit.h" #include "./sql_table.h" #include <mysys_err.h> #include <mysql/psi/mysql_table.h> @@ -44,7 +46,6 @@ #include "rocksdb/slice_transform.h" #include "rocksdb/utilities/checkpoint.h" #include "rocksdb/utilities/convenience.h" -#include "rocksdb/utilities/flashcache.h" #include "rocksdb/utilities/memory_util.h" /* MyRocks includes */ @@ -59,10 +60,6 @@ #include "./rdb_mutex_wrapper.h" #include "./rdb_threads.h" -#ifdef TARGET_OS_LINUX -extern my_bool cachedev_enabled; -#endif /* TARGET_OS_LINUX */ - // Internal MySQL APIs not exposed in any header. extern "C" { @@ -96,7 +93,7 @@ static st_export_stats export_stats; /** Updates row counters based on the table type and operation type. */ -void ha_rocksdb::update_row_stats(operation_type type) { +void ha_rocksdb::update_row_stats(const operation_type &type) { DBUG_ASSERT(type < ROWS_MAX); // Find if we are modifying system databases. if (table->s && m_tbl_def->m_is_mysql_system_table) @@ -111,7 +108,7 @@ static handler *rocksdb_create_handler(my_core::handlerton *hton, my_core::MEM_ROOT *mem_root); bool can_use_bloom_filter(THD *thd, - const std::shared_ptr<const Rdb_key_def>& kd, + const Rdb_key_def& kd, const rocksdb::Slice &eq_cond, const bool use_all_keys, bool is_ascending); @@ -138,6 +135,7 @@ static std::shared_ptr<Rdb_tbl_prop_coll_factory> Rdb_dict_manager dict_manager; Rdb_cf_manager cf_manager; Rdb_ddl_manager ddl_manager; +const char* m_mysql_gtid; Rdb_binlog_manager binlog_manager; @@ -164,30 +162,32 @@ static const char* const ERRSTR_ROLLBACK_ONLY static void rocksdb_flush_all_memtables() { - Rdb_cf_manager& cf_manager= rdb_get_cf_manager(); - for (auto cf_handle : cf_manager.get_all_cf()) { + const Rdb_cf_manager& cf_manager= rdb_get_cf_manager(); + for (const auto &cf_handle : cf_manager.get_all_cf()) { rdb->Flush(rocksdb::FlushOptions(), cf_handle); } } static void -rocksdb_compact_column_family_stub(THD* thd, - struct st_mysql_sys_var* var, - void* var_ptr, - const void* save) +rocksdb_compact_column_family_stub(THD* const thd, + struct st_mysql_sys_var* const var, + void* const var_ptr, + const void* const save) { } static int -rocksdb_compact_column_family(THD* thd, - struct st_mysql_sys_var* var, - void* var_ptr, - struct st_mysql_value* value) +rocksdb_compact_column_family(THD* const thd, + struct st_mysql_sys_var* const var, + void* const var_ptr, + struct st_mysql_value* const value) { char buff[STRING_BUFFER_USUAL_SIZE]; int len = sizeof(buff); - if (const char* cf = value->val_str(value, buff, &len)) { + DBUG_ASSERT(value != nullptr); + + if (const char* const cf = value->val_str(value, buff, &len)) { bool is_automatic; auto cfh = cf_manager.get_cf(cf, "", nullptr, &is_automatic); if (cfh != nullptr && rdb != nullptr) { @@ -205,6 +205,8 @@ rocksdb_compact_column_family(THD* thd, namespace // anonymous namespace = not visible outside this source file { +const ulong TABLE_HASH_SIZE= 32; + struct Rdb_open_tables_map { /* Hash table used to track the handlers of open tables */ @@ -214,7 +216,8 @@ struct Rdb_open_tables_map void init_hash(void) { - (void) my_hash_init(&m_hash, my_core::system_charset_info, 32, 0, 0, + (void) my_hash_init(&m_hash, my_core::system_charset_info, TABLE_HASH_SIZE, + 0, 0, (my_hash_get_key) Rdb_open_tables_map::get_hash_key, 0, 0); } @@ -224,12 +227,12 @@ struct Rdb_open_tables_map my_hash_free(&m_hash); } - static uchar* get_hash_key(Rdb_table_handler *table_handler, - size_t *length, + static uchar* get_hash_key(Rdb_table_handler* const table_handler, + size_t* const length, my_bool not_used __attribute__((__unused__))); - Rdb_table_handler* get_table_handler(const char *table_name); - void release_table_handler(Rdb_table_handler *table_handler); + Rdb_table_handler* get_table_handler(const char* const table_name); + void release_table_handler(Rdb_table_handler* const table_handler); std::vector<std::string> get_table_names(void) const; }; @@ -250,14 +253,14 @@ static std::string rdb_normalize_dir(std::string dir) static int rocksdb_create_checkpoint( - THD* thd __attribute__((__unused__)), - struct st_mysql_sys_var* var __attribute__((__unused__)), - void* save __attribute__((__unused__)), - struct st_mysql_value* value) + THD* const thd __attribute__((__unused__)), + struct st_mysql_sys_var* const var __attribute__((__unused__)), + void* const save __attribute__((__unused__)), + struct st_mysql_value* const value) { - char buf[512]; + char buf[FN_REFLEN]; int len = sizeof(buf); - const char* checkpoint_dir_raw= value->val_str(value, buf, &len); + const char* const checkpoint_dir_raw= value->val_str(value, buf, &len); if (checkpoint_dir_raw) { if (rdb != nullptr) { std::string checkpoint_dir= rdb_normalize_dir(checkpoint_dir_raw); @@ -280,7 +283,7 @@ static int rocksdb_create_checkpoint( } delete checkpoint; } else { - std::string err_text(status.ToString()); + const std::string err_text(status.ToString()); my_printf_error(ER_UNKNOWN_ERROR, "RocksDB: failed to initialize checkpoint. status %d %s\n", MYF(0), status.code(), err_text.c_str()); @@ -294,26 +297,26 @@ static int rocksdb_create_checkpoint( /* This method is needed to indicate that the ROCKSDB_CREATE_CHECKPOINT command is not read-only */ static void -rocksdb_create_checkpoint_stub(THD* thd, - struct st_mysql_sys_var* var, - void* var_ptr, - const void* save) +rocksdb_create_checkpoint_stub(THD* const thd, + struct st_mysql_sys_var* const var, + void* const var_ptr, + const void* const save) { } static void -rocksdb_force_flush_memtable_now_stub(THD* thd, - struct st_mysql_sys_var* var, - void* var_ptr, - const void* save) +rocksdb_force_flush_memtable_now_stub(THD* const thd, + struct st_mysql_sys_var* const var, + void* const var_ptr, + const void* const save) { } static int -rocksdb_force_flush_memtable_now(THD* thd, - struct st_mysql_sys_var* var, - void* var_ptr, - struct st_mysql_value* value) +rocksdb_force_flush_memtable_now(THD* const thd, + struct st_mysql_sys_var* const var, + void* const var_ptr, + struct st_mysql_value* const value) { sql_print_information("RocksDB: Manual memtable flush\n"); rocksdb_flush_all_memtables(); @@ -321,22 +324,22 @@ rocksdb_force_flush_memtable_now(THD* thd, } static void rocksdb_drop_index_wakeup_thread( - my_core::THD* thd __attribute__((__unused__)), - struct st_mysql_sys_var* var __attribute__((__unused__)), - void* var_ptr __attribute__((__unused__)), - const void* save); + my_core::THD* const thd __attribute__((__unused__)), + struct st_mysql_sys_var* const var __attribute__((__unused__)), + void* const var_ptr __attribute__((__unused__)), + const void* const save); static my_bool rocksdb_pause_background_work= 0; static mysql_mutex_t rdb_sysvars_mutex; static void rocksdb_set_pause_background_work( - my_core::THD* thd __attribute__((__unused__)), - struct st_mysql_sys_var* var __attribute__((__unused__)), - void* var_ptr __attribute__((__unused__)), - const void* save) + my_core::THD* const thd __attribute__((__unused__)), + struct st_mysql_sys_var* const var __attribute__((__unused__)), + void* const var_ptr __attribute__((__unused__)), + const void* const save) { mysql_mutex_lock(&rdb_sysvars_mutex); - bool pause_requested= *static_cast<const bool*>(save); + const bool pause_requested= *static_cast<const bool*>(save); if (rocksdb_pause_background_work != pause_requested) { if (pause_requested) { rdb->PauseBackgroundWork(); @@ -411,8 +414,10 @@ static char * rocksdb_datadir; static uint32_t rocksdb_table_stats_sampling_pct; static my_bool rocksdb_enable_bulk_load_api= 1; static my_bool rpl_skip_tx_api_var= 0; +static my_bool rocksdb_print_snapshot_conflict_queries= 0; std::atomic<uint64_t> rocksdb_snapshot_conflict_errors(0); +std::atomic<uint64_t> rocksdb_wal_group_syncs(0); static rocksdb::DBOptions rdb_init_rocksdb_db_options(void) { @@ -449,11 +454,13 @@ static TYPELIB info_log_level_typelib = { }; static void -rocksdb_set_rocksdb_info_log_level(THD* thd, - struct st_mysql_sys_var* var, - void* var_ptr, - const void* save) +rocksdb_set_rocksdb_info_log_level(THD* const thd, + struct st_mysql_sys_var* const var, + void* const var_ptr, + const void* const save) { + DBUG_ASSERT(save != nullptr); + mysql_mutex_lock(&rdb_sysvars_mutex); rocksdb_info_log_level = *static_cast<const uint64_t*>(save); rocksdb_db_options.info_log->SetInfoLogLevel( @@ -474,10 +481,30 @@ static TYPELIB index_type_typelib = { nullptr }; +const ulong RDB_MAX_LOCK_WAIT_SECONDS= 1024*1024*1024; +const ulong RDB_MAX_ROW_LOCKS= 1024*1024*1024; +const ulong RDB_DEFAULT_BULK_LOAD_SIZE= 1000; +const ulong RDB_MAX_BULK_LOAD_SIZE= 1024*1024*1024; +const size_t RDB_DEFAULT_MERGE_BUF_SIZE= 64*1024*1024; +const size_t RDB_MIN_MERGE_BUF_SIZE= 100; +const size_t RDB_DEFAULT_MERGE_COMBINE_READ_SIZE= 1024*1024*1024; +const size_t RDB_MIN_MERGE_COMBINE_READ_SIZE= 100; +const int64 RDB_DEFAULT_BLOCK_CACHE_SIZE= 512*1024*1024; +const int64 RDB_MIN_BLOCK_CACHE_SIZE= 1024; +const int RDB_MAX_CHECKSUMS_PCT= 100; + //TODO: 0 means don't wait at all, and we don't support it yet? static MYSQL_THDVAR_ULONG(lock_wait_timeout, PLUGIN_VAR_RQCMDARG, "Number of seconds to wait for lock", - nullptr, nullptr, /*default*/ 1, /*min*/ 1, /*max*/ 1024*1024*1024, 0); + nullptr, nullptr, /*default*/ 1, /*min*/ 1, + /*max*/ RDB_MAX_LOCK_WAIT_SECONDS, 0); + +static MYSQL_THDVAR_BOOL(deadlock_detect, PLUGIN_VAR_RQCMDARG, + "Enables deadlock detection", nullptr, nullptr, FALSE); + +static MYSQL_THDVAR_BOOL(trace_sst_api, PLUGIN_VAR_RQCMDARG, + "Generate trace output in the log for each call to the SstFileWriter", + nullptr, nullptr, FALSE); static MYSQL_THDVAR_BOOL(bulk_load, PLUGIN_VAR_RQCMDARG, "Use bulk-load mode for inserts. This enables both " @@ -490,6 +517,11 @@ static MYSQL_SYSVAR_BOOL(enable_bulk_load_api, "Enables using SstFileWriter for bulk loading", nullptr, nullptr, rocksdb_enable_bulk_load_api); +static MYSQL_THDVAR_STR(tmpdir, + PLUGIN_VAR_OPCMDARG|PLUGIN_VAR_MEMALLOC, + "Directory for temporary files during DDL operations.", + nullptr, nullptr, ""); + static MYSQL_THDVAR_STR(skip_unique_check_tables, PLUGIN_VAR_RQCMDARG|PLUGIN_VAR_MEMALLOC, "Skip unique constraint checking for the specified tables", nullptr, nullptr, @@ -520,8 +552,10 @@ static MYSQL_THDVAR_BOOL(skip_bloom_filter_on_read, PLUGIN_VAR_RQCMDARG, static MYSQL_THDVAR_ULONG(max_row_locks, PLUGIN_VAR_RQCMDARG, "Maximum number of locks a transaction can have", - nullptr, nullptr, /*default*/ 1024*1024*1024, /*min*/ 1, - /*max*/ 1024*1024*1024, 0); + nullptr, nullptr, + /*default*/ RDB_MAX_ROW_LOCKS, + /*min*/ 1, + /*max*/ RDB_MAX_ROW_LOCKS, 0); static MYSQL_THDVAR_BOOL(lock_scanned_rows, PLUGIN_VAR_RQCMDARG, "Take and hold locks on rows that are scanned but not updated", @@ -529,22 +563,25 @@ static MYSQL_THDVAR_BOOL(lock_scanned_rows, PLUGIN_VAR_RQCMDARG, static MYSQL_THDVAR_ULONG(bulk_load_size, PLUGIN_VAR_RQCMDARG, "Max #records in a batch for bulk-load mode", - nullptr, nullptr, /*default*/ 1000, /*min*/ 1, /*max*/ 1024*1024*1024, 0); + nullptr, nullptr, + /*default*/ RDB_DEFAULT_BULK_LOAD_SIZE, + /*min*/ 1, + /*max*/ RDB_MAX_BULK_LOAD_SIZE, 0); static MYSQL_THDVAR_ULONGLONG(merge_buf_size, PLUGIN_VAR_RQCMDARG, "Size to allocate for merge sort buffers written out to disk " "during inplace index creation.", nullptr, nullptr, - /* default (64MB) */ (ulonglong) 67108864, - /* min (100B) */ 100, + /* default (64MB) */ RDB_DEFAULT_MERGE_BUF_SIZE, + /* min (100B) */ RDB_MIN_MERGE_BUF_SIZE, /* max */ SIZE_T_MAX, 1); static MYSQL_THDVAR_ULONGLONG(merge_combine_read_size, PLUGIN_VAR_RQCMDARG, "Size that we have to work with during combine (reading from disk) phase of " "external sort during fast index creation.", nullptr, nullptr, - /* default (1GB) */ (ulonglong) 1073741824, - /* min (100B) */ 100, + /* default (1GB) */ RDB_DEFAULT_MERGE_COMBINE_READ_SIZE, + /* min (100B) */ RDB_MIN_MERGE_COMBINE_READ_SIZE, /* max */ SIZE_T_MAX, 1); static MYSQL_SYSVAR_BOOL(create_if_missing, @@ -600,8 +637,10 @@ static MYSQL_SYSVAR_UINT(wal_recovery_mode, rocksdb_wal_recovery_mode, PLUGIN_VAR_RQCMDARG, "DBOptions::wal_recovery_mode for RocksDB", - nullptr, nullptr, 2, - /* min */ 0L, /* max */ 3, 0); + nullptr, nullptr, + /* default */ (uint) rocksdb::WALRecoveryMode::kPointInTimeRecovery, + /* min */ (uint) rocksdb::WALRecoveryMode::kTolerateCorruptedTailRecords, + /* max */ (uint) rocksdb::WALRecoveryMode::kSkipAnyCorruptedRecords, 0); static MYSQL_SYSVAR_ULONG(compaction_readahead_size, rocksdb_db_options.compaction_readahead_size, @@ -621,22 +660,24 @@ static MYSQL_SYSVAR_UINT(access_hint_on_compaction_start, rocksdb_access_hint_on_compaction_start, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, "DBOptions::access_hint_on_compaction_start for RocksDB", - nullptr, nullptr, 1, - /* min */ 0L, /* max */ 3, 0); + nullptr, nullptr, + /* default */ (uint) rocksdb::Options::AccessHint::NORMAL, + /* min */ (uint) rocksdb::Options::AccessHint::NONE, + /* max */ (uint) rocksdb::Options::AccessHint::WILLNEED, 0); static MYSQL_SYSVAR_BOOL(allow_concurrent_memtable_write, *reinterpret_cast<my_bool*>( &rocksdb_db_options.allow_concurrent_memtable_write), PLUGIN_VAR_RQCMDARG, "DBOptions::allow_concurrent_memtable_write for RocksDB", - nullptr, nullptr, rocksdb_db_options.allow_concurrent_memtable_write); + nullptr, nullptr, false); static MYSQL_SYSVAR_BOOL(enable_write_thread_adaptive_yield, *reinterpret_cast<my_bool*>( &rocksdb_db_options.enable_write_thread_adaptive_yield), PLUGIN_VAR_RQCMDARG, "DBOptions::enable_write_thread_adaptive_yield for RocksDB", - nullptr, nullptr, rocksdb_db_options.enable_write_thread_adaptive_yield); + nullptr, nullptr, false); static MYSQL_SYSVAR_INT(max_open_files, rocksdb_db_options.max_open_files, @@ -760,11 +801,17 @@ static MYSQL_SYSVAR_ULONG(manifest_preallocation_size, nullptr, nullptr, rocksdb_db_options.manifest_preallocation_size, /* min */ 0L, /* max */ LONG_MAX, 0); -static MYSQL_SYSVAR_BOOL(allow_os_buffer, - *reinterpret_cast<my_bool*>(&rocksdb_db_options.allow_os_buffer), +static MYSQL_SYSVAR_BOOL(use_direct_reads, + *reinterpret_cast<my_bool*>(&rocksdb_db_options.use_direct_reads), PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, - "DBOptions::allow_os_buffer for RocksDB", - nullptr, nullptr, rocksdb_db_options.allow_os_buffer); + "DBOptions::use_direct_reads for RocksDB", + nullptr, nullptr, rocksdb_db_options.use_direct_reads); + +static MYSQL_SYSVAR_BOOL(use_direct_writes, + *reinterpret_cast<my_bool*>(&rocksdb_db_options.use_direct_writes), + PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, + "DBOptions::use_direct_writes for RocksDB", + nullptr, nullptr, rocksdb_db_options.use_direct_writes); static MYSQL_SYSVAR_BOOL(allow_mmap_reads, *reinterpret_cast<my_bool*>(&rocksdb_db_options.allow_mmap_reads), @@ -833,8 +880,10 @@ static MYSQL_SYSVAR_BOOL(enable_thread_tracking, static MYSQL_SYSVAR_LONGLONG(block_cache_size, rocksdb_block_cache_size, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, "block_cache size for RocksDB", - nullptr, nullptr, /* RocksDB's default is 8 MB: */ 8*1024*1024L, - /* min */ 1024L, /* max */ LONGLONG_MAX, /* Block size */1024L); + nullptr, nullptr, + /* default */ RDB_DEFAULT_BLOCK_CACHE_SIZE, + /* min */ RDB_MIN_BLOCK_CACHE_SIZE, + /* max */ LONGLONG_MAX, /* Block size */ RDB_MIN_BLOCK_CACHE_SIZE); static MYSQL_SYSVAR_BOOL(cache_index_and_filter_blocks, *reinterpret_cast<my_bool*>( @@ -1070,18 +1119,24 @@ static MYSQL_SYSVAR_BOOL(compaction_sequential_deletes_count_sd, "Counting SingleDelete as rocksdb_compaction_sequential_deletes", nullptr, nullptr, rocksdb_compaction_sequential_deletes_count_sd); +static MYSQL_SYSVAR_BOOL(print_snapshot_conflict_queries, + rocksdb_print_snapshot_conflict_queries, + PLUGIN_VAR_RQCMDARG, + "Logging queries that got snapshot conflict errors into *.err log", + nullptr, nullptr, rocksdb_print_snapshot_conflict_queries); + static MYSQL_THDVAR_INT(checksums_pct, PLUGIN_VAR_RQCMDARG, "How many percentages of rows to be checksummed", - nullptr, nullptr, 100, - /* min */ 0, /* max */ 100, 0); + nullptr, nullptr, RDB_MAX_CHECKSUMS_PCT, + /* min */ 0, /* max */ RDB_MAX_CHECKSUMS_PCT, 0); -static MYSQL_THDVAR_BOOL(store_checksums, +static MYSQL_THDVAR_BOOL(store_row_debug_checksums, PLUGIN_VAR_RQCMDARG, "Include checksums when writing index/table records", nullptr, nullptr, false /* default value */); -static MYSQL_THDVAR_BOOL(verify_checksums, +static MYSQL_THDVAR_BOOL(verify_row_debug_checksums, PLUGIN_VAR_RQCMDARG, "Verify checksums when reading index/table records", nullptr, nullptr, false /* default value */); @@ -1114,15 +1169,16 @@ static MYSQL_SYSVAR_UINT( RDB_DEFAULT_TBL_STATS_SAMPLE_PCT, /* everything */ 0, /* max */ RDB_TBL_STATS_SAMPLE_PCT_MAX, 0); -static const longlong ROCKSDB_WRITE_BUFFER_SIZE_DEFAULT= 4194304; static const int ROCKSDB_ASSUMED_KEY_VALUE_DISK_SIZE= 100; static struct st_mysql_sys_var* rocksdb_system_variables[]= { MYSQL_SYSVAR(lock_wait_timeout), + MYSQL_SYSVAR(deadlock_detect), MYSQL_SYSVAR(max_row_locks), MYSQL_SYSVAR(lock_scanned_rows), MYSQL_SYSVAR(bulk_load), MYSQL_SYSVAR(skip_unique_check_tables), + MYSQL_SYSVAR(trace_sst_api), MYSQL_SYSVAR(skip_unique_check), MYSQL_SYSVAR(commit_in_the_middle), MYSQL_SYSVAR(read_free_rpl_tables), @@ -1130,6 +1186,7 @@ static struct st_mysql_sys_var* rocksdb_system_variables[]= { MYSQL_SYSVAR(bulk_load_size), MYSQL_SYSVAR(merge_buf_size), MYSQL_SYSVAR(enable_bulk_load_api), + MYSQL_SYSVAR(tmpdir), MYSQL_SYSVAR(merge_combine_read_size), MYSQL_SYSVAR(skip_bloom_filter_on_read), @@ -1157,7 +1214,8 @@ static struct st_mysql_sys_var* rocksdb_system_variables[]= { MYSQL_SYSVAR(wal_ttl_seconds), MYSQL_SYSVAR(wal_size_limit_mb), MYSQL_SYSVAR(manifest_preallocation_size), - MYSQL_SYSVAR(allow_os_buffer), + MYSQL_SYSVAR(use_direct_reads), + MYSQL_SYSVAR(use_direct_writes), MYSQL_SYSVAR(allow_mmap_reads), MYSQL_SYSVAR(allow_mmap_writes), MYSQL_SYSVAR(is_fd_close_on_exec), @@ -1219,13 +1277,14 @@ static struct st_mysql_sys_var* rocksdb_system_variables[]= { MYSQL_SYSVAR(compaction_sequential_deletes_window), MYSQL_SYSVAR(compaction_sequential_deletes_file_size), MYSQL_SYSVAR(compaction_sequential_deletes_count_sd), + MYSQL_SYSVAR(print_snapshot_conflict_queries), MYSQL_SYSVAR(datadir), MYSQL_SYSVAR(create_checkpoint), MYSQL_SYSVAR(checksums_pct), - MYSQL_SYSVAR(store_checksums), - MYSQL_SYSVAR(verify_checksums), + MYSQL_SYSVAR(store_row_debug_checksums), + MYSQL_SYSVAR(verify_row_debug_checksums), MYSQL_SYSVAR(validate_tables), MYSQL_SYSVAR(table_stats_sampling_pct), @@ -1233,7 +1292,8 @@ static struct st_mysql_sys_var* rocksdb_system_variables[]= { }; -static rocksdb::WriteOptions rdb_get_rocksdb_write_options(my_core::THD* thd) +static rocksdb::WriteOptions rdb_get_rocksdb_write_options( + my_core::THD* const thd) { rocksdb::WriteOptions opt; @@ -1253,7 +1313,7 @@ static rocksdb::WriteOptions rdb_get_rocksdb_write_options(my_core::THD* thd) */ uchar* Rdb_open_tables_map::get_hash_key( - Rdb_table_handler *table_handler, size_t *length, + Rdb_table_handler* const table_handler, size_t* const length, my_bool not_used __attribute__((__unused__))) { *length= table_handler->m_table_name_length; @@ -1325,7 +1385,7 @@ static PSI_thread_info all_rocksdb_threads[]= static void init_rocksdb_psi_keys() { - const char* category= "rocksdb"; + const char* const category= "rocksdb"; int count; if (PSI_server == nullptr) @@ -1357,21 +1417,21 @@ static void init_rocksdb_psi_keys() static Rdb_drop_index_thread rdb_drop_idx_thread; static void rocksdb_drop_index_wakeup_thread( - my_core::THD* thd __attribute__((__unused__)), - struct st_mysql_sys_var* var __attribute__((__unused__)), - void* var_ptr __attribute__((__unused__)), - const void* save) + my_core::THD* const thd __attribute__((__unused__)), + struct st_mysql_sys_var* const var __attribute__((__unused__)), + void* const var_ptr __attribute__((__unused__)), + const void* const save) { if (*static_cast<const bool*>(save)) { rdb_drop_idx_thread.signal(); } } -static inline uint32_t rocksdb_perf_context_level(THD* thd) +static inline uint32_t rocksdb_perf_context_level(THD* const thd) { DBUG_ASSERT(thd != nullptr); - int session_perf_context_level= THDVAR(thd, perf_context_level); + const int session_perf_context_level= THDVAR(thd, perf_context_level); if (session_perf_context_level > rocksdb::PerfLevel::kUninitialized) { return session_perf_context_level; } @@ -1381,7 +1441,7 @@ static inline uint32_t rocksdb_perf_context_level(THD* thd) value. */ - int global_perf_context_level= THDVAR(nullptr, perf_context_level); + const int global_perf_context_level= THDVAR(nullptr, perf_context_level); if (global_perf_context_level > rocksdb::PerfLevel::kUninitialized) { return global_perf_context_level; } @@ -1397,7 +1457,7 @@ static inline uint32_t rocksdb_perf_context_level(THD* thd) interface Rdb_tx_list_walker { virtual ~Rdb_tx_list_walker() {} - virtual void process_tran(const Rdb_transaction*) = 0; + virtual void process_tran(const Rdb_transaction* const) = 0; }; /* @@ -1412,7 +1472,10 @@ class Rdb_snapshot_notifier : public rocksdb::TransactionNotifier void SnapshotCreated(const rocksdb::Snapshot *snapshot) override; public: - explicit Rdb_snapshot_notifier(Rdb_transaction* owning_tx) + Rdb_snapshot_notifier(const Rdb_snapshot_notifier&) = delete; + Rdb_snapshot_notifier& operator=(const Rdb_snapshot_notifier&) = delete; + + explicit Rdb_snapshot_notifier(Rdb_transaction* const owning_tx) : m_owning_tx(owning_tx) {} // If the owning Rdb_transaction gets destructed we need to not reference @@ -1467,10 +1530,17 @@ public: const char* m_mysql_log_file_name; my_off_t m_mysql_log_offset; const char* m_mysql_gtid; + const char* m_mysql_max_gtid; String m_detailed_error; int64_t m_snapshot_timestamp= 0; bool m_ddl_transaction; + /* + for distinction between rdb_transaction_impl and rdb_writebatch_impl + when using walk tx list + */ + virtual bool is_writebatch_trx() const = 0; + static void init_mutex() { mysql_mutex_init(key_mutex_tx_list, &s_tx_list_mutex, MY_MUTEX_INIT_FAST); @@ -1484,17 +1554,20 @@ public: static void walk_tx_list(Rdb_tx_list_walker* walker) { + DBUG_ASSERT(walker != nullptr); + mysql_mutex_lock(&s_tx_list_mutex); for (auto it : s_tx_list) walker->process_tran(it); mysql_mutex_unlock(&s_tx_list_mutex); } - int set_status_error(THD *thd, const rocksdb::Status &s, - const std::shared_ptr<const Rdb_key_def>& kd, - Rdb_tbl_def *tbl_def) + int set_status_error(THD* const thd, const rocksdb::Status &s, + const Rdb_key_def& kd, + Rdb_tbl_def* const tbl_def) { DBUG_ASSERT(!s.ok()); + DBUG_ASSERT(tbl_def != nullptr); if (s.IsTimedOut()) { @@ -1509,17 +1582,40 @@ public: my_core::thd_mark_transaction_to_rollback(thd, false /*just statement*/); m_detailed_error.copy(timeout_message("index", tbl_def->full_tablename().c_str(), - kd->get_name().c_str())); + kd.get_name().c_str())); return HA_ERR_LOCK_WAIT_TIMEOUT; } - if (s.IsBusy()) + + if (s.IsDeadlock()) + { + my_core::thd_mark_transaction_to_rollback(thd, + false /* just statement */); + return HA_ERR_LOCK_DEADLOCK; + } + else if (s.IsBusy()) { rocksdb_snapshot_conflict_errors++; + if (rocksdb_print_snapshot_conflict_queries) + { + char user_host_buff[MAX_USER_HOST_SIZE + 1]; + make_user_name(thd, user_host_buff); + // NO_LINT_DEBUG + sql_print_warning("Got snapshot conflict errors: User: %s " + "Query: %s", user_host_buff, thd->query()); + } return HA_ERR_LOCK_DEADLOCK; } - /* TODO: who returns HA_ERR_ROCKSDB_TOO_MANY_LOCKS now?? */ + if (s.IsLockLimit()) + { + return HA_ERR_ROCKSDB_TOO_MANY_LOCKS; + } + + if (s.IsIOError() || s.IsCorruption()) + { + rdb_handle_io_error(s, RDB_IO_ERROR_GENERAL); + } my_error(ER_INTERNAL_ERROR, MYF(0), s.ToString().c_str()); return HA_ERR_INTERNAL_ERROR; } @@ -1527,7 +1623,7 @@ public: THD* get_thd() const { return m_thd; } /* Used for tracking io_perf counters */ - void io_perf_start(Rdb_io_perf *io_perf) + void io_perf_start(Rdb_io_perf* const io_perf) { /* Since perf_context is tracked per thread, it is difficult and expensive @@ -1560,7 +1656,7 @@ public: } } - void io_perf_end_and_record(Rdb_io_perf *io_perf) + void io_perf_end_and_record(Rdb_io_perf* const io_perf) { if (m_tbl_io_perf == io_perf) { @@ -1579,11 +1675,13 @@ public: ulonglong get_write_count() const { return m_write_count; } + int get_timeout_sec() const { return m_timeout_sec; } + ulonglong get_lock_count() const { return m_lock_count; } virtual void set_sync(bool sync)= 0; - virtual void release_lock(rocksdb::ColumnFamilyHandle* column_family, + virtual void release_lock(rocksdb::ColumnFamilyHandle* const column_family, const std::string& rowkey)= 0; virtual bool prepare(const rocksdb::TransactionName& name)= 0; @@ -1626,18 +1724,21 @@ public: else { my_core::thd_binlog_pos(m_thd, &m_mysql_log_file_name, - &m_mysql_log_offset, &m_mysql_gtid); + &m_mysql_log_offset, &m_mysql_gtid, + &m_mysql_max_gtid); binlog_manager.update(m_mysql_log_file_name, m_mysql_log_offset, - m_mysql_gtid, get_write_batch()); + m_mysql_max_gtid, get_write_batch()); return commit_no_binlog(); } } virtual void rollback()= 0; - void snapshot_created(const rocksdb::Snapshot *snapshot) + void snapshot_created(const rocksdb::Snapshot* const snapshot) { + DBUG_ASSERT(snapshot != nullptr); + m_read_opts.snapshot = snapshot; rdb->GetEnv()->GetCurrentTime(&m_snapshot_timestamp); m_is_delayed_snapshot = false; @@ -1676,7 +1777,7 @@ public: return rc; } - void start_bulk_load(ha_rocksdb* bulk_load) + void start_bulk_load(ha_rocksdb* const bulk_load) { /* If we already have an open bulk load of a table and the name doesn't @@ -1684,17 +1785,19 @@ public: multiple bulk loads to occur on a partitioned table, but then closes them all out when we switch to another table. */ + DBUG_ASSERT(bulk_load != nullptr); + if (!m_curr_bulk_load.empty() && !bulk_load->same_table(*m_curr_bulk_load[0])) { - auto res= finish_bulk_load(); + const auto res= finish_bulk_load(); SHIP_ASSERT(res == 0); } m_curr_bulk_load.push_back(bulk_load); } - void end_bulk_load(ha_rocksdb* bulk_load) + void end_bulk_load(ha_rocksdb* const bulk_load) { for (auto it = m_curr_bulk_load.begin(); it != m_curr_bulk_load.end(); it++) @@ -1710,6 +1813,11 @@ public: SHIP_ASSERT(0); } + int num_ongoing_bulk_load() const + { + return m_curr_bulk_load.size(); + } + /* Flush the data accumulated so far. This assumes we're doing a bulk insert. @@ -1737,13 +1845,14 @@ public: return false; } - virtual rocksdb::Status put(rocksdb::ColumnFamilyHandle* column_family, + virtual rocksdb::Status put(rocksdb::ColumnFamilyHandle* const column_family, const rocksdb::Slice& key, const rocksdb::Slice& value)= 0; - virtual rocksdb::Status delete_key(rocksdb::ColumnFamilyHandle* column_family, - const rocksdb::Slice& key)= 0; + virtual rocksdb::Status delete_key( + rocksdb::ColumnFamilyHandle* const column_family, + const rocksdb::Slice& key)= 0; virtual rocksdb::Status single_delete( - rocksdb::ColumnFamilyHandle* column_family, + rocksdb::ColumnFamilyHandle* const column_family, const rocksdb::Slice& key)= 0; virtual bool has_modifications() const= 0; @@ -1758,22 +1867,24 @@ public: return get_indexed_write_batch()->GetWriteBatch(); } - virtual rocksdb::Status get(rocksdb::ColumnFamilyHandle* column_family, + virtual rocksdb::Status get(rocksdb::ColumnFamilyHandle* const column_family, const rocksdb::Slice& key, std::string* value) const= 0; virtual rocksdb::Status get_for_update( - rocksdb::ColumnFamilyHandle* column_family, - const rocksdb::Slice& key, std::string* value)= 0; + rocksdb::ColumnFamilyHandle* const column_family, + const rocksdb::Slice& key, std::string* const value, bool exclusive)= 0; - rocksdb::Iterator *get_iterator(rocksdb::ColumnFamilyHandle* column_family, - bool skip_bloom_filter, - bool fill_cache, - bool read_current= false, - bool create_snapshot= true) + rocksdb::Iterator *get_iterator( + rocksdb::ColumnFamilyHandle* const column_family, + bool skip_bloom_filter, + bool fill_cache, + bool read_current= false, + bool create_snapshot= true) { // Make sure we are not doing both read_current (which implies we don't // want a snapshot) and create_snapshot which makes sure we create // a snapshot + DBUG_ASSERT(column_family != nullptr); DBUG_ASSERT(!read_current || !create_snapshot); if (create_snapshot) @@ -1817,7 +1928,7 @@ public: return true; } - int rollback_to_savepoint(void *savepoint) + int rollback_to_savepoint(void* const savepoint) { if (has_modifications()) { @@ -1852,7 +1963,7 @@ public: m_tx_read_only= val; } - explicit Rdb_transaction(THD *thd): m_thd(thd), m_tbl_io_perf(nullptr) + explicit Rdb_transaction(THD* const thd): m_thd(thd), m_tbl_io_perf(nullptr) { mysql_mutex_lock(&s_tx_list_mutex); s_tx_list.insert(this); @@ -1884,7 +1995,7 @@ class Rdb_transaction_impl : public Rdb_transaction void set_lock_timeout(int timeout_sec_arg) override { if (m_rocksdb_tx) - m_rocksdb_tx->SetLockTimeout(m_timeout_sec * 1000); + m_rocksdb_tx->SetLockTimeout(rdb_convert_sec_to_ms(m_timeout_sec)); } void set_sync(bool sync) override @@ -1892,7 +2003,7 @@ class Rdb_transaction_impl : public Rdb_transaction m_rocksdb_tx->GetWriteOptions()->sync= sync; } - void release_lock(rocksdb::ColumnFamilyHandle* column_family, + void release_lock(rocksdb::ColumnFamilyHandle* const column_family, const std::string &rowkey) override { if (!THDVAR(m_thd, lock_scanned_rows)) @@ -1901,6 +2012,8 @@ class Rdb_transaction_impl : public Rdb_transaction } } + virtual bool is_writebatch_trx() const override { return false; } + private: void release_tx(void) { @@ -1934,7 +2047,7 @@ class Rdb_transaction_impl : public Rdb_transaction { bool res= false; release_snapshot(); - rocksdb::Status s= m_rocksdb_tx->Commit(); + const rocksdb::Status s= m_rocksdb_tx->Commit(); if (!s.ok()) { rdb_handle_io_error(s, RDB_IO_ERROR_TX_COMMIT); @@ -2016,37 +2129,35 @@ class Rdb_transaction_impl : public Rdb_transaction return m_read_opts.snapshot != nullptr; } - const char *err_too_many_locks= - "Number of locks held by the transaction exceeded @@rocksdb_max_row_locks"; - - rocksdb::Status put(rocksdb::ColumnFamilyHandle* column_family, + rocksdb::Status put(rocksdb::ColumnFamilyHandle* const column_family, const rocksdb::Slice& key, const rocksdb::Slice& value) override { ++m_write_count; ++m_lock_count; if (m_write_count > m_max_row_locks || m_lock_count > m_max_row_locks) - return rocksdb::Status::Aborted(rocksdb::Slice(err_too_many_locks)); + return rocksdb::Status::Aborted(rocksdb::Status::kLockLimit); return m_rocksdb_tx->Put(column_family, key, value); } - rocksdb::Status delete_key(rocksdb::ColumnFamilyHandle* column_family, + rocksdb::Status delete_key(rocksdb::ColumnFamilyHandle* const column_family, const rocksdb::Slice& key) override { ++m_write_count; ++m_lock_count; if (m_write_count > m_max_row_locks || m_lock_count > m_max_row_locks) - return rocksdb::Status::Aborted(rocksdb::Slice(err_too_many_locks)); + return rocksdb::Status::Aborted(rocksdb::Status::kLockLimit); return m_rocksdb_tx->Delete(column_family, key); } - rocksdb::Status single_delete(rocksdb::ColumnFamilyHandle* column_family, - const rocksdb::Slice& key) override + rocksdb::Status single_delete( + rocksdb::ColumnFamilyHandle* const column_family, + const rocksdb::Slice& key) override { ++m_write_count; ++m_lock_count; if (m_write_count > m_max_row_locks || m_lock_count > m_max_row_locks) - return rocksdb::Status::Aborted(rocksdb::Slice(err_too_many_locks)); + return rocksdb::Status::Aborted(rocksdb::Status::kLockLimit); return m_rocksdb_tx->SingleDelete(column_family, key); } @@ -2076,29 +2187,34 @@ class Rdb_transaction_impl : public Rdb_transaction return m_rocksdb_tx->GetWriteBatch(); } - rocksdb::Status get(rocksdb::ColumnFamilyHandle* column_family, + rocksdb::Status get(rocksdb::ColumnFamilyHandle* const column_family, const rocksdb::Slice& key, std::string* value) const override { return m_rocksdb_tx->Get(m_read_opts, column_family, key, value); } - rocksdb::Status get_for_update(rocksdb::ColumnFamilyHandle* column_family, - const rocksdb::Slice& key, - std::string* value) override + rocksdb::Status get_for_update( + rocksdb::ColumnFamilyHandle* const column_family, + const rocksdb::Slice& key, + std::string* const value, bool exclusive) override { if (++m_lock_count > m_max_row_locks) - return rocksdb::Status::Aborted(rocksdb::Slice(err_too_many_locks)); - return m_rocksdb_tx->GetForUpdate(m_read_opts, column_family, key, value); + return rocksdb::Status::Aborted(rocksdb::Status::kLockLimit); + + return m_rocksdb_tx->GetForUpdate(m_read_opts, column_family, key, value, + exclusive); } rocksdb::Iterator *get_iterator(const rocksdb::ReadOptions &options, - rocksdb::ColumnFamilyHandle* column_family) - override + rocksdb::ColumnFamilyHandle* const column_family) + override { return m_rocksdb_tx->GetIterator(options, column_family); } + const rocksdb::Transaction* get_rdb_trx() const { return m_rocksdb_tx; } + bool is_tx_started() const override { return (m_rocksdb_tx != nullptr); @@ -2109,7 +2225,8 @@ class Rdb_transaction_impl : public Rdb_transaction rocksdb::TransactionOptions tx_opts; rocksdb::WriteOptions write_opts; tx_opts.set_snapshot= false; - tx_opts.lock_timeout= m_timeout_sec * 1000; + tx_opts.lock_timeout= rdb_convert_sec_to_ms(m_timeout_sec); + tx_opts.deadlock_detect= THDVAR(m_thd, deadlock_detect); write_opts.sync= THDVAR(m_thd, write_sync); write_opts.disableWAL= THDVAR(m_thd, write_disable_wal); @@ -2155,10 +2272,10 @@ class Rdb_transaction_impl : public Rdb_transaction /* TODO: here we must release the locks taken since the start_stmt() call */ if (m_rocksdb_tx) { - const rocksdb::Snapshot *org_snapshot = m_rocksdb_tx->GetSnapshot(); + const rocksdb::Snapshot* const org_snapshot = m_rocksdb_tx->GetSnapshot(); m_rocksdb_tx->RollbackToSavePoint(); - const rocksdb::Snapshot *cur_snapshot = m_rocksdb_tx->GetSnapshot(); + const rocksdb::Snapshot* const cur_snapshot = m_rocksdb_tx->GetSnapshot(); if (org_snapshot != cur_snapshot) { if (org_snapshot != nullptr) @@ -2173,7 +2290,7 @@ class Rdb_transaction_impl : public Rdb_transaction } } - explicit Rdb_transaction_impl(THD *thd) : + explicit Rdb_transaction_impl(THD* const thd) : Rdb_transaction(thd), m_rocksdb_tx(nullptr) { // Create a notifier that can be called when a snapshot gets generated. @@ -2224,8 +2341,8 @@ class Rdb_writebatch_impl : public Rdb_transaction { bool res= false; release_snapshot(); - rocksdb::Status s= rdb->GetBaseDB()->Write(write_opts, - m_batch->GetWriteBatch()); + const rocksdb::Status s= rdb->GetBaseDB()->Write(write_opts, + m_batch->GetWriteBatch()); if (!s.ok()) { rdb_handle_io_error(s, RDB_IO_ERROR_TX_COMMIT); @@ -2239,6 +2356,8 @@ class Rdb_writebatch_impl : public Rdb_transaction return res; } public: + bool is_writebatch_trx() const override { return true; } + void set_lock_timeout(int timeout_sec_arg) override { // Nothing to do here. @@ -2249,7 +2368,7 @@ class Rdb_writebatch_impl : public Rdb_transaction write_opts.sync= sync; } - void release_lock(rocksdb::ColumnFamilyHandle* column_family, + void release_lock(rocksdb::ColumnFamilyHandle* const column_family, const std::string &rowkey) override { // Nothing to do here since we don't hold any row locks. @@ -2281,7 +2400,7 @@ class Rdb_writebatch_impl : public Rdb_transaction } } - rocksdb::Status put(rocksdb::ColumnFamilyHandle* column_family, + rocksdb::Status put(rocksdb::ColumnFamilyHandle* const column_family, const rocksdb::Slice& key, const rocksdb::Slice& value) override { @@ -2292,7 +2411,7 @@ class Rdb_writebatch_impl : public Rdb_transaction return rocksdb::Status::OK(); } - rocksdb::Status delete_key(rocksdb::ColumnFamilyHandle* column_family, + rocksdb::Status delete_key(rocksdb::ColumnFamilyHandle* const column_family, const rocksdb::Slice& key) override { ++m_write_count; @@ -2300,8 +2419,9 @@ class Rdb_writebatch_impl : public Rdb_transaction return rocksdb::Status::OK(); } - rocksdb::Status single_delete(rocksdb::ColumnFamilyHandle* column_family, - const rocksdb::Slice& key) override + rocksdb::Status single_delete( + rocksdb::ColumnFamilyHandle* const column_family, + const rocksdb::Slice& key) override { ++m_write_count; m_batch->SingleDelete(column_family, key); @@ -2324,26 +2444,27 @@ class Rdb_writebatch_impl : public Rdb_transaction return m_batch; } - rocksdb::Status get(rocksdb::ColumnFamilyHandle* column_family, + rocksdb::Status get(rocksdb::ColumnFamilyHandle* const column_family, const rocksdb::Slice& key, - std::string* value) const override + std::string* const value) const override { return m_batch->GetFromBatchAndDB( rdb, m_read_opts, column_family, key, value); } - rocksdb::Status get_for_update(rocksdb::ColumnFamilyHandle* column_family, - const rocksdb::Slice& key, - std::string* value) override + rocksdb::Status get_for_update( + rocksdb::ColumnFamilyHandle* const column_family, + const rocksdb::Slice& key, + std::string* const value, bool exclusive) override { return get(column_family, key, value); } rocksdb::Iterator *get_iterator(const rocksdb::ReadOptions &options, - rocksdb::ColumnFamilyHandle* column_family) - override + rocksdb::ColumnFamilyHandle* const column_family) + override { - auto it = rdb->NewIterator(options); + const auto it = rdb->NewIterator(options); return m_batch->NewIteratorWithBase(it); } @@ -2372,7 +2493,7 @@ class Rdb_writebatch_impl : public Rdb_transaction m_batch->RollbackToSavePoint(); } - explicit Rdb_writebatch_impl(THD *thd) : + explicit Rdb_writebatch_impl(THD* const thd) : Rdb_transaction(thd), m_batch(nullptr) { m_batch = new rocksdb::WriteBatchWithIndex(rocksdb::BytewiseComparator(), @@ -2386,7 +2507,8 @@ class Rdb_writebatch_impl : public Rdb_transaction } }; -void Rdb_snapshot_notifier::SnapshotCreated(const rocksdb::Snapshot *snapshot) +void Rdb_snapshot_notifier::SnapshotCreated( + const rocksdb::Snapshot* const snapshot) { if (m_owning_tx != nullptr) { @@ -2397,7 +2519,7 @@ void Rdb_snapshot_notifier::SnapshotCreated(const rocksdb::Snapshot *snapshot) std::multiset<Rdb_transaction*> Rdb_transaction::s_tx_list; mysql_mutex_t Rdb_transaction::s_tx_list_mutex; -static Rdb_transaction* &get_tx_from_thd(THD *thd) +static Rdb_transaction* &get_tx_from_thd(THD* const thd) { return *reinterpret_cast<Rdb_transaction**>( my_core::thd_ha_data(thd, rocksdb_hton)); @@ -2411,7 +2533,10 @@ class Rdb_perf_context_guard THD *m_thd; public: - explicit Rdb_perf_context_guard(THD *thd) : m_thd(thd) + Rdb_perf_context_guard(const Rdb_perf_context_guard&) = delete; + Rdb_perf_context_guard& operator=(const Rdb_perf_context_guard&) = delete; + + explicit Rdb_perf_context_guard(THD* const thd) : m_thd(thd) { Rdb_transaction*& tx= get_tx_from_thd(m_thd); /* @@ -2440,7 +2565,7 @@ class Rdb_perf_context_guard TODO: maybe, call this in external_lock() and store in ha_rocksdb.. */ -static Rdb_transaction *get_or_create_tx(THD *thd) +static Rdb_transaction *get_or_create_tx(THD* const thd) { Rdb_transaction*& tx= get_tx_from_thd(thd); // TODO: this is called too many times.. O(#rows) @@ -2468,7 +2593,7 @@ static Rdb_transaction *get_or_create_tx(THD *thd) } -static int rocksdb_close_connection(handlerton* hton, THD* thd) +static int rocksdb_close_connection(handlerton* const hton, THD* const thd) { Rdb_transaction*& tx= get_tx_from_thd(thd); if (tx != nullptr) @@ -2506,7 +2631,7 @@ static std::string rdb_xid_to_string(const XID& src) */ uchar fidbuf[RDB_FORMATID_SZ]; int64 signed_fid8= src.formatID; - uint64 raw_fid8= *reinterpret_cast<uint64*>(&signed_fid8); + const uint64 raw_fid8= *reinterpret_cast<uint64*>(&signed_fid8); rdb_netbuf_store_uint64(fidbuf, raw_fid8); buf.append(reinterpret_cast<const char*>(fidbuf), RDB_FORMATID_SZ); @@ -2521,11 +2646,13 @@ static std::string rdb_xid_to_string(const XID& src) Called by hton->flush_logs after MySQL group commit prepares a set of transactions. */ -static bool rocksdb_flush_wal(handlerton* hton __attribute__((__unused__)), - ulonglong target_lsn __attribute__((__unused__))) +static bool rocksdb_flush_wal( + handlerton* const hton __attribute__((__unused__)), + ulonglong target_lsn __attribute__((__unused__))) { DBUG_ASSERT(rdb != nullptr); - rocksdb::Status s= rdb->SyncWAL(); + rocksdb_wal_group_syncs++; + const rocksdb::Status s= rdb->SyncWAL(); if (!s.ok()) { return 1; } @@ -2536,8 +2663,8 @@ static bool rocksdb_flush_wal(handlerton* hton __attribute__((__unused__)), For a slave, prepare() updates the slave_gtid_info table which tracks the replication progress. */ -static int rocksdb_prepare(handlerton* hton, THD* thd, bool prepare_tx, - bool async) +static int rocksdb_prepare(handlerton* const hton, THD* const thd, + bool prepare_tx, bool async) { Rdb_transaction*& tx= get_tx_from_thd(thd); if (!tx->can_prepare()) @@ -2550,8 +2677,8 @@ static int rocksdb_prepare(handlerton* hton, THD* thd, bool prepare_tx, this is an SQL statement end and autocommit is on */ std::vector<st_slave_gtid_info> slave_gtid_info; my_core::thd_slave_gtid_info(thd, &slave_gtid_info); - for (auto it : slave_gtid_info) { - rocksdb::WriteBatchBase* write_batch = tx->get_blind_write_batch(); + for (const auto &it : slave_gtid_info) { + rocksdb::WriteBatchBase* const write_batch = tx->get_blind_write_batch(); binlog_manager.update_slave_gtid_info(it.id, it.db, it.gtid, write_batch); } @@ -2564,7 +2691,8 @@ static int rocksdb_prepare(handlerton* hton, THD* thd, bool prepare_tx, if (!tx->prepare(rdb_xid_to_string(xid))) { return 1; } - if (thd->durability_property == HA_IGNORE_DURABILITY) { + if (thd->durability_property == HA_IGNORE_DURABILITY + && THDVAR(thd, write_sync)) { /** we set the log sequence as '1' just to trigger hton->flush_logs */ @@ -2582,14 +2710,14 @@ static int rocksdb_prepare(handlerton* hton, THD* thd, bool prepare_tx, do nothing for prepare/commit by xid this is needed to avoid crashes in XA scenarios */ -static int rocksdb_commit_by_xid(handlerton* hton, XID* xid) +static int rocksdb_commit_by_xid(handlerton* const hton, XID* const xid) { - auto name= rdb_xid_to_string(*xid); - rocksdb::Transaction *trx= rdb->GetTransactionByName(name); + const auto name= rdb_xid_to_string(*xid); + rocksdb::Transaction* const trx= rdb->GetTransactionByName(name); if (trx == nullptr) { return 1; } - rocksdb::Status s= trx->Commit(); + const rocksdb::Status s= trx->Commit(); if (!s.ok()) { return 1; } @@ -2597,15 +2725,16 @@ static int rocksdb_commit_by_xid(handlerton* hton, XID* xid) return 0; } -static int rocksdb_rollback_by_xid(handlerton* hton __attribute__((__unused__)), - XID* xid) +static int rocksdb_rollback_by_xid( + handlerton* const hton __attribute__((__unused__)), + XID* const xid) { - auto name= rdb_xid_to_string(*xid); - rocksdb::Transaction *trx= rdb->GetTransactionByName(name); + const auto name= rdb_xid_to_string(*xid); + rocksdb::Transaction* const trx= rdb->GetTransactionByName(name); if (trx == nullptr) { return 1; } - rocksdb::Status s= trx->Rollback(); + const rocksdb::Status s= trx->Rollback(); if (!s.ok()) { return 1; } @@ -2616,13 +2745,13 @@ static int rocksdb_rollback_by_xid(handlerton* hton __attribute__((__unused__)), /** Rebuilds an XID from a serialized version stored in a string. */ -static void rdb_xid_from_string(const std::string& src, XID *dst) +static void rdb_xid_from_string(const std::string& src, XID* const dst) { DBUG_ASSERT(dst != nullptr); uint offset= 0; uint64 raw_fid8= rdb_netbuf_to_uint64(reinterpret_cast<const uchar*>(src.data())); - int64 signed_fid8= *reinterpret_cast<int64*>(&raw_fid8); + const int64 signed_fid8= *reinterpret_cast<int64*>(&raw_fid8); dst->formatID= signed_fid8; offset += RDB_FORMATID_SZ; dst->gtrid_length= src.at(offset); @@ -2640,8 +2769,10 @@ static void rdb_xid_from_string(const std::string& src, XID *dst) Reading last committed binary log info from RocksDB system row. The info is needed for crash safe slave/master to work. */ -static int rocksdb_recover(handlerton* hton, XID* xid_list, uint len, - char* binlog_file, my_off_t* binlog_pos) +static int rocksdb_recover(handlerton* const hton, XID* const xid_list, + uint len, char* const binlog_file, + my_off_t* const binlog_pos, + Gtid* const binlog_max_gtid) { if (binlog_file && binlog_pos) { @@ -2658,6 +2789,9 @@ static int rocksdb_recover(handlerton* hton, XID* xid_list, uint len, " file name %s\n", pos, file_buf); if (*gtid_buf) { + global_sid_lock->rdlock(); + binlog_max_gtid->parse(global_sid_map, gtid_buf); + global_sid_lock->unlock(); fprintf(stderr, "RocksDB: Last MySQL Gtid %s\n", gtid_buf); } } @@ -2686,7 +2820,8 @@ static int rocksdb_recover(handlerton* hton, XID* xid_list, uint len, return count; } -static int rocksdb_commit(handlerton* hton, THD* thd, bool commit_tx, bool) +static int rocksdb_commit(handlerton* const hton, THD* const thd, + bool commit_tx, bool) { DBUG_ENTER("rocksdb_commit"); @@ -2731,7 +2866,8 @@ static int rocksdb_commit(handlerton* hton, THD* thd, bool commit_tx, bool) } -static int rocksdb_rollback(handlerton* hton, THD* thd, bool rollback_tx) +static int rocksdb_rollback(handlerton* const hton, THD* const thd, + bool rollback_tx) { Rdb_perf_context_guard guard(thd); Rdb_transaction*& tx= get_tx_from_thd(thd); @@ -2771,7 +2907,7 @@ static int rocksdb_rollback(handlerton* hton, THD* thd, bool rollback_tx) return 0; } -static bool print_stats(THD* thd, +static bool print_stats(THD* const thd, std::string const& type, std::string const& name, std::string const& status, @@ -2782,26 +2918,55 @@ static bool print_stats(THD* thd, } static std::string format_string( - const char *format, + const char* const format, ...) { std::string res; va_list args; va_list args_copy; + char static_buff[256]; + + DBUG_ASSERT(format != nullptr); va_start(args, format); va_copy(args_copy, args); - size_t len = vsnprintf(nullptr, 0, format, args) + 1; + // Calculate how much space we will need + int len = vsnprintf(nullptr, 0, format, args); va_end(args); - if (len == 0) { + if (len < 0) + { + res = std::string("<format error>"); + } + else if (len == 0) + { + // Shortcut for an empty string res = std::string(""); } - else { - char buff[len]; + else + { + // For short enough output use a static buffer + char* buff= static_buff; + std::unique_ptr<char[]> dynamic_buff= nullptr; + + len++; // Add one for null terminator + + // for longer output use an allocated buffer + if (static_cast<uint>(len) > sizeof(static_buff)) + { + dynamic_buff.reset(new char[len]); + buff= dynamic_buff.get(); + } + + // Now re-do the vsnprintf with the buffer which is now large enough (void) vsnprintf(buff, len, format, args_copy); + // Convert to a std::string. Note we could have created a std::string + // large enough and then converted the buffer to a 'char*' and created + // the output in place. This would probably work but feels like a hack. + // Since this isn't code that needs to be super-performant we are going + // with this 'safer' method. res = std::string(buff); } @@ -2858,8 +3023,10 @@ class Rdb_snapshot_status : public Rdb_tx_list_walker /* Implement Rdb_transaction interface */ /* Create one row in the snapshot status table */ - void process_tran(const Rdb_transaction *tx) override + void process_tran(const Rdb_transaction* const tx) override { + DBUG_ASSERT(tx != nullptr); + /* Calculate the duration the snapshot has existed */ int64_t snapshot_timestamp = tx->m_snapshot_timestamp; if (snapshot_timestamp != 0) @@ -2868,27 +3035,129 @@ class Rdb_snapshot_status : public Rdb_tx_list_walker rdb->GetEnv()->GetCurrentTime(&curr_time); THD* thd = tx->get_thd(); - + char buffer[1024]; + thd_security_context(thd, buffer, sizeof buffer, 0); m_data += format_string("---SNAPSHOT, ACTIVE %lld sec\n" - "MySQL thread id %lu, OS thread handle %p\n" + "%s\n" "lock count %llu, write count %llu\n", curr_time - snapshot_timestamp, - my_core::thd_get_thread_id(thd), thd, + buffer, tx->get_lock_count(), tx->get_write_count()); } } }; +/** + * @brief + * walks through all non-replication transactions and copies + * out relevant information for information_schema.rocksdb_trx + */ +class Rdb_trx_info_aggregator : public Rdb_tx_list_walker +{ + private: + std::vector<Rdb_trx_info> *m_trx_info; + + public: + explicit Rdb_trx_info_aggregator(std::vector<Rdb_trx_info>* const trx_info) : + m_trx_info(trx_info) {} + + void process_tran(const Rdb_transaction* const tx) override + { + static const std::map<int, std::string> state_map = { + {rocksdb::Transaction::STARTED, "STARTED"}, + {rocksdb::Transaction::AWAITING_PREPARE, "AWAITING_PREPARE"}, + {rocksdb::Transaction::PREPARED, "PREPARED"}, + {rocksdb::Transaction::AWAITING_COMMIT, "AWAITING_COMMIT"}, + {rocksdb::Transaction::COMMITED, "COMMITED"}, + {rocksdb::Transaction::AWAITING_ROLLBACK, "AWAITING_ROLLBACK"}, + {rocksdb::Transaction::ROLLEDBACK, "ROLLEDBACK"}, + }; + + DBUG_ASSERT(tx != nullptr); + + THD* const thd = tx->get_thd(); + ulong thread_id = thd_thread_id(thd); + + if (tx->is_writebatch_trx()) { + const auto wb_impl = static_cast<const Rdb_writebatch_impl*>(tx); + DBUG_ASSERT(wb_impl); + m_trx_info->push_back({"", /* name */ + 0, /* trx_id */ + wb_impl->get_write_count(), + 0, /* lock_count */ + 0, /* timeout_sec */ + "", /* state */ + "", /* waiting_key */ + 0, /* waiting_cf_id */ + 1, /*is_replication */ + 1, /* skip_trx_api */ + wb_impl->is_tx_read_only(), + 0, /* deadlock detection */ + wb_impl->num_ongoing_bulk_load(), + thread_id, + "" /* query string */ }); + } else { + const auto tx_impl= static_cast<const Rdb_transaction_impl*>(tx); + DBUG_ASSERT(tx_impl); + const rocksdb::Transaction *rdb_trx = tx_impl->get_rdb_trx(); + + if (rdb_trx == nullptr) { + return; + } + + std::string query_str; + LEX_STRING* const lex_str = thd_query_string(thd); + if (lex_str != nullptr && lex_str->str != nullptr) { + query_str = std::string(lex_str->str); + } + + const auto state_it = state_map.find(rdb_trx->GetState()); + DBUG_ASSERT(state_it != state_map.end()); + const int is_replication = (thd->rli_slave != nullptr); + uint32_t waiting_cf_id; + std::string waiting_key; + rdb_trx->GetWaitingTxns(&waiting_cf_id, &waiting_key), + + m_trx_info->push_back({rdb_trx->GetName(), + rdb_trx->GetID(), + tx_impl->get_write_count(), + tx_impl->get_lock_count(), + tx_impl->get_timeout_sec(), + state_it->second, + waiting_key, + waiting_cf_id, + is_replication, + 0, /* skip_trx_api */ + tx_impl->is_tx_read_only(), + rdb_trx->IsDeadlockDetect(), + tx_impl->num_ongoing_bulk_load(), + thread_id, + query_str}); + } + } +}; + +/* + returns a vector of info for all non-replication threads + for use by information_schema.rocksdb_trx +*/ +std::vector<Rdb_trx_info> rdb_get_all_trx_info() { + std::vector<Rdb_trx_info> trx_info; + Rdb_trx_info_aggregator trx_info_agg(&trx_info); + Rdb_transaction::walk_tx_list(&trx_info_agg); + return trx_info; +} + /* Generate the snapshot status table */ -static bool rocksdb_show_snapshot_status(handlerton* hton, - THD* thd, - stat_print_fn* stat_print) +static bool rocksdb_show_snapshot_status(handlerton* const hton, + THD* const thd, + stat_print_fn* const stat_print) { Rdb_snapshot_status showStatus; Rdb_transaction::walk_tx_list(&showStatus); - // Send the result data back to MySQL */ + /* Send the result data back to MySQL */ return print_stats(thd, "SNAPSHOTS", "rocksdb", showStatus.getResult(), stat_print); } @@ -2900,9 +3169,9 @@ static bool rocksdb_show_snapshot_status(handlerton* hton, what column families are there) */ -static bool rocksdb_show_status(handlerton* hton, - THD* thd, - stat_print_fn* stat_print, +static bool rocksdb_show_status(handlerton* const hton, + THD* const thd, + stat_print_fn* const stat_print, enum ha_stat_type stat_type) { bool res= false; @@ -2916,7 +3185,7 @@ static bool rocksdb_show_status(handlerton* hton, } /* Per column family stats */ - for (auto cf_name : cf_manager.get_cf_names()) + for (const auto &cf_name : cf_manager.get_cf_names()) { rocksdb::ColumnFamilyHandle* cfh; bool is_automatic; @@ -2948,13 +3217,13 @@ static bool rocksdb_show_status(handlerton* hton, { rocksdb::ColumnFamilyDescriptor cf_desc; cf_handle->GetDescriptor(&cf_desc); - auto* table_factory = cf_desc.options.table_factory.get(); + auto* const table_factory = cf_desc.options.table_factory.get(); if (table_factory != nullptr) { std::string tf_name = table_factory->Name(); if (tf_name.find("BlockBasedTable") != std::string::npos) { - const rocksdb::BlockBasedTableOptions* bbt_opt = + const rocksdb::BlockBasedTableOptions* const bbt_opt = reinterpret_cast<rocksdb::BlockBasedTableOptions*>( table_factory->GetOptions()); if (bbt_opt != nullptr) @@ -3003,9 +3272,11 @@ static bool rocksdb_show_status(handlerton* hton, return res; } -static inline void rocksdb_register_tx(handlerton *hton, THD *thd, - Rdb_transaction *tx) +static inline void rocksdb_register_tx(handlerton* const hton, THD* const thd, + Rdb_transaction* const tx) { + DBUG_ASSERT(tx != nullptr); + trans_register_ha(thd, FALSE, rocksdb_hton); if (my_core::thd_test_options(thd, OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN)) { @@ -3036,14 +3307,14 @@ static inline void rocksdb_register_tx(handlerton *hton, THD *thd, InnoDB and RocksDB transactions. */ static int rocksdb_start_tx_and_assign_read_view( - handlerton* hton, /*!< in: RocksDB handlerton */ - THD* thd, /*!< in: MySQL thread handle of the + handlerton* const hton, /*!< in: RocksDB handlerton */ + THD* const thd, /*!< in: MySQL thread handle of the user for whom the transaction should be committed */ - char* binlog_file, /* out: binlog file for last commit */ - ulonglong* binlog_pos, /* out: binlog pos for last commit */ - char** gtid_executed, /* out: Gtids logged until last commit */ - int* gtid_executed_length) /*out: Length of gtid_executed string */ + char* const binlog_file, /* out: binlog file for last commit */ + ulonglong* const binlog_pos, /* out: binlog pos for last commit */ + char** gtid_executed, /* out: Gtids logged until last commit */ + int* const gtid_executed_length)/*out: Length of gtid_executed string */ { Rdb_perf_context_guard guard(thd); @@ -3066,7 +3337,7 @@ static int rocksdb_start_tx_and_assign_read_view( return 1; } - Rdb_transaction* tx= get_or_create_tx(thd); + Rdb_transaction* const tx= get_or_create_tx(thd); DBUG_ASSERT(!tx->has_snapshot()); tx->set_tx_read_only(true); rocksdb_register_tx(hton, thd, tx); @@ -3084,20 +3355,22 @@ static int rocksdb_start_tx_and_assign_read_view( * Current SAVEPOINT does not correctly handle ROLLBACK and does not return * errors. This needs to be addressed in future versions (Issue#96). */ -static int rocksdb_savepoint(handlerton *hton, THD *thd, void *savepoint) +static int rocksdb_savepoint(handlerton* const hton, THD* const thd, + void* const savepoint) { return 0; } -static int rocksdb_rollback_to_savepoint(handlerton *hton, THD *thd, - void *savepoint) +static int rocksdb_rollback_to_savepoint(handlerton* const hton, THD* const thd, + void* const savepoint) { Rdb_transaction*& tx= get_tx_from_thd(thd); return tx->rollback_to_savepoint(savepoint); } -static bool rocksdb_rollback_to_savepoint_can_release_mdl(handlerton *hton, - THD *thd) +static bool rocksdb_rollback_to_savepoint_can_release_mdl( + handlerton* const hton, + THD* const thd) { return true; } @@ -3195,7 +3468,7 @@ static void rocksdb_update_table_stats( static rocksdb::Status check_rocksdb_options_compatibility( - const char *dbpath, + const char* const dbpath, const rocksdb::Options& main_opts, const std::vector<rocksdb::ColumnFamilyDescriptor>& cf_descr) { @@ -3253,7 +3526,7 @@ static rocksdb::Status check_rocksdb_options_compatibility( Storage Engine initialization function, invoked when plugin is loaded. */ -static int rocksdb_init_func(void *p) +static int rocksdb_init_func(void* const p) { DBUG_ENTER("rocksdb_init_func"); @@ -3348,12 +3621,22 @@ static int rocksdb_init_func(void *p) (rocksdb_access_hint_on_compaction_start); if (rocksdb_db_options.allow_mmap_reads && - !rocksdb_db_options.allow_os_buffer) + rocksdb_db_options.use_direct_reads) + { + // allow_mmap_reads implies !use_direct_reads and RocksDB will not open if + // mmap_reads and direct_reads are both on. (NO_LINT_DEBUG) + sql_print_error("RocksDB: Can't enable both use_direct_reads " + "and allow_mmap_reads\n"); + rdb_open_tables.free_hash(); + DBUG_RETURN(1); + } + + if (rocksdb_db_options.allow_mmap_writes && + rocksdb_db_options.use_direct_writes) { - // allow_mmap_reads implies allow_os_buffer and RocksDB will not open if - // mmap_reads is on and os_buffer is off. (NO_LINT_DEBUG) - sql_print_error("RocksDB: Can't disable allow_os_buffer " - "if allow_mmap_reads is enabled\n"); + // See above comment for allow_mmap_reads. (NO_LINT_DEBUG) + sql_print_error("RocksDB: Can't enable both use_direct_writes " + "and allow_mmap_writes\n"); rdb_open_tables.free_hash(); DBUG_RETURN(1); } @@ -3418,7 +3701,7 @@ static int rocksdb_init_func(void *p) mysql_mutex_unlock(&rdb_sysvars_mutex); } - if (!rocksdb_cf_options_map.init(ROCKSDB_WRITE_BUFFER_SIZE_DEFAULT, + if (!rocksdb_cf_options_map.init( rocksdb_tbl_options, properties_collector_factory, rocksdb_default_cf_options, @@ -3464,33 +3747,6 @@ static int rocksdb_init_func(void *p) rocksdb::Options main_opts(rocksdb_db_options, rocksdb_cf_options_map.get_defaults()); - /* - Flashcache configuration: - When running on Flashcache, mysqld opens Flashcache device before - initializing storage engines, and setting file descriptor at - cachedev_fd global variable. - RocksDB has Flashcache-aware configuration. When this is enabled, - RocksDB adds background threads into Flashcache blacklists, which - makes sense for Flashcache use cases. - */ - if (cachedev_enabled) - { - flashcache_aware_env= - rocksdb::NewFlashcacheAwareEnv(rocksdb::Env::Default(), - cachedev_fd); - if (flashcache_aware_env.get() == nullptr) - { - // NO_LINT_DEBUG - sql_print_error("RocksDB: Failed to open flashcache device at fd %d", - cachedev_fd); - rdb_open_tables.free_hash(); - DBUG_RETURN(1); - } - sql_print_information("RocksDB: Disabling flashcache on background " - "writer threads, fd %d", cachedev_fd); - main_opts.env= flashcache_aware_env.get(); - } - main_opts.env->SetBackgroundThreads(main_opts.max_background_flushes, rocksdb::Env::Priority::HIGH); main_opts.env->SetBackgroundThreads(main_opts.max_background_compactions, @@ -3557,7 +3813,7 @@ static int rocksdb_init_func(void *p) */ std::vector<rocksdb::ColumnFamilyHandle*> compaction_enabled_cf_handles; compaction_enabled_cf_handles.reserve(compaction_enabled_cf_indices.size()); - for (auto index : compaction_enabled_cf_indices) + for (const auto &index : compaction_enabled_cf_indices) { compaction_enabled_cf_handles.push_back(cf_handles[index]); } @@ -3566,7 +3822,7 @@ static int rocksdb_init_func(void *p) if (!status.ok()) { - std::string err_text= status.ToString(); + const std::string err_text= status.ToString(); // NO_LINT_DEBUG sql_print_error("RocksDB: Error enabling compaction: %s", err_text.c_str()); rdb_open_tables.free_hash(); @@ -3603,6 +3859,18 @@ static int rocksdb_init_func(void *p) rdb->PauseBackgroundWork(); } + // NO_LINT_DEBUG + sql_print_information("RocksDB: global statistics using %s indexer", + STRINGIFY_ARG(RDB_INDEXER)); +#if defined(HAVE_SCHED_GETCPU) + if (sched_getcpu() == -1) + { + // NO_LINT_DEBUG + sql_print_information("RocksDB: sched_getcpu() failed - " + "global statistics will use thread_id_indexer_t instead"); + } +#endif + sql_print_information("RocksDB instance opened"); DBUG_RETURN(0); } @@ -3611,7 +3879,7 @@ static int rocksdb_init_func(void *p) Storage Engine deinitialization function, invoked when plugin is unloaded. */ -static int rocksdb_done_func(void *p) +static int rocksdb_done_func(void* const p) { int error= 0; DBUG_ENTER("rocksdb_done_func"); @@ -3704,7 +3972,7 @@ static int rocksdb_done_func(void *p) */ Rdb_table_handler* Rdb_open_tables_map::get_table_handler( - const char *table_name) + const char* const table_name) { Rdb_table_handler *table_handler; uint length; @@ -3746,7 +4014,7 @@ Rdb_table_handler* Rdb_open_tables_map::get_table_handler( } thr_lock_init(&table_handler->m_thr_lock); - my_io_perf_atomic_init(&table_handler->m_io_perf_read); + table_handler->m_io_perf_read.init(); } DBUG_ASSERT(table_handler->m_ref_count >= 0); table_handler->m_ref_count++; @@ -3787,9 +4055,9 @@ std::vector<std::string> Rdb_open_tables_map::get_table_names(void) const void ha_rocksdb::load_auto_incr_value() { - int save_active_index= active_index; + const int save_active_index= active_index; active_index= table->s->next_number_index; - uint8 save_table_status= table->status; + const uint8 save_table_status= table->status; /* load_auto_incr_value() may be called by statements that @@ -3797,13 +4065,13 @@ void ha_rocksdb::load_auto_incr_value() index_last() creates a snapshot. When a snapshot is created here, it has to be released as well. (GitHub issue#189) */ - Rdb_transaction *tx= get_or_create_tx(table->in_use); - bool is_new_snapshot= !tx->has_snapshot(); + Rdb_transaction* const tx= get_or_create_tx(table->in_use); + const bool is_new_snapshot= !tx->has_snapshot(); // Do a lookup. We only need index column, so it should be index-only. // (another reason to make it index-only is that table->read_set is // not set appropriately and non-index-only lookup will not read the value) - bool save_keyread_only= m_keyread_only; + const bool save_keyread_only= m_keyread_only; m_keyread_only= true; if (!index_last(table->record[0])) @@ -3838,7 +4106,8 @@ void ha_rocksdb::update_auto_incr_val() longlong new_val; field= table->key_info[table->s->next_number_index].key_part[0].field; - my_bitmap_map *old_map= dbug_tmp_use_all_columns(table, table->read_set); + my_bitmap_map* const old_map= + dbug_tmp_use_all_columns(table, table->read_set); new_val= field->val_int() + 1; dbug_tmp_restore_column_map(table->read_set, old_map); @@ -3854,12 +4123,12 @@ void ha_rocksdb::update_auto_incr_val() int ha_rocksdb::load_hidden_pk_value() { - int save_active_index= active_index; + const int save_active_index= active_index; active_index= m_tbl_def->m_key_count - 1; - uint8 save_table_status= table->status; + const uint8 save_table_status= table->status; - Rdb_transaction *tx= get_or_create_tx(table->in_use); - bool is_new_snapshot= !tx->has_snapshot(); + Rdb_transaction* const tx= get_or_create_tx(table->in_use); + const bool is_new_snapshot= !tx->has_snapshot(); // Do a lookup. if (!index_last(table->record[0])) @@ -3901,13 +4170,12 @@ int ha_rocksdb::load_hidden_pk_value() longlong ha_rocksdb::update_hidden_pk_val() { DBUG_ASSERT(has_hidden_pk(table)); - longlong new_val; - new_val= m_tbl_def->m_hidden_pk_val++; + const longlong new_val= m_tbl_def->m_hidden_pk_val++; return new_val; } /* Get the id of the hidden pk id from m_last_rowkey */ -int ha_rocksdb::read_hidden_pk_id_from_rowkey(longlong* hidden_pk_id) +int ha_rocksdb::read_hidden_pk_id_from_rowkey(longlong* const hidden_pk_id) { DBUG_ASSERT(hidden_pk_id != nullptr); DBUG_ASSERT(table != nullptr); @@ -3921,7 +4189,7 @@ int ha_rocksdb::read_hidden_pk_id_from_rowkey(longlong* hidden_pk_id) return 1; const int length= Field_longlong::PACK_LENGTH; - const uchar *from= reinterpret_cast<const uchar*>(reader.read(length)); + const uchar* from= reinterpret_cast<const uchar*>(reader.read(length)); if (from == nullptr) { return 1; /* Mem-comparable image doesn't have enough bytes */ @@ -3940,15 +4208,16 @@ int ha_rocksdb::read_hidden_pk_id_from_rowkey(longlong* hidden_pk_id) */ void Rdb_open_tables_map::release_table_handler( - Rdb_table_handler *table_handler) + Rdb_table_handler* const table_handler) { mysql_mutex_lock(&m_mutex); + DBUG_ASSERT(table_handler != nullptr); DBUG_ASSERT(table_handler->m_ref_count > 0); if (!--table_handler->m_ref_count) { // Last rereference was released. Tear down the hash entry. - auto ret __attribute__((__unused__)) = + const auto ret __attribute__((__unused__)) = my_hash_delete(&m_hash, reinterpret_cast<uchar*>(table_handler)); DBUG_ASSERT(!ret); // the hash entry must actually be found and deleted my_core::thr_lock_delete(&table_handler->m_thr_lock); @@ -3959,19 +4228,19 @@ void Rdb_open_tables_map::release_table_handler( } -static handler* rocksdb_create_handler(my_core::handlerton *hton, - my_core::TABLE_SHARE *table_arg, - my_core::MEM_ROOT *mem_root) +static handler* rocksdb_create_handler(my_core::handlerton* const hton, + my_core::TABLE_SHARE* const table_arg, + my_core::MEM_ROOT* const mem_root) { return new (mem_root) ha_rocksdb(hton, table_arg); } -ha_rocksdb::ha_rocksdb(my_core::handlerton *hton, - my_core::TABLE_SHARE *table_arg) +ha_rocksdb::ha_rocksdb(my_core::handlerton* const hton, + my_core::TABLE_SHARE* const table_arg) : handler(hton, table_arg), m_table_handler(nullptr), m_scan_it(nullptr), - m_scan_it_skips_bloom(false), m_tbl_def(nullptr), - m_pk_descr(nullptr), m_key_descr_arr(nullptr), + m_scan_it_skips_bloom(false), m_scan_it_snapshot(nullptr), + m_tbl_def(nullptr), m_pk_descr(nullptr), m_key_descr_arr(nullptr), m_pk_can_be_decoded(false), m_maybe_unpack_info(false), m_pk_tuple(nullptr), m_pk_packed_tuple(nullptr), @@ -3979,11 +4248,14 @@ ha_rocksdb::ha_rocksdb(my_core::handlerton *hton, m_sk_match_prefix(nullptr), m_sk_match_prefix_buf(nullptr), m_sk_packed_tuple_old(nullptr), m_pack_buffer(nullptr), m_lock_rows(RDB_LOCK_NONE), - m_keyread_only(FALSE), m_encoder_arr(nullptr), + m_keyread_only(FALSE), m_bulk_load_tx(nullptr), m_encoder_arr(nullptr), m_row_checksums_checked(0), m_in_rpl_delete_rows(false), m_in_rpl_update_rows(false), m_force_skip_unique_check(false) -{} +{ + // TODO(alexyang): create a valid PSI_mutex_key for this mutex + mysql_mutex_init(0, &m_bulk_load_mutex, MY_MUTEX_INIT_FAST); +} static const char *ha_rocksdb_exts[] = { @@ -4003,10 +4275,10 @@ bool ha_rocksdb::same_table(const ha_rocksdb& other) const bool ha_rocksdb::init_with_fields() { - uint pk= table_share->primary_key; + const uint pk= table_share->primary_key; if (pk != MAX_KEY) { - uint key_parts= table_share->key_info[pk].user_defined_key_parts; + const uint key_parts= table_share->key_info[pk].user_defined_key_parts; check_keyread_allowed(pk /*PK*/, key_parts-1, true); } else @@ -4028,8 +4300,8 @@ bool ha_rocksdb::init_with_fields() void ha_rocksdb::convert_record_to_storage_format( const rocksdb::Slice& pk_packed_slice, - Rdb_string_writer *pk_unpack_info, - rocksdb::Slice *packed_rec) + Rdb_string_writer* const pk_unpack_info, + rocksdb::Slice* const packed_rec) { DBUG_ASSERT_IMP(m_maybe_unpack_info, pk_unpack_info); m_storage_record.length(0); @@ -4040,9 +4312,7 @@ void ha_rocksdb::convert_record_to_storage_format( // If a primary key may have non-empty unpack_info for certain values, // (m_maybe_unpack_info=TRUE), we write the unpack_info block. The block // itself was prepared in Rdb_key_def::pack_record. - if (m_maybe_unpack_info && - m_pk_descr->m_kv_format_version >= - Rdb_key_def::PRIMARY_FORMAT_VERSION_UPDATE1) + if (m_maybe_unpack_info) { m_storage_record.append(reinterpret_cast<char*>(pk_unpack_info->ptr()), pk_unpack_info->get_current_pos()); @@ -4056,10 +4326,10 @@ void ha_rocksdb::convert_record_to_storage_format( continue; } - Field *field= table->field[i]; + Field* const field= table->field[i]; if (m_encoder_arr[i].maybe_null()) { - char *data= (char*)m_storage_record.ptr(); + char* const data= (char*)m_storage_record.ptr(); if (field->is_null()) { data[m_encoder_arr[i].m_null_offset]|= m_encoder_arr[i].m_null_mask; @@ -4072,7 +4342,7 @@ void ha_rocksdb::convert_record_to_storage_format( { my_core::Field_blob *blob= (my_core::Field_blob*)field; /* Get the number of bytes needed to store length*/ - uint length_bytes= blob->pack_length() - portable_sizeof_char_ptr; + const uint length_bytes= blob->pack_length() - portable_sizeof_char_ptr; /* Store the length of the value */ m_storage_record.append(reinterpret_cast<char*>(blob->ptr), length_bytes); @@ -4084,7 +4354,7 @@ void ha_rocksdb::convert_record_to_storage_format( } else if (m_encoder_arr[i].m_field_type == MYSQL_TYPE_VARCHAR) { - Field_varstring* field_var= (Field_varstring*)field; + Field_varstring* const field_var= (Field_varstring*)field; uint data_len; /* field_var->length_bytes is 1 or 2 */ if (field_var->length_bytes == 1) @@ -4102,17 +4372,17 @@ void ha_rocksdb::convert_record_to_storage_format( else { /* Copy the field data */ - uint len= field->pack_length_in_rec(); + const uint len= field->pack_length_in_rec(); m_storage_record.append(reinterpret_cast<char*>(field->ptr), len); } } - if (should_store_checksums()) + if (should_store_row_debug_checksums()) { - uint32_t key_crc32= my_core::crc32(0, + const uint32_t key_crc32= my_core::crc32(0, rdb_slice_to_uchar_ptr(&pk_packed_slice), pk_packed_slice.size()); - uint32_t val_crc32= my_core::crc32( + const uint32_t val_crc32= my_core::crc32( 0, rdb_mysql_str_to_uchar_str(&m_storage_record), m_storage_record.length()); uchar key_crc_buf[RDB_CHECKSUM_SIZE]; @@ -4136,7 +4406,7 @@ void ha_rocksdb::convert_record_to_storage_format( @detail Two special cases when we still unpack all fields: - When this table is being updated (m_lock_rows==RDB_LOCK_WRITE). - - When @@rocksdb_verify_checksums is ON (In this mode, we need to read all + - When @@rocksdb_verify_row_debug_checksums is ON (In this mode, we need to read all fields to find whether there is a row checksum at the end. We could skip the fields instead of decoding them, but currently we do decoding.) @@ -4159,7 +4429,7 @@ void ha_rocksdb::setup_read_decoders() continue; } - if (m_lock_rows == RDB_LOCK_WRITE || m_verify_checksums || + if (m_lock_rows == RDB_LOCK_WRITE || m_verify_row_debug_checksums || bitmap_is_set(table->read_set, table->field[i]->field_index)) { // We will need to decode this field @@ -4232,8 +4502,9 @@ void dbug_modify_key_varchar8(String &on_disk_rec) #endif -int ha_rocksdb::convert_record_from_storage_format(const rocksdb::Slice *key, - uchar * buf) +int ha_rocksdb::convert_record_from_storage_format( + const rocksdb::Slice* const key, + uchar* const buf) { DBUG_EXECUTE_IF("myrocks_simulate_bad_row_read1", dbug_append_garbage_at_end(m_retrieved_record);); @@ -4242,8 +4513,8 @@ int ha_rocksdb::convert_record_from_storage_format(const rocksdb::Slice *key, DBUG_EXECUTE_IF("myrocks_simulate_bad_row_read3", dbug_modify_rec_varchar12(m_retrieved_record);); - rocksdb::Slice retrieved_rec_slice(&m_retrieved_record.front(), - m_retrieved_record.size()); + const rocksdb::Slice retrieved_rec_slice(&m_retrieved_record.front(), + m_retrieved_record.size()); return convert_record_from_storage_format(key, &retrieved_rec_slice, buf); } @@ -4272,15 +4543,16 @@ int ha_rocksdb::convert_record_from_storage_format(const rocksdb::Slice *key, other Error inpacking the data */ -int ha_rocksdb::convert_record_from_storage_format(const rocksdb::Slice *key, - const rocksdb::Slice *value, - uchar * buf) +int ha_rocksdb::convert_record_from_storage_format( + const rocksdb::Slice* const key, + const rocksdb::Slice* const value, + uchar* const buf) { DBUG_ASSERT(key != nullptr); DBUG_ASSERT(buf != nullptr); Rdb_string_reader reader(value); - my_ptrdiff_t ptr_diff= buf - table->record[0]; + const my_ptrdiff_t ptr_diff= buf - table->record[0]; /* Decode PK fields from the key @@ -4301,8 +4573,7 @@ int ha_rocksdb::convert_record_from_storage_format(const rocksdb::Slice *key, return HA_ERR_INTERNAL_ERROR; } - if (m_maybe_unpack_info && m_pk_descr->m_kv_format_version >= - Rdb_key_def::PRIMARY_FORMAT_VERSION_UPDATE1) + if (m_maybe_unpack_info) { unpack_info= reader.read(RDB_UNPACK_HEADER_SIZE); @@ -4329,10 +4600,10 @@ int ha_rocksdb::convert_record_from_storage_format(const rocksdb::Slice *key, { const Rdb_field_encoder* const field_dec= it->m_field_enc; const bool decode= it->m_decode; - bool isNull = field_dec->maybe_null() && + const bool isNull = field_dec->maybe_null() && ((null_bytes[field_dec->m_null_offset] & field_dec->m_null_mask) != 0); - Field *field= table->field[field_dec->m_field_index]; + Field* const field= table->field[field_dec->m_field_index]; /* Skip the bytes we need to skip */ if (it->m_skip && !reader.read(it->m_skip)) @@ -4363,9 +4634,9 @@ int ha_rocksdb::convert_record_from_storage_format(const rocksdb::Slice *key, if (field_dec->m_field_type == MYSQL_TYPE_BLOB) { - my_core::Field_blob *blob= (my_core::Field_blob*)field; + my_core::Field_blob* const blob= (my_core::Field_blob*)field; /* Get the number of bytes needed to store length*/ - uint length_bytes= blob->pack_length() - portable_sizeof_char_ptr; + const uint length_bytes= blob->pack_length() - portable_sizeof_char_ptr; blob->move_field_offset(ptr_diff); @@ -4378,8 +4649,9 @@ int ha_rocksdb::convert_record_from_storage_format(const rocksdb::Slice *key, memcpy(blob->ptr, data_len_str, length_bytes); - uint32 data_len= blob->get_length((uchar*)data_len_str, length_bytes, - table->s->db_low_byte_first); + const uint32 data_len= blob->get_length((uchar*)data_len_str, + length_bytes, + table->s->db_low_byte_first); const char *blob_ptr; if (!(blob_ptr= reader.read(data_len))) { @@ -4398,7 +4670,7 @@ int ha_rocksdb::convert_record_from_storage_format(const rocksdb::Slice *key, } else if (field_dec->m_field_type == MYSQL_TYPE_VARCHAR) { - Field_varstring* field_var= (Field_varstring*)field; + Field_varstring* const field_var= (Field_varstring*)field; const char *data_len_str; if (!(data_len_str= reader.read(field_var->length_bytes))) return HA_ERR_INTERNAL_ERROR; @@ -4431,7 +4703,7 @@ int ha_rocksdb::convert_record_from_storage_format(const rocksdb::Slice *key, else { const char *data_bytes; - uint len= field_dec->m_pack_length_in_rec; + const uint len= field_dec->m_pack_length_in_rec; if (len > 0) { if ((data_bytes= reader.read(len)) == nullptr) @@ -4444,7 +4716,7 @@ int ha_rocksdb::convert_record_from_storage_format(const rocksdb::Slice *key, } } - if (m_verify_checksums) + if (m_verify_row_debug_checksums) { if (reader.remaining_bytes() == RDB_CHECKSUM_CHUNK_SIZE && reader.read(1)[0] == RDB_CHECKSUM_DATA_TAG) @@ -4454,9 +4726,9 @@ int ha_rocksdb::convert_record_from_storage_format(const rocksdb::Slice *key, uint32_t stored_val_chksum= rdb_netbuf_to_uint32((const uchar*)reader.read(RDB_CHECKSUM_SIZE)); - uint32_t computed_key_chksum= + const uint32_t computed_key_chksum= my_core::crc32(0, rdb_slice_to_uchar_ptr(key), key->size()); - uint32_t computed_val_chksum= + const uint32_t computed_val_chksum= my_core::crc32(0, rdb_slice_to_uchar_ptr(value), value->size() - RDB_CHECKSUM_CHUNK_SIZE); @@ -4488,55 +4760,19 @@ int ha_rocksdb::convert_record_from_storage_format(const rocksdb::Slice *key, } -void ha_rocksdb::get_storage_type(Rdb_field_encoder *encoder, uint kp) +void ha_rocksdb::get_storage_type(Rdb_field_encoder* const encoder, + const uint &kp) { - Field *field= table->key_info[table->s->primary_key].key_part[kp].field; - - if (field->real_type() == MYSQL_TYPE_NEWDECIMAL) + // STORE_SOME uses unpack_info. + if (m_pk_descr->has_unpack_info(kp)) { - // Index-only is supported for DECIMAL columns. - // A DECIMAL value can be restored from its mem-comparable form. - // This works for both the old data format and the new data format. - if (m_pk_descr->m_kv_format_version >= - Rdb_key_def::PRIMARY_FORMAT_VERSION_UPDATE1) - { - // New format, don't store Decimal value in the row. - encoder->m_storage_type= Rdb_field_encoder::STORE_NONE; - } - else - { - // Old format. Pretend there's no way to unpack the decimal from - // its mem-comparable form. - encoder->m_storage_type= Rdb_field_encoder::STORE_ALL; - } - return; + DBUG_ASSERT(m_pk_descr->can_unpack(kp)); + encoder->m_storage_type= Rdb_field_encoder::STORE_SOME; + m_maybe_unpack_info= true; } - - if (m_pk_descr->m_kv_format_version >= - Rdb_key_def::PRIMARY_FORMAT_VERSION_UPDATE1) - { - // STORE_SOME uses unpack_info, so our key must be of version - // higher than PRIMARY_FORMAT_VERSION_UPDATE1 to use this - // feature. - if (m_pk_descr->has_unpack_info(kp)) - { - DBUG_ASSERT(m_pk_descr->can_unpack(kp)); - encoder->m_storage_type= Rdb_field_encoder::STORE_SOME; - m_maybe_unpack_info= true; - } - else if (m_pk_descr->can_unpack(kp)) - { - encoder->m_storage_type= Rdb_field_encoder::STORE_NONE; - } - } - else + else if (m_pk_descr->can_unpack(kp)) { - // For old versions, we can only store none if there is no - // unpack_info (and it's unpackable). - if (m_pk_descr->can_unpack(kp) && !m_pk_descr->has_unpack_info(kp)) - { - encoder->m_storage_type= Rdb_field_encoder::STORE_NONE; - } + encoder->m_storage_type= Rdb_field_encoder::STORE_NONE; } } @@ -4564,7 +4800,7 @@ void ha_rocksdb::setup_field_converters() for (i= 0; i < table->s->fields; i++) { - Field *field= table->field[i]; + Field* const field= table->field[i]; m_encoder_arr[i].m_storage_type= Rdb_field_encoder::STORE_ALL; /* @@ -4580,7 +4816,7 @@ void ha_rocksdb::setup_field_converters() if (!has_hidden_pk(table) && field->part_of_key.is_set(table->s->primary_key)) { - KEY *pk_info= &table->key_info[table->s->primary_key]; + KEY* const pk_info= &table->key_info[table->s->primary_key]; for (uint kp= 0; kp < pk_info->user_defined_key_parts; kp++) { /* key_part->fieldnr is counted from 1 */ @@ -4621,13 +4857,14 @@ void ha_rocksdb::setup_field_converters() m_null_bytes_in_rec= null_bytes; } -int ha_rocksdb::alloc_key_buffers(const TABLE* table_arg, - const Rdb_tbl_def* tbl_def_arg) +int ha_rocksdb::alloc_key_buffers(const TABLE* const table_arg, + const Rdb_tbl_def* const tbl_def_arg) { DBUG_ENTER("ha_rocksdb::alloc_key_buffers"); DBUG_ASSERT(m_pk_tuple == nullptr); + DBUG_ASSERT(tbl_def_arg != nullptr); - std::shared_ptr<Rdb_key_def>* kd_arr= tbl_def_arg->m_key_descr_arr; + std::shared_ptr<Rdb_key_def>* const kd_arr= tbl_def_arg->m_key_descr_arr; uint key_len= 0; m_pk_descr= kd_arr[pk_index(table_arg, tbl_def_arg)]; @@ -4651,7 +4888,7 @@ int ha_rocksdb::alloc_key_buffers(const TABLE* table_arg, DBUG_RETURN(HA_ERR_INTERNAL_ERROR); } - uint pack_key_len= m_pk_descr->max_storage_fmt_length(); + const uint pack_key_len= m_pk_descr->max_storage_fmt_length(); m_pk_packed_tuple= reinterpret_cast<uchar*>(my_malloc(pack_key_len, MYF(0))); if (m_pk_packed_tuple == nullptr) { @@ -4668,7 +4905,7 @@ int ha_rocksdb::alloc_key_buffers(const TABLE* table_arg, // TODO: move this into get_table_handler() ?? kd_arr[i]->setup(table_arg, tbl_def_arg); - uint packed_len= kd_arr[i]->max_storage_fmt_length(); + const uint packed_len= kd_arr[i]->max_storage_fmt_length(); if (packed_len > max_packed_sk_len) { max_packed_sk_len= packed_len; @@ -4716,7 +4953,7 @@ void ha_rocksdb::free_key_buffers() m_pack_buffer= nullptr; } -void ha_rocksdb::set_use_read_free_rpl(const char* whitelist) +void ha_rocksdb::set_use_read_free_rpl(const char* const whitelist) { DBUG_ASSERT(whitelist != nullptr); @@ -4734,7 +4971,7 @@ void ha_rocksdb::set_use_read_free_rpl(const char* whitelist) m_use_read_free_rpl = regex_handler.matches(m_tbl_def->base_tablename()); } -void ha_rocksdb::set_skip_unique_check_tables(const char* whitelist) +void ha_rocksdb::set_skip_unique_check_tables(const char* const whitelist) { DBUG_ASSERT(whitelist != nullptr); @@ -4752,7 +4989,7 @@ void ha_rocksdb::set_skip_unique_check_tables(const char* whitelist) m_skip_unique_check= regex_handler.matches(m_tbl_def->base_tablename()); } -int ha_rocksdb::open(const char *name, int mode, uint test_if_locked) +int ha_rocksdb::open(const char* const name, int mode, uint test_if_locked) { DBUG_ENTER("ha_rocksdb::open"); @@ -4806,6 +5043,18 @@ int ha_rocksdb::open(const char *name, int mode, uint test_if_locked) DBUG_RETURN(err); } + /* + init_with_fields() is used to initialize table flags based on the field + definitions in table->field[]. + It is called by open_binary_frm(), but that function calls the method for + a temporary ha_rocksdb object which is later destroyed. + + If we are here in ::open(), then init_with_fields() has not been called + for this object. Call it ourselves, we want all member variables to be + properly initialized. + */ + init_with_fields(); + setup_field_converters(); info(HA_STATUS_NO_LOCK | HA_STATUS_VARIABLE | HA_STATUS_CONST); @@ -4818,7 +5067,7 @@ int ha_rocksdb::open(const char *name, int mode, uint test_if_locked) Initialize the necessary variables for them: */ - m_verify_checksums= false; + m_verify_row_debug_checksums= false; /* TODO: move the following to where TABLE_SHARE is opened: */ if (table->found_next_number_field) @@ -4858,7 +5107,7 @@ int ha_rocksdb::close(void) DBUG_RETURN(0); } -bool ha_rocksdb::get_error_message(const int error, String *buf) +bool ha_rocksdb::get_error_message(const int error, String* const buf) { static_assert(HA_ERR_ROCKSDB_LAST > HA_ERR_FIRST, "HA_ERR_ROCKSDB_LAST > HA_ERR_FIRST"); @@ -4869,7 +5118,7 @@ bool ha_rocksdb::get_error_message(const int error, String *buf) DBUG_ASSERT(error <= HA_ERR_ROCKSDB_LAST); DBUG_ASSERT(buf != nullptr); - Rdb_transaction* tx = get_tx_from_thd(ha_thd()); + Rdb_transaction* const tx = get_tx_from_thd(ha_thd()); bool temp_error = false; switch (error) { @@ -4901,9 +5150,9 @@ static const std::set<const my_core::CHARSET_INFO *> RDB_INDEX_COLLATIONS= {&my_charset_bin, &my_charset_utf8_bin, &my_charset_latin1_bin}; -static bool rdb_is_index_collation_supported(const my_core::Field *field) +static bool rdb_is_index_collation_supported(const my_core::Field* const field) { - my_core::enum_field_types type= field->real_type(); + const my_core::enum_field_types type= field->real_type(); /* Handle [VAR](CHAR|BINARY) or TEXT|BLOB */ if (type == MYSQL_TYPE_VARCHAR || type == MYSQL_TYPE_STRING || type == MYSQL_TYPE_BLOB) @@ -4930,11 +5179,11 @@ static bool rdb_is_index_collation_supported(const my_core::Field *field) 0 - Ok other - error, either given table ddl is not supported by rocksdb or OOM. */ -int ha_rocksdb::create_key_defs(const TABLE *table_arg, - Rdb_tbl_def *tbl_def_arg, - const TABLE *old_table_arg /* = nullptr */, - const Rdb_tbl_def *old_tbl_def_arg - /* = nullptr */) +int ha_rocksdb::create_key_defs(const TABLE* const table_arg, + Rdb_tbl_def* const tbl_def_arg, + const TABLE* const old_table_arg /* = nullptr */, + const Rdb_tbl_def* const old_tbl_def_arg + /* = nullptr */) const { DBUG_ASSERT(table_arg != nullptr); DBUG_ASSERT(table_arg->s != nullptr); @@ -5011,8 +5260,9 @@ int ha_rocksdb::create_key_defs(const TABLE *table_arg, 0 - Ok other - error */ -int ha_rocksdb::create_cfs(const TABLE *table_arg, Rdb_tbl_def *tbl_def_arg, - std::array<struct key_def_cf_info, MAX_INDEXES + 1>* cfs) +int ha_rocksdb::create_cfs(const TABLE* const table_arg, + Rdb_tbl_def* const tbl_def_arg, + std::array<struct key_def_cf_info, MAX_INDEXES + 1>* const cfs) const { DBUG_ASSERT(table_arg != nullptr); DBUG_ASSERT(table_arg->s != nullptr); @@ -5043,7 +5293,7 @@ int ha_rocksdb::create_cfs(const TABLE *table_arg, Rdb_tbl_def *tbl_def_arg, !rdb_collation_exceptions->matches(tablename_sys)) { std::string collation_err; - for (auto coll : RDB_INDEX_COLLATIONS) + for (const auto &coll : RDB_INDEX_COLLATIONS) { if (collation_err != "") { @@ -5066,8 +5316,8 @@ int ha_rocksdb::create_cfs(const TABLE *table_arg, Rdb_tbl_def *tbl_def_arg, index comment has Column Family name. If there was no comment, we get NULL, and it means use the default column family. */ - const char *comment = get_key_comment(i, table_arg, tbl_def_arg); - const char *key_name = get_key_name(i, table_arg, tbl_def_arg); + const char* const comment = get_key_comment(i, table_arg, tbl_def_arg); + const char* const key_name = get_key_name(i, table_arg, tbl_def_arg); if (looks_like_per_index_cf_typo(comment)) { @@ -5112,11 +5362,11 @@ int ha_rocksdb::create_cfs(const TABLE *table_arg, Rdb_tbl_def *tbl_def_arg, 0 - Ok other - error, either given table ddl is not supported by rocksdb or OOM. */ -int ha_rocksdb::create_inplace_key_defs(const TABLE *table_arg, - Rdb_tbl_def *tbl_def_arg, - const TABLE *old_table_arg, - const Rdb_tbl_def *old_tbl_def_arg, - const std::array<key_def_cf_info, MAX_INDEXES + 1>& cfs) +int ha_rocksdb::create_inplace_key_defs(const TABLE* const table_arg, + Rdb_tbl_def* const tbl_def_arg, + const TABLE* const old_table_arg, + const Rdb_tbl_def* const old_tbl_def_arg, + const std::array<key_def_cf_info, MAX_INDEXES + 1>& cfs) const { DBUG_ASSERT(table_arg != nullptr); DBUG_ASSERT(tbl_def_arg != nullptr); @@ -5124,31 +5374,30 @@ int ha_rocksdb::create_inplace_key_defs(const TABLE *table_arg, DBUG_ENTER("create_key_def"); - std::shared_ptr<Rdb_key_def>* old_key_descr= + std::shared_ptr<Rdb_key_def>* const old_key_descr= old_tbl_def_arg->m_key_descr_arr; - std::shared_ptr<Rdb_key_def>* new_key_descr= + std::shared_ptr<Rdb_key_def>* const new_key_descr= tbl_def_arg->m_key_descr_arr; - std::unordered_map<std::string, uint> old_key_pos = + const std::unordered_map<std::string, uint> old_key_pos = get_old_key_positions(table_arg, tbl_def_arg, old_table_arg, old_tbl_def_arg); uint i; for (i= 0; i < tbl_def_arg->m_key_count; i++) { - auto it = old_key_pos.find(get_key_name(i, table_arg, tbl_def_arg)); + const auto &it = old_key_pos.find(get_key_name(i, table_arg, tbl_def_arg)); if (it != old_key_pos.end()) { /* Found matching index in old table definition, so copy it over to the new one created. */ - const std::shared_ptr<Rdb_key_def>& okd= - old_key_descr[it->second]; + const Rdb_key_def& okd= *old_key_descr[it->second]; uint16 index_dict_version= 0; uchar index_type= 0; uint16 kv_version= 0; - GL_INDEX_ID gl_index_id= okd->get_gl_index_id(); + const GL_INDEX_ID gl_index_id= okd.get_gl_index_id(); if (!dict_manager.get_index_info(gl_index_id, &index_dict_version, &index_type, &kv_version)) { @@ -5166,15 +5415,15 @@ int ha_rocksdb::create_inplace_key_defs(const TABLE *table_arg, itself. */ new_key_descr[i]= std::make_shared<Rdb_key_def>( - okd->get_index_number(), + okd.get_index_number(), i, - okd->get_cf(), + okd.get_cf(), index_dict_version, index_type, kv_version, - okd->m_is_reverse_cf, - okd->m_is_auto_cf, - okd->m_name.c_str(), + okd.m_is_reverse_cf, + okd.m_is_auto_cf, + okd.m_name.c_str(), dict_manager.get_stats(gl_index_id)); } else if (create_key_def(table_arg, i, tbl_def_arg, @@ -5191,10 +5440,10 @@ int ha_rocksdb::create_inplace_key_defs(const TABLE *table_arg, } std::unordered_map<std::string, uint> ha_rocksdb::get_old_key_positions( - const TABLE* table_arg, - const Rdb_tbl_def* tbl_def_arg, - const TABLE* old_table_arg, - const Rdb_tbl_def* old_tbl_def_arg) + const TABLE* const table_arg, + const Rdb_tbl_def* const tbl_def_arg, + const TABLE* const old_table_arg, + const Rdb_tbl_def* const old_tbl_def_arg) const { DBUG_ASSERT(table_arg != nullptr); DBUG_ASSERT(old_table_arg != nullptr); @@ -5203,7 +5452,7 @@ std::unordered_map<std::string, uint> ha_rocksdb::get_old_key_positions( DBUG_ENTER("get_old_key_positions"); - std::shared_ptr<Rdb_key_def>* old_key_descr= + std::shared_ptr<Rdb_key_def>* const old_key_descr= old_tbl_def_arg->m_key_descr_arr; std::unordered_map<std::string, uint> old_key_pos; std::unordered_map<std::string, uint> new_key_pos; @@ -5231,14 +5480,14 @@ std::unordered_map<std::string, uint> ha_rocksdb::get_old_key_positions( CREATE TABLE t1 (a INT, b INT, KEY ka(a)) ENGINE=RocksDB; ALTER TABLE t1 DROP INDEX ka, ADD INDEX ka(b), ALGORITHM=INPLACE; */ - const KEY* old_key = &old_table_arg->key_info[i]; - auto it = new_key_pos.find(old_key->name); + const KEY* const old_key = &old_table_arg->key_info[i]; + const auto &it = new_key_pos.find(old_key->name); if (it == new_key_pos.end()) { continue; } - KEY* new_key = &table_arg->key_info[it->second]; + KEY* const new_key = &table_arg->key_info[it->second]; if (!compare_key_parts(old_key, new_key)) { @@ -5250,7 +5499,8 @@ std::unordered_map<std::string, uint> ha_rocksdb::get_old_key_positions( } /* Check two keys to ensure that key parts within keys match */ -int ha_rocksdb::compare_key_parts(const KEY* old_key, const KEY* new_key) +int ha_rocksdb::compare_key_parts(const KEY* const old_key, + const KEY* const new_key) const { DBUG_ASSERT(old_key != nullptr); DBUG_ASSERT(new_key != nullptr); @@ -5293,17 +5543,17 @@ int ha_rocksdb::compare_key_parts(const KEY* old_key, const KEY* new_key) 0 - Ok other - error, either given table ddl is not supported by rocksdb or OOM. */ -int ha_rocksdb::create_key_def(const TABLE *table_arg, uint i, - const Rdb_tbl_def* tbl_def_arg, - std::shared_ptr<Rdb_key_def>* new_key_def, - const struct key_def_cf_info& cf_info) +int ha_rocksdb::create_key_def(const TABLE* const table_arg, const uint &i, + const Rdb_tbl_def* const tbl_def_arg, + std::shared_ptr<Rdb_key_def>* const new_key_def, + const struct key_def_cf_info& cf_info) const { DBUG_ENTER("create_key_def"); DBUG_ASSERT(new_key_def != nullptr); DBUG_ASSERT(*new_key_def == nullptr); - uint index_id= ddl_manager.get_and_update_next_number(&dict_manager); - uint16_t index_dict_version= Rdb_key_def::INDEX_INFO_VERSION_LATEST; + const uint index_id= ddl_manager.get_and_update_next_number(&dict_manager); + const uint16_t index_dict_version= Rdb_key_def::INDEX_INFO_VERSION_LATEST; uchar index_type; uint16_t kv_version; @@ -5316,24 +5566,16 @@ int ha_rocksdb::create_key_def(const TABLE *table_arg, uint i, { index_type= Rdb_key_def::INDEX_TYPE_PRIMARY; uint16 pk_latest_version= Rdb_key_def::PRIMARY_FORMAT_VERSION_LATEST; - DBUG_EXECUTE_IF("MYROCKS_FORMAT_VERSION_INITIAL", - {pk_latest_version= - Rdb_key_def::PRIMARY_FORMAT_VERSION_INITIAL; - }); kv_version= pk_latest_version; } else { index_type= Rdb_key_def::INDEX_TYPE_SECONDARY; uint16 sk_latest_version= Rdb_key_def::SECONDARY_FORMAT_VERSION_LATEST; - DBUG_EXECUTE_IF("MYROCKS_FORMAT_VERSION_INITIAL", - {sk_latest_version= - Rdb_key_def::SECONDARY_FORMAT_VERSION_INITIAL; - }); kv_version= sk_latest_version; } - const char *key_name = get_key_name(i, table_arg, m_tbl_def); + const char* const key_name = get_key_name(i, table_arg, m_tbl_def); *new_key_def= std::make_shared<Rdb_key_def>( index_id, i, cf_info.cf_handle, index_dict_version, index_type, kv_version, cf_info.is_reverse_cf, cf_info.is_auto_cf, key_name); @@ -5342,7 +5584,7 @@ int ha_rocksdb::create_key_def(const TABLE *table_arg, uint i, } int rdb_normalize_tablename(const std::string& tablename, - std::string* strbuf) + std::string* const strbuf) { DBUG_ASSERT(strbuf != nullptr); @@ -5368,7 +5610,7 @@ int rdb_normalize_tablename(const std::string& tablename, Check to see if the user's original statement includes foreign key references */ -bool ha_rocksdb::contains_foreign_key(THD* thd) +bool ha_rocksdb::contains_foreign_key(THD* const thd) { bool success; const char* str = thd_query_string(thd)->str; @@ -5431,9 +5673,9 @@ bool ha_rocksdb::contains_foreign_key(THD* thd) @return 0 on success, non-zero on failure to split */ int rdb_split_normalized_tablename(const std::string& fullname, - std::string* db, - std::string* table, - std::string* partition) + std::string* const db, + std::string* const table, + std::string* const partition) { DBUG_ASSERT(!fullname.empty()); @@ -5458,7 +5700,7 @@ int rdb_split_normalized_tablename(const std::string& fullname, dotpos++; - size_t partpos = fullname.find(RDB_PARTITION_STR, dotpos, + const size_t partpos = fullname.find(RDB_PARTITION_STR, dotpos, strlen(RDB_PARTITION_STR)); if (partpos != std::string::npos) @@ -5503,14 +5745,14 @@ int rdb_split_normalized_tablename(const std::string& fullname, ha_create_table() in handle.cc */ -int ha_rocksdb::create(const char *name, TABLE *table_arg, - HA_CREATE_INFO *create_info) +int ha_rocksdb::create(const char* const name, TABLE* const table_arg, + HA_CREATE_INFO* const create_info) { int res; std::string str; DBUG_ENTER("ha_rocksdb::create"); - THD *thd = my_core::thd_get_current_thd(); + THD* const thd = my_core::thd_get_current_thd(); if (get_table_if_exists(name)) { if (thd->lex->sql_command == SQLCOM_TRUNCATE) { @@ -5564,8 +5806,8 @@ int ha_rocksdb::create(const char *name, TABLE *table_arg, } bool write_err= false; - std::unique_ptr<rocksdb::WriteBatch> wb= dict_manager.begin(); - rocksdb::WriteBatch *batch= wb.get(); + const std::unique_ptr<rocksdb::WriteBatch> wb= dict_manager.begin(); + rocksdb::WriteBatch* const batch= wb.get(); /* Create table/key descriptions and put them into the data dictionary */ m_tbl_def= new Rdb_tbl_def(str); @@ -5642,7 +5884,7 @@ error: bool ha_rocksdb::check_keyread_allowed(uint inx, uint part, bool all_parts) const { bool res= true; - KEY *key_info= &table_share->key_info[inx]; + KEY* const key_info= &table_share->key_info[inx]; Rdb_field_packing dummy1; res= dummy1.setup(nullptr, key_info->key_part[part].field, inx, part, @@ -5666,7 +5908,7 @@ bool ha_rocksdb::check_keyread_allowed(uint inx, uint part, bool all_parts) cons } } - uint pk= table_share->primary_key; + const uint pk= table_share->primary_key; if (inx == pk && all_parts && part + 1 == table_share->key_info[pk].user_defined_key_parts) { @@ -5676,12 +5918,11 @@ bool ha_rocksdb::check_keyread_allowed(uint inx, uint part, bool all_parts) cons return res; } -int ha_rocksdb::read_key_exact(const std::shared_ptr<const Rdb_key_def>& kd, - rocksdb::Iterator *iter, - bool full_key_match, +int ha_rocksdb::read_key_exact(const Rdb_key_def& kd, + rocksdb::Iterator* const iter, + const bool &full_key_match, const rocksdb::Slice& key_slice) const { - DBUG_ASSERT(kd != nullptr); DBUG_ASSERT(iter != nullptr); /* @@ -5689,7 +5930,7 @@ int ha_rocksdb::read_key_exact(const std::shared_ptr<const Rdb_key_def>& kd, index_tuple= lookup_tuple. lookup_tuple may be a prefix of the index. */ - if (kd->m_is_reverse_cf) + if (kd.m_is_reverse_cf) { if (!full_key_match) { @@ -5700,7 +5941,7 @@ int ha_rocksdb::read_key_exact(const std::shared_ptr<const Rdb_key_def>& kd, } } - if (!iter->Valid() || !kd->value_matches_prefix(iter->key(), key_slice)) + if (!iter->Valid() || !kd.value_matches_prefix(iter->key(), key_slice)) { /* Got a record that is not equal to the lookup value, or even a record @@ -5711,20 +5952,18 @@ int ha_rocksdb::read_key_exact(const std::shared_ptr<const Rdb_key_def>& kd, return 0; } -int ha_rocksdb::read_before_key(const std::shared_ptr<const Rdb_key_def>& kd, - bool full_key_match, +int ha_rocksdb::read_before_key(const Rdb_key_def& kd, + const bool &full_key_match, const rocksdb::Slice& key_slice) { - DBUG_ASSERT(kd != nullptr); - /* We are looking for record with the biggest t.key such that t.key < lookup_tuple. */ - if (kd->m_is_reverse_cf) + if (kd.m_is_reverse_cf) { if (m_scan_it->Valid() && full_key_match && - kd->value_matches_prefix(m_scan_it->key(), key_slice)) + kd.value_matches_prefix(m_scan_it->key(), key_slice)) { /* We are using full key and we've hit an exact match */ m_scan_it->Next(); @@ -5741,12 +5980,10 @@ int ha_rocksdb::read_before_key(const std::shared_ptr<const Rdb_key_def>& kd, return m_scan_it->Valid() ? 0 : HA_ERR_KEY_NOT_FOUND; } -int ha_rocksdb::read_after_key(const std::shared_ptr<const Rdb_key_def>& kd, - bool full_key_match, +int ha_rocksdb::read_after_key(const Rdb_key_def& kd, + const bool &full_key_match, const rocksdb::Slice& key_slice) { - DBUG_ASSERT(kd != nullptr); - /* We are looking for the first record such that @@ -5755,7 +5992,7 @@ int ha_rocksdb::read_after_key(const std::shared_ptr<const Rdb_key_def>& kd, with HA_READ_AFTER_KEY, $GT = '>', with HA_READ_KEY_OR_NEXT, $GT = '>=' */ - if (kd->m_is_reverse_cf) + if (kd.m_is_reverse_cf) { if (!m_scan_it->Valid()) { @@ -5769,7 +6006,7 @@ int ha_rocksdb::read_after_key(const std::shared_ptr<const Rdb_key_def>& kd, - when using full extended key and when we've got an exact match */ if (!full_key_match || - !kd->value_matches_prefix(m_scan_it->key(), key_slice)) + !kd.value_matches_prefix(m_scan_it->key(), key_slice)) { m_scan_it->Prev(); } @@ -5780,13 +6017,13 @@ int ha_rocksdb::read_after_key(const std::shared_ptr<const Rdb_key_def>& kd, } int ha_rocksdb::position_to_correct_key( - const std::shared_ptr<const Rdb_key_def>& kd, - enum ha_rkey_function find_flag, - bool full_key_match, - const uchar* key, - key_part_map keypart_map, + const Rdb_key_def& kd, + const enum ha_rkey_function &find_flag, + const bool &full_key_match, + const uchar* const key, + const key_part_map &keypart_map, const rocksdb::Slice& key_slice, - bool* move_forward) + bool* const move_forward) { int rc= 0; @@ -5799,7 +6036,7 @@ int ha_rocksdb::position_to_correct_key( case HA_READ_BEFORE_KEY: *move_forward= false; rc= read_before_key(kd, full_key_match, key_slice); - if (rc == 0 && !kd->covers_key(m_scan_it->key())) + if (rc == 0 && !kd.covers_key(m_scan_it->key())) { /* The record we've got is not from this index */ rc= HA_ERR_KEY_NOT_FOUND; @@ -5808,7 +6045,7 @@ int ha_rocksdb::position_to_correct_key( case HA_READ_AFTER_KEY: case HA_READ_KEY_OR_NEXT: rc= read_after_key(kd, full_key_match, key_slice); - if (rc == 0 && !kd->covers_key(m_scan_it->key())) + if (rc == 0 && !kd.covers_key(m_scan_it->key())) { /* The record we've got is not from this index */ rc= HA_ERR_KEY_NOT_FOUND; @@ -5835,21 +6072,21 @@ int ha_rocksdb::position_to_correct_key( if (rc == 0) { const rocksdb::Slice& rkey= m_scan_it->key(); - if (!kd->covers_key(rkey)) + if (!kd.covers_key(rkey)) { /* The record we've got is not from this index */ rc= HA_ERR_KEY_NOT_FOUND; } else if (find_flag == HA_READ_PREFIX_LAST) { - uint size = kd->pack_index_tuple(table, m_pack_buffer, + uint size = kd.pack_index_tuple(table, m_pack_buffer, m_sk_packed_tuple, key, keypart_map); rocksdb::Slice lookup_tuple( reinterpret_cast<char*>(m_sk_packed_tuple), size); // We need to compare the key we've got with the original search prefix. - if (!kd->value_matches_prefix(rkey, lookup_tuple)) + if (!kd.value_matches_prefix(rkey, lookup_tuple)) { rc= HA_ERR_KEY_NOT_FOUND; } @@ -5864,12 +6101,12 @@ int ha_rocksdb::position_to_correct_key( return rc; } -int ha_rocksdb::calc_eq_cond_len(const std::shared_ptr<const Rdb_key_def>& kd, - enum ha_rkey_function find_flag, +int ha_rocksdb::calc_eq_cond_len(const Rdb_key_def& kd, + const enum ha_rkey_function &find_flag, const rocksdb::Slice& slice, - int bytes_changed_by_succ, - const key_range *end_key, - uint* end_key_packed_size) + const int &bytes_changed_by_succ, + const key_range* const end_key, + uint* const end_key_packed_size) { if (find_flag == HA_READ_KEY_EXACT) return slice.size(); @@ -5877,7 +6114,7 @@ int ha_rocksdb::calc_eq_cond_len(const std::shared_ptr<const Rdb_key_def>& kd, if (find_flag == HA_READ_PREFIX_LAST) { /* - We have made the kd->successor(m_sk_packed_tuple) call above. + We have made the kd.successor(m_sk_packed_tuple) call above. The slice is at least Rdb_key_def::INDEX_NUMBER_SIZE bytes long. */ @@ -5886,7 +6123,7 @@ int ha_rocksdb::calc_eq_cond_len(const std::shared_ptr<const Rdb_key_def>& kd, if (end_key) { - *end_key_packed_size= kd->pack_index_tuple(table, m_pack_buffer, + *end_key_packed_size= kd.pack_index_tuple(table, m_pack_buffer, m_end_key_packed_tuple, end_key->key, end_key->keypart_map); @@ -5914,13 +6151,13 @@ int ha_rocksdb::calc_eq_cond_len(const std::shared_ptr<const Rdb_key_def>& kd, return Rdb_key_def::INDEX_NUMBER_SIZE; } -int ha_rocksdb::read_row_from_primary_key(uchar *buf) +int ha_rocksdb::read_row_from_primary_key(uchar* const buf) { DBUG_ASSERT(buf != nullptr); int rc; const rocksdb::Slice& rkey= m_scan_it->key(); - uint pk_size= rkey.size(); + const uint pk_size= rkey.size(); const char *pk_data= rkey.data(); memcpy(m_pk_packed_tuple, pk_data, pk_size); @@ -5941,11 +6178,10 @@ int ha_rocksdb::read_row_from_primary_key(uchar *buf) return rc; } -int ha_rocksdb::read_row_from_secondary_key(uchar *buf, - const std::shared_ptr<const Rdb_key_def>& kd, bool move_forward) +int ha_rocksdb::read_row_from_secondary_key(uchar* const buf, + const Rdb_key_def& kd, bool move_forward) { DBUG_ASSERT(buf != nullptr); - DBUG_ASSERT(kd != nullptr); int rc= 0; uint pk_size; @@ -5954,7 +6190,7 @@ int ha_rocksdb::read_row_from_secondary_key(uchar *buf, { /* Get the key columns and primary key value */ const rocksdb::Slice& rkey= m_scan_it->key(); - pk_size= kd->get_primary_key_tuple(table, m_pk_descr, &rkey, + pk_size= kd.get_primary_key_tuple(table, *m_pk_descr, &rkey, m_pk_packed_tuple); const rocksdb::Slice& value= m_scan_it->value(); if (pk_size == RDB_INVALID_KEY_LEN) @@ -5968,14 +6204,14 @@ int ha_rocksdb::read_row_from_secondary_key(uchar *buf, } else { - if (kd->m_is_reverse_cf) + if (kd.m_is_reverse_cf) move_forward= !move_forward; rc= find_icp_matching_index_rec(move_forward, buf); if (!rc) { const rocksdb::Slice& rkey= m_scan_it->key(); - pk_size= kd->get_primary_key_tuple(table, m_pk_descr, &rkey, + pk_size= kd.get_primary_key_tuple(table, *m_pk_descr, &rkey, m_pk_packed_tuple); if (pk_size == RDB_INVALID_KEY_LEN) { @@ -6006,20 +6242,21 @@ int ha_rocksdb::read_row_from_secondary_key(uchar *buf, available. In this case, we should just do a primary key lookup. */ int ha_rocksdb::try_keyonly_read_from_sk(uchar* buf, - const std::shared_ptr<const Rdb_key_def>& kd, + const Rdb_key_def& kd, const rocksdb::Slice& key, const rocksdb::Slice& value, const uint rowid_size) { - int res= kd->unpack_record(table, buf, &key, &value, m_verify_checksums); + const int res= + kd.unpack_record(table, buf, &key, &value, m_verify_row_debug_checksums); uint size= rowid_size; if (res == UNPACK_INFO_MISSING) { // If we can't unpack, then do a PK lookup. if (rowid_size == RDB_INVALID_KEY_LEN) { - size= kd->get_primary_key_tuple( - table, m_pk_descr, &key, m_pk_packed_tuple); + size= kd.get_primary_key_tuple( + table, *m_pk_descr, &key, m_pk_packed_tuple); } return get_row_by_rowid(buf, m_pk_packed_tuple, size); } @@ -6094,7 +6331,7 @@ ulong ha_rocksdb::index_flags(uint inx, uint part, bool all_parts) const m_scan_it points at the index key-value pair that we should read the (pk,row) pair for. */ -int ha_rocksdb::secondary_index_read(const int keyno, uchar *buf) +int ha_rocksdb::secondary_index_read(const int keyno, uchar* const buf) { DBUG_ASSERT(buf != nullptr); DBUG_ASSERT(table != nullptr); @@ -6114,8 +6351,8 @@ int ha_rocksdb::secondary_index_read(const int keyno, uchar *buf) int rc; //TODO: We could here check if we have ran out of range we're scanning - uint size= m_key_descr_arr[keyno]->get_primary_key_tuple( - table, m_pk_descr, &key, m_pk_packed_tuple); + const uint size= m_key_descr_arr[keyno]->get_primary_key_tuple( + table, *m_pk_descr, &key, m_pk_packed_tuple); if (size == RDB_INVALID_KEY_LEN) { return HA_ERR_INTERNAL_ERROR; @@ -6126,7 +6363,7 @@ int ha_rocksdb::secondary_index_read(const int keyno, uchar *buf) if (m_keyread_only && m_lock_rows == RDB_LOCK_NONE && !has_hidden_pk(table)) { - rc= try_keyonly_read_from_sk(buf, m_key_descr_arr[keyno], + rc= try_keyonly_read_from_sk(buf, *m_key_descr_arr[keyno], key, m_scan_it->value(), size); } else @@ -6154,8 +6391,8 @@ int ha_rocksdb::secondary_index_read(const int keyno, uchar *buf) ha_rocksdb::read_range_first passes end_key to ha_rocksdb::index_read_map_impl function. */ -int ha_rocksdb::read_range_first(const key_range *start_key, - const key_range *end_key, +int ha_rocksdb::read_range_first(const key_range* const start_key, + const key_range* const end_key, bool eq_range_arg, bool sorted) { int result; @@ -6207,7 +6444,7 @@ int ha_rocksdb::read_range_first(const key_range *start_key, } -int ha_rocksdb::index_read_map(uchar *buf, const uchar *key, +int ha_rocksdb::index_read_map(uchar* const buf, const uchar* const key, key_part_map keypart_map, enum ha_rkey_function find_flag) { @@ -6226,17 +6463,17 @@ int ha_rocksdb::index_read_map(uchar *buf, const uchar *key, On range scan, equal condition length is MIN(start_key, end_key) of the rocksdb::Slice expression. */ -int ha_rocksdb::index_read_map_impl(uchar *buf, const uchar *key, +int ha_rocksdb::index_read_map_impl(uchar* const buf, const uchar* const key, key_part_map keypart_map, enum ha_rkey_function find_flag, - const key_range *end_key) + const key_range* end_key) { int rc= 0; DBUG_ENTER("ha_rocksdb::index_read_map"); ha_statistic_increment(&SSV::ha_read_key_count); - const std::shared_ptr<const Rdb_key_def>& kd= m_key_descr_arr[active_index]; - uint actual_key_parts= kd->get_key_parts(); + const Rdb_key_def& kd= *m_key_descr_arr[active_index]; + const uint actual_key_parts= kd.get_key_parts(); bool using_full_key= is_using_full_key(keypart_map, actual_key_parts); if (!end_key) @@ -6253,8 +6490,9 @@ int ha_rocksdb::index_read_map_impl(uchar *buf, const uchar *key, Equality lookup over primary key, using full tuple. This is a special case, use DB::Get. */ - uint size= kd->pack_index_tuple(table, m_pack_buffer, m_pk_packed_tuple, - key, keypart_map); + const uint size= kd.pack_index_tuple( + table, m_pack_buffer, m_pk_packed_tuple, + key, keypart_map); rc= get_row_by_rowid(buf, m_pk_packed_tuple, size); if (!rc) { @@ -6275,15 +6513,15 @@ int ha_rocksdb::index_read_map_impl(uchar *buf, const uchar *key, { key_part_map tmp_map= (key_part_map(1) << table->key_info[active_index].user_defined_key_parts) - 1; - packed_size= kd->pack_index_tuple(table, m_pack_buffer, m_sk_packed_tuple, + packed_size= kd.pack_index_tuple(table, m_pack_buffer, m_sk_packed_tuple, key, tmp_map); if (table->key_info[active_index].user_defined_key_parts != - kd->get_key_parts()) + kd.get_key_parts()) using_full_key= false; } else { - packed_size= kd->pack_index_tuple(table, m_pack_buffer, m_sk_packed_tuple, + packed_size= kd.pack_index_tuple(table, m_pack_buffer, m_sk_packed_tuple, key, keypart_map); } @@ -6313,24 +6551,24 @@ int ha_rocksdb::index_read_map_impl(uchar *buf, const uchar *key, find_flag == HA_READ_AFTER_KEY) { /* See below */ - bytes_changed_by_succ= kd->successor(m_sk_packed_tuple, packed_size); + bytes_changed_by_succ= kd.successor(m_sk_packed_tuple, packed_size); } rocksdb::Slice slice( reinterpret_cast<const char*>(m_sk_packed_tuple), packed_size); uint end_key_packed_size= 0; - uint eq_cond_len= calc_eq_cond_len(kd, find_flag, slice, - bytes_changed_by_succ, end_key, - &end_key_packed_size); + const uint eq_cond_len= calc_eq_cond_len(kd, find_flag, slice, + bytes_changed_by_succ, end_key, + &end_key_packed_size); bool use_all_keys= false; if (find_flag == HA_READ_KEY_EXACT && - my_count_bits(keypart_map) == kd->get_key_parts()) + my_count_bits(keypart_map) == kd.get_key_parts()) use_all_keys= true; - Rdb_transaction *tx= get_or_create_tx(table->in_use); - bool is_new_snapshot= !tx->has_snapshot(); + Rdb_transaction* const tx= get_or_create_tx(table->in_use); + const bool is_new_snapshot= !tx->has_snapshot(); // Loop as long as we get a deadlock error AND we end up creating the // snapshot here (i.e. it did not exist prior to this) for (;;) @@ -6413,14 +6651,14 @@ int ha_rocksdb::index_read_map_impl(uchar *buf, const uchar *key, other - error code */ -int ha_rocksdb::find_icp_matching_index_rec(bool move_forward, uchar *buf) +int ha_rocksdb::find_icp_matching_index_rec(const bool &move_forward, + uchar* const buf) { DBUG_ASSERT(buf != nullptr); if (pushed_idx_cond && pushed_idx_cond_keyno == active_index) { - const std::shared_ptr<const Rdb_key_def>& kd= m_key_descr_arr[active_index]; - DBUG_ASSERT(kd != nullptr); + const Rdb_key_def& kd= *m_key_descr_arr[active_index]; while (1) { @@ -6429,9 +6667,9 @@ int ha_rocksdb::find_icp_matching_index_rec(bool move_forward, uchar *buf) table->status= STATUS_NOT_FOUND; return HA_ERR_END_OF_FILE; } - rocksdb::Slice rkey= m_scan_it->key(); + const rocksdb::Slice rkey= m_scan_it->key(); - if (!kd->covers_key(rkey)) + if (!kd.covers_key(rkey)) { table->status= STATUS_NOT_FOUND; return HA_ERR_END_OF_FILE; @@ -6439,22 +6677,22 @@ int ha_rocksdb::find_icp_matching_index_rec(bool move_forward, uchar *buf) if (m_sk_match_prefix) { - rocksdb::Slice prefix((const char*)m_sk_match_prefix, + const rocksdb::Slice prefix((const char*)m_sk_match_prefix, m_sk_match_length); - if (!kd->value_matches_prefix(rkey, prefix)) + if (!kd.value_matches_prefix(rkey, prefix)) { table->status= STATUS_NOT_FOUND; return HA_ERR_END_OF_FILE; } } - rocksdb::Slice value= m_scan_it->value(); + const rocksdb::Slice value= m_scan_it->value(); if (try_keyonly_read_from_sk(buf, kd, rkey, value, RDB_INVALID_KEY_LEN)) { return HA_ERR_INTERNAL_ERROR; } - enum icp_result icp_status= check_index_cond(); + const enum icp_result icp_status= check_index_cond(); if (icp_status == ICP_NO_MATCH) { if (move_forward) @@ -6488,26 +6726,26 @@ int ha_rocksdb::find_icp_matching_index_rec(bool move_forward, uchar *buf) } -int ha_rocksdb::index_read_last_map(uchar * buf, const uchar * key, +int ha_rocksdb::index_read_last_map(uchar* const buf, const uchar* const key, key_part_map keypart_map) { return index_read_map(buf, key, keypart_map, HA_READ_PREFIX_LAST); } -int ha_rocksdb::check(THD* thd, HA_CHECK_OPT* check_opt) +int ha_rocksdb::check(THD* const thd, HA_CHECK_OPT* const check_opt) { - uint pk= pk_index(table, m_tbl_def); + const uint pk= pk_index(table, m_tbl_def); String rowkey_copy; String sec_key_copy; - const char* table_name = table->s->table_name.str; + const char* const table_name = table->s->table_name.str; // Only when debugging: don't use snapshot when reading // Rdb_transaction *tx= get_or_create_tx(table->in_use); // tx->snapshot= nullptr; - bool save_verify_checksums= m_verify_checksums; - m_verify_checksums= true; + const bool save_verify_row_debug_checksums= m_verify_row_debug_checksums; + m_verify_row_debug_checksums= true; /* For each secondary index, check that we can get a PK value from it */ // NO_LINT_DEBUG @@ -6606,18 +6844,20 @@ int ha_rocksdb::check(THD* thd, HA_CHECK_OPT* check_opt) print_and_error: { std::string buf; - buf = rdb_hexdump(rowkey_copy.ptr(), rowkey_copy.length(), 1000); + buf = rdb_hexdump(rowkey_copy.ptr(), rowkey_copy.length(), + RDB_MAX_HEXDUMP_LEN); // NO_LINT_DEBUG sql_print_error("CHECKTABLE %s: rowkey: %s", table_name, buf.c_str()); buf= rdb_hexdump(m_retrieved_record.data(), m_retrieved_record.size(), - 1000); + RDB_MAX_HEXDUMP_LEN); // NO_LINT_DEBUG sql_print_error("CHECKTABLE %s: record: %s", table_name, buf.c_str()); - buf = rdb_hexdump(sec_key_copy.ptr(), sec_key_copy.length(), 1000); + buf = rdb_hexdump(sec_key_copy.ptr(), sec_key_copy.length(), + RDB_MAX_HEXDUMP_LEN); // NO_LINT_DEBUG sql_print_error("CHECKTABLE %s: index: %s", table_name, buf.c_str()); @@ -6646,7 +6886,7 @@ print_and_error: } extra(HA_EXTRA_NO_KEYREAD); - m_verify_checksums= save_verify_checksums; + m_verify_row_debug_checksums= save_verify_row_debug_checksums; /* TODO: we should check also for PK records that are missing in the secondary indexes. @@ -6655,14 +6895,14 @@ print_and_error: */ return HA_ADMIN_OK; error: - m_verify_checksums= save_verify_checksums; + m_verify_row_debug_checksums= save_verify_row_debug_checksums; ha_index_or_rnd_end(); extra(HA_EXTRA_NO_KEYREAD); return HA_ADMIN_CORRUPT; } -static void dbug_dump_str(FILE *out, const char *str, int len) +static void dbug_dump_str(FILE* const out, const char* const str, int len) { fprintf(out, "\""); for (int i= 0; i < len; i++) @@ -6682,9 +6922,9 @@ static void dbug_dump_str(FILE *out, const char *str, int len) dbug_dump_database(rdb); */ -void dbug_dump_database(rocksdb::DB *db) +void dbug_dump_database(rocksdb::DB* const db) { - FILE *out= fopen("/tmp/rocksdb.dump", "wt"); + FILE* const out= fopen("/tmp/rocksdb.dump", "wt"); if (!out) return; @@ -6702,6 +6942,30 @@ void dbug_dump_database(rocksdb::DB *db) fclose(out); } +rocksdb::Status ha_rocksdb::get_for_update( + Rdb_transaction* const tx, + rocksdb::ColumnFamilyHandle* const column_family, + const rocksdb::Slice& key, + std::string* const value) const +{ + DBUG_ASSERT(m_lock_rows != RDB_LOCK_NONE); + const bool exclusive = m_lock_rows != RDB_LOCK_READ; + + rocksdb::Status s= tx->get_for_update(column_family, key, value, exclusive); + + // If we have a lock conflict and we are running in READ COMMITTTED mode + // release and reacquire the snapshot and then retry the get_for_update(). + if (s.IsBusy() && !s.IsDeadlock() && + my_core::thd_tx_isolation(ha_thd()) == ISO_READ_COMMITTED) + { + tx->release_snapshot(); + tx->acquire_snapshot(false); + + s= tx->get_for_update(column_family, key, value, exclusive); + } + + return s; +} /* Given a rowid (i.e. packed PK) as a parameter, get the record. @@ -6711,7 +6975,7 @@ void dbug_dump_database(rocksdb::DB *db) other - HA_ERR error code. */ -int ha_rocksdb::get_row_by_rowid(uchar *buf, const char *rowid, +int ha_rocksdb::get_row_by_rowid(uchar* const buf, const char* const rowid, const uint rowid_size) { DBUG_ASSERT(buf != nullptr); @@ -6723,7 +6987,7 @@ int ha_rocksdb::get_row_by_rowid(uchar *buf, const char *rowid, rocksdb::Slice key_slice(rowid, rowid_size); - Rdb_transaction *tx= get_or_create_tx(table->in_use); + Rdb_transaction* const tx= get_or_create_tx(table->in_use); DBUG_ASSERT(tx != nullptr); DEBUG_SYNC(ha_thd(), "rocksdb.get_row_by_rowid"); @@ -6747,11 +7011,13 @@ int ha_rocksdb::get_row_by_rowid(uchar *buf, const char *rowid, s= tx->get(m_pk_descr->get_cf(), key_slice, &m_retrieved_record); } else - s= tx->get_for_update(m_pk_descr->get_cf(), key_slice, &m_retrieved_record); + { + s= get_for_update(tx, m_pk_descr->get_cf(), key_slice, &m_retrieved_record); + } if (!s.IsNotFound() && !s.ok()) { - DBUG_RETURN(tx->set_status_error(table->in_use, s, m_pk_descr, m_tbl_def)); + DBUG_RETURN(tx->set_status_error(table->in_use, s, *m_pk_descr, m_tbl_def)); } found= !s.IsNotFound(); @@ -6776,7 +7042,7 @@ int ha_rocksdb::get_row_by_rowid(uchar *buf, const char *rowid, } -int ha_rocksdb::index_next(uchar *buf) +int ha_rocksdb::index_next(uchar* const buf) { bool moves_forward= true; ha_statistic_increment(&SSV::ha_read_next_count); @@ -6792,7 +7058,7 @@ int ha_rocksdb::index_next(uchar *buf) } -int ha_rocksdb::index_prev(uchar *buf) +int ha_rocksdb::index_prev(uchar* const buf) { bool moves_forward= false; ha_statistic_increment(&SSV::ha_read_prev_count); @@ -6809,7 +7075,7 @@ int ha_rocksdb::index_prev(uchar *buf) } -int ha_rocksdb::index_next_with_direction(uchar *buf, bool move_forward) +int ha_rocksdb::index_next_with_direction(uchar* const buf, bool move_forward) { int rc; DBUG_ENTER("ha_rocksdb::index_next"); @@ -6840,7 +7106,7 @@ int ha_rocksdb::index_next_with_direction(uchar *buf, bool move_forward) } -int ha_rocksdb::index_first(uchar *buf) +int ha_rocksdb::index_first(uchar* const buf) { m_sk_match_prefix= nullptr; ha_statistic_increment(&SSV::ha_read_first_count); @@ -6852,7 +7118,7 @@ int ha_rocksdb::index_first(uchar *buf) } -int ha_rocksdb::index_last(uchar *buf) +int ha_rocksdb::index_last(uchar* const buf) { m_sk_match_prefix= nullptr; ha_statistic_increment(&SSV::ha_read_last_count); @@ -6895,7 +7161,7 @@ int ha_rocksdb::index_last(uchar *buf) */ -int ha_rocksdb::index_first_intern(uchar *buf) +int ha_rocksdb::index_first_intern(uchar* const buf) { DBUG_ASSERT(buf != nullptr); @@ -6915,27 +7181,27 @@ int ha_rocksdb::index_first_intern(uchar *buf) DBUG_ASSERT(key != nullptr); - const std::shared_ptr<const Rdb_key_def>& kd= m_key_descr_arr[active_index]; - if (kd->m_is_reverse_cf) + const Rdb_key_def& kd= *m_key_descr_arr[active_index]; + if (kd.m_is_reverse_cf) { - kd->get_supremum_key(key, &key_size); + kd.get_supremum_key(key, &key_size); } else { - kd->get_infimum_key(key, &key_size); + kd.get_infimum_key(key, &key_size); } rocksdb::Slice index_key((const char*)key, key_size); - Rdb_transaction *tx= get_or_create_tx(table->in_use); + Rdb_transaction* const tx= get_or_create_tx(table->in_use); DBUG_ASSERT(tx != nullptr); - bool is_new_snapshot= !tx->has_snapshot(); + const bool is_new_snapshot= !tx->has_snapshot(); // Loop as long as we get a deadlock error AND we end up creating the // snapshot here (i.e. it did not exist prior to this) for (;;) { - setup_scan_iterator(kd, &index_key, false, !kd->m_is_reverse_cf, + setup_scan_iterator(kd, &index_key, false, !kd.m_is_reverse_cf, Rdb_key_def::INDEX_NUMBER_SIZE); m_skip_scan_it_next_call= true; @@ -6998,7 +7264,7 @@ int ha_rocksdb::index_first_intern(uchar *buf) So, need to: it->Seek(n) || it->SeekToLast(); it->Prev(); */ -int ha_rocksdb::index_last_intern(uchar *buf) +int ha_rocksdb::index_last_intern(uchar* const buf) { DBUG_ASSERT(buf != nullptr); @@ -7018,19 +7284,19 @@ int ha_rocksdb::index_last_intern(uchar *buf) DBUG_ASSERT(key != nullptr); - const std::shared_ptr<const Rdb_key_def>& kd= m_key_descr_arr[active_index]; - if (kd->m_is_reverse_cf) + const Rdb_key_def& kd= *m_key_descr_arr[active_index]; + if (kd.m_is_reverse_cf) { - kd->get_infimum_key(key, &key_size); + kd.get_infimum_key(key, &key_size); } else { - kd->get_supremum_key(key, &key_size); + kd.get_supremum_key(key, &key_size); } rocksdb::Slice index_key((const char*)key, key_size); - Rdb_transaction *tx= get_or_create_tx(table->in_use); + Rdb_transaction* const tx= get_or_create_tx(table->in_use); DBUG_ASSERT(tx != nullptr); bool is_new_snapshot= !tx->has_snapshot(); @@ -7038,7 +7304,7 @@ int ha_rocksdb::index_last_intern(uchar *buf) // snapshot here (i.e. it did not exist prior to this) for (;;) { - setup_scan_iterator(kd, &index_key, false, kd->m_is_reverse_cf, + setup_scan_iterator(kd, &index_key, false, kd.m_is_reverse_cf, Rdb_key_def::INDEX_NUMBER_SIZE); if (!m_scan_it->Valid()) @@ -7097,7 +7363,7 @@ void ha_rocksdb::unlock_row() { if (m_lock_rows != RDB_LOCK_NONE) { - Rdb_transaction *tx= get_or_create_tx(table->in_use); + Rdb_transaction* const tx= get_or_create_tx(table->in_use); tx->release_lock(m_pk_descr->get_cf(), std::string(m_last_rowkey.ptr(), m_last_rowkey.length())); } @@ -7109,14 +7375,14 @@ void ha_rocksdb::unlock_row() - If the index is PRIMARY KEY, and if all of the columns of the table are covered by the PRIMARY KEY, SingleDelete can be used. */ -bool ha_rocksdb::can_use_single_delete(uint index) +bool ha_rocksdb::can_use_single_delete(const uint &index) const { return (index != pk_index(table, m_tbl_def) || (!has_hidden_pk(table) && table->key_info[index].actual_key_parts == table->s->fields)); } -bool ha_rocksdb::skip_unique_check() +bool ha_rocksdb::skip_unique_check() const { /* We want to skip unique checks if: @@ -7147,7 +7413,7 @@ bool ha_rocksdb::commit_in_the_middle() @retval true if bulk commit failed @retval false if bulk commit was skipped or succeeded */ -bool ha_rocksdb::do_bulk_commit(Rdb_transaction *tx) +bool ha_rocksdb::do_bulk_commit(Rdb_transaction* const tx) { DBUG_ASSERT(tx != nullptr); return commit_in_the_middle() && @@ -7161,7 +7427,7 @@ bool ha_rocksdb::do_bulk_commit(Rdb_transaction *tx) does not contain a primary key. (In which case we generate a hidden 'auto-incremented' pk.) */ -bool ha_rocksdb::has_hidden_pk(const TABLE* table) +bool ha_rocksdb::has_hidden_pk(const TABLE* const table) const { DBUG_ASSERT(table != nullptr); return Rdb_key_def::table_has_hidden_pk(table); @@ -7171,8 +7437,8 @@ bool ha_rocksdb::has_hidden_pk(const TABLE* table) Returns true if given index number is a hidden_pk. - This is used when a table is created with no primary key. */ -bool ha_rocksdb::is_hidden_pk(const uint index, const TABLE* table_arg, - const Rdb_tbl_def* tbl_def_arg) +bool ha_rocksdb::is_hidden_pk(const uint index, const TABLE* const table_arg, + const Rdb_tbl_def* const tbl_def_arg) { DBUG_ASSERT(table_arg != nullptr); DBUG_ASSERT(table_arg->s != nullptr); @@ -7183,8 +7449,8 @@ bool ha_rocksdb::is_hidden_pk(const uint index, const TABLE* table_arg, } /* Returns index of primary key */ -uint ha_rocksdb::pk_index(const TABLE* table_arg, - const Rdb_tbl_def* tbl_def_arg) +uint ha_rocksdb::pk_index(const TABLE* const table_arg, + const Rdb_tbl_def* const tbl_def_arg) { DBUG_ASSERT(table_arg != nullptr); DBUG_ASSERT(table_arg->s != nullptr); @@ -7196,8 +7462,8 @@ uint ha_rocksdb::pk_index(const TABLE* table_arg, } /* Returns true if given index number is a primary key */ -bool ha_rocksdb::is_pk(const uint index, const TABLE* table_arg, - const Rdb_tbl_def *tbl_def_arg) +bool ha_rocksdb::is_pk(const uint index, const TABLE* const table_arg, + const Rdb_tbl_def* const tbl_def_arg) { DBUG_ASSERT(table_arg != nullptr); DBUG_ASSERT(table_arg->s != nullptr); @@ -7207,8 +7473,9 @@ bool ha_rocksdb::is_pk(const uint index, const TABLE* table_arg, is_hidden_pk(index, table_arg, tbl_def_arg); } -const char* ha_rocksdb::get_key_name(const uint index, const TABLE* table_arg, - const Rdb_tbl_def* tbl_def_arg) +const char* ha_rocksdb::get_key_name(const uint index, + const TABLE* const table_arg, + const Rdb_tbl_def* const tbl_def_arg) { DBUG_ASSERT(table_arg != nullptr); @@ -7221,8 +7488,8 @@ const char* ha_rocksdb::get_key_name(const uint index, const TABLE* table_arg, } const char* ha_rocksdb::get_key_comment(const uint index, - const TABLE* table_arg, - const Rdb_tbl_def* tbl_def_arg) + const TABLE* const table_arg, + const Rdb_tbl_def* const tbl_def_arg) { DBUG_ASSERT(table_arg != nullptr); @@ -7234,7 +7501,7 @@ const char* ha_rocksdb::get_key_comment(const uint index, return table_arg->key_info[index].comment.str; } -int ha_rocksdb::write_row(uchar *buf) +int ha_rocksdb::write_row(uchar* const buf) { DBUG_ENTER("ha_rocksdb::write_row"); DBUG_ASSERT(buf == table->record[0]); @@ -7253,7 +7520,7 @@ int ha_rocksdb::write_row(uchar *buf) } } - int rv= update_write_row(nullptr, buf, skip_unique_check()); + const int rv= update_write_row(nullptr, buf, skip_unique_check()); if (rv == 0) { @@ -7273,18 +7540,18 @@ int ha_rocksdb::write_row(uchar *buf) called without setting m_last_rowkey. This function sets m_last_rowkey for Read Free Replication. */ -void ha_rocksdb::set_last_rowkey(const uchar *old_data) +void ha_rocksdb::set_last_rowkey(const uchar* const old_data) { if (old_data && use_read_free_rpl()) { - int old_pk_size= m_pk_descr->pack_record( + const int old_pk_size= m_pk_descr->pack_record( table, m_pack_buffer, old_data, m_pk_packed_tuple, nullptr, false); m_last_rowkey.copy((const char*)m_pk_packed_tuple, old_pk_size, &my_charset_bin); } } -int ha_rocksdb::get_pk_for_update(struct update_row_info* row_info) +int ha_rocksdb::get_pk_for_update(struct update_row_info* const row_info) { int size; @@ -7327,9 +7594,10 @@ int ha_rocksdb::get_pk_for_update(struct update_row_info* row_info) return 0; } -int ha_rocksdb::check_and_lock_unique_pk(uint key_id, +int ha_rocksdb::check_and_lock_unique_pk(const uint &key_id, const struct update_row_info& row_info, - bool* found, bool* pk_changed) + bool* const found, + bool* const pk_changed) { DBUG_ASSERT(found != nullptr); DBUG_ASSERT(pk_changed != nullptr); @@ -7374,22 +7642,21 @@ int ha_rocksdb::check_and_lock_unique_pk(uint key_id, 2) T1 Get(empty) -> T1 Put(insert, not committed yet) -> T2 Get(empty) -> T2 Put(insert, blocked) -> T1 commit -> T2 commit(overwrite) */ - rocksdb::Status s= row_info.tx->get_for_update(m_pk_descr->get_cf(), - row_info.new_pk_slice, - &m_retrieved_record); + const rocksdb::Status s= get_for_update(row_info.tx, m_pk_descr->get_cf(), + row_info.new_pk_slice, &m_retrieved_record); if (!s.ok() && !s.IsNotFound()) { return row_info.tx->set_status_error(table->in_use, s, - m_key_descr_arr[key_id], m_tbl_def); + *m_key_descr_arr[key_id], m_tbl_def); } *found= !s.IsNotFound(); return 0; } -int ha_rocksdb::check_and_lock_sk(uint key_id, - const struct update_row_info& row_info, - bool* found) const +int ha_rocksdb::check_and_lock_sk(const uint &key_id, + const struct update_row_info& row_info, + bool* const found) const { DBUG_ASSERT(found != nullptr); *found= false; @@ -7417,7 +7684,7 @@ int ha_rocksdb::check_and_lock_sk(uint key_id, return 0; } - const std::shared_ptr<const Rdb_key_def>& kd= m_key_descr_arr[key_id]; + const Rdb_key_def& kd= *m_key_descr_arr[key_id]; /* Calculate the new key for obtaining the lock @@ -7425,9 +7692,9 @@ int ha_rocksdb::check_and_lock_sk(uint key_id, For unique secondary indexes, the key used for locking does not include the extended fields. */ - int size= kd->pack_record(table, m_pack_buffer, row_info.new_data, - m_sk_packed_tuple, nullptr, false, 0, - user_defined_key_parts, &n_null_fields); + int size= kd.pack_record(table, m_pack_buffer, row_info.new_data, + m_sk_packed_tuple, nullptr, false, 0, + user_defined_key_parts, &n_null_fields); if (n_null_fields > 0) { /* @@ -7437,8 +7704,8 @@ int ha_rocksdb::check_and_lock_sk(uint key_id, return 0; } - rocksdb::Slice new_slice= rocksdb::Slice((const char*)m_sk_packed_tuple, - size); + const rocksdb::Slice new_slice= rocksdb::Slice((const char*)m_sk_packed_tuple, + size); /* For UPDATEs, if the key has changed, we need to obtain a lock. INSERTs @@ -7446,11 +7713,11 @@ int ha_rocksdb::check_and_lock_sk(uint key_id, */ if (row_info.old_data != nullptr) { - size= kd->pack_record(table, m_pack_buffer, row_info.old_data, + size= kd.pack_record(table, m_pack_buffer, row_info.old_data, m_sk_packed_tuple_old, nullptr, false, row_info.hidden_pk_id, user_defined_key_parts); - rocksdb::Slice old_slice= rocksdb::Slice( + const rocksdb::Slice old_slice= rocksdb::Slice( (const char*)m_sk_packed_tuple_old, size); /* @@ -7474,7 +7741,7 @@ int ha_rocksdb::check_and_lock_sk(uint key_id, the latest committed data. */ - bool all_parts_used= (user_defined_key_parts == kd->get_key_parts()); + const bool all_parts_used= (user_defined_key_parts == kd.get_key_parts()); /* This iterator seems expensive since we need to allocate and free @@ -7487,24 +7754,24 @@ int ha_rocksdb::check_and_lock_sk(uint key_id, The bloom filter may need to be disabled for this lookup. */ - bool total_order_seek= + const bool total_order_seek= !can_use_bloom_filter(ha_thd(), kd, new_slice, all_parts_used, - is_ascending(m_key_descr_arr[key_id], + is_ascending(*m_key_descr_arr[key_id], HA_READ_KEY_EXACT)); - bool fill_cache= !THDVAR(ha_thd(), skip_fill_cache); + const bool fill_cache= !THDVAR(ha_thd(), skip_fill_cache); /* psergey-todo: we just need to take lock, lookups not needed: */ std::string dummy_value; - rocksdb::Status s= row_info.tx->get_for_update(kd->get_cf(), new_slice, - &dummy_value); + const rocksdb::Status s= get_for_update(row_info.tx, kd.get_cf(), new_slice, + &dummy_value); if (!s.ok() && !s.IsNotFound()) { return row_info.tx->set_status_error(table->in_use, s, kd, m_tbl_def); } - rocksdb::Iterator *iter= row_info.tx->get_iterator(kd->get_cf(), + rocksdb::Iterator* const iter= row_info.tx->get_iterator(kd.get_cf(), total_order_seek, fill_cache, true /* read current data */, @@ -7523,7 +7790,7 @@ int ha_rocksdb::check_and_lock_sk(uint key_id, int ha_rocksdb::check_uniqueness_and_lock( const struct update_row_info& row_info, - bool* pk_changed) + bool* const pk_changed) { /* Go through each index and determine if the index has uniqueness @@ -7562,20 +7829,22 @@ int ha_rocksdb::check_uniqueness_and_lock( return 0; } -int ha_rocksdb::bulk_load_key(Rdb_transaction* tx, - const std::shared_ptr<const Rdb_key_def>& kd, +int ha_rocksdb::bulk_load_key(Rdb_transaction* const tx, + const Rdb_key_def& kd, const rocksdb::Slice& key, const rocksdb::Slice& value) { - rocksdb::ColumnFamilyHandle* cf= kd->get_cf(); + rocksdb::ColumnFamilyHandle* const cf= kd.get_cf(); DBUG_ASSERT(cf != nullptr); if (m_sst_info == nullptr) { m_sst_info= std::make_shared<Rdb_sst_info>(rdb, m_table_handler->m_table_name, - kd->get_name(), cf, rocksdb_db_options); + kd.get_name(), cf, rocksdb_db_options, + THDVAR(ha_thd(), trace_sst_api)); tx->start_bulk_load(this); + m_bulk_load_tx= tx; } DBUG_ASSERT(m_sst_info != nullptr); @@ -7596,6 +7865,19 @@ int ha_rocksdb::finalize_bulk_load() { int rc= 0; + /* Skip if there are no possible ongoing bulk loads */ + if (m_sst_info == nullptr && m_bulk_load_tx == nullptr) + { + return rc; + } + + mysql_mutex_lock(&m_bulk_load_mutex); + + /* + We need this check because it's possible that m_sst_info has been + flushed and cleared by another thread by the time the mutex has been + acquired. + */ if (m_sst_info != nullptr) { rc= m_sst_info->commit(); @@ -7609,29 +7891,27 @@ int ha_rocksdb::finalize_bulk_load() } m_sst_info= nullptr; - - Rdb_transaction *tx= get_or_create_tx(ha_thd()); - if (tx != nullptr) - { - tx->end_bulk_load(this); - } + m_bulk_load_tx->end_bulk_load(this); + m_bulk_load_tx= nullptr; } + mysql_mutex_unlock(&m_bulk_load_mutex); return rc; } -int ha_rocksdb::update_pk(const std::shared_ptr<const Rdb_key_def>& kd, +int ha_rocksdb::update_pk(const Rdb_key_def& kd, const struct update_row_info& row_info, - bool pk_changed) + const bool &pk_changed) { - uint key_id= kd->get_keyno(); - bool hidden_pk= is_hidden_pk(key_id, table, m_tbl_def); + const uint key_id= kd.get_keyno(); + const bool hidden_pk= is_hidden_pk(key_id, table, m_tbl_def); if (!hidden_pk && pk_changed) { /* The old key needs to be deleted. */ - rocksdb::Status s= delete_or_singledelete(key_id, row_info.tx, kd->get_cf(), + const rocksdb::Status s= delete_or_singledelete( + key_id, row_info.tx, kd.get_cf(), row_info.old_pk_slice); if (!s.ok()) { @@ -7650,7 +7930,7 @@ int ha_rocksdb::update_pk(const std::shared_ptr<const Rdb_key_def>& kd, &value_slice); int rc= 0; - auto cf= m_pk_descr->get_cf(); + const auto cf= m_pk_descr->get_cf(); if (rocksdb_enable_bulk_load_api && THDVAR(table->in_use, bulk_load) && !hidden_pk) { @@ -7680,7 +7960,7 @@ int ha_rocksdb::update_pk(const std::shared_ptr<const Rdb_key_def>& kd, } else { - auto s= row_info.tx->put(cf, row_info.new_pk_slice, value_slice); + const auto s= row_info.tx->put(cf, row_info.new_pk_slice, value_slice); if (!s.ok()) { if (s.IsBusy()) @@ -7691,7 +7971,7 @@ int ha_rocksdb::update_pk(const std::shared_ptr<const Rdb_key_def>& kd, } else { - rc = row_info.tx->set_status_error(table->in_use, s, m_pk_descr, + rc = row_info.tx->set_status_error(table->in_use, s, *m_pk_descr, m_tbl_def); } } @@ -7700,8 +7980,8 @@ int ha_rocksdb::update_pk(const std::shared_ptr<const Rdb_key_def>& kd, return rc; } -int ha_rocksdb::update_sk(const TABLE* table_arg, - const std::shared_ptr<const Rdb_key_def>& kd, +int ha_rocksdb::update_sk(const TABLE* const table_arg, + const Rdb_key_def& kd, const struct update_row_info& row_info) { int new_packed_size; @@ -7711,7 +7991,7 @@ int ha_rocksdb::update_sk(const TABLE* table_arg, rocksdb::Slice new_value_slice; rocksdb::Slice old_key_slice; - uint key_id = kd->get_keyno(); + const uint key_id = kd.get_keyno(); /* Can skip updating this key if none of the key fields have changed. */ @@ -7720,20 +8000,20 @@ int ha_rocksdb::update_sk(const TABLE* table_arg, return 0; } - bool store_checksums= should_store_checksums(); + const bool store_row_debug_checksums= should_store_row_debug_checksums(); - new_packed_size= kd->pack_record(table_arg, m_pack_buffer, row_info.new_data, + new_packed_size= kd.pack_record(table_arg, m_pack_buffer, row_info.new_data, m_sk_packed_tuple, &m_sk_tails, - store_checksums, + store_row_debug_checksums, row_info.hidden_pk_id); if (row_info.old_data != nullptr) { // The old value - old_packed_size= kd->pack_record(table_arg, m_pack_buffer, + old_packed_size= kd.pack_record(table_arg, m_pack_buffer, row_info.old_data, m_sk_packed_tuple_old, &m_sk_tails_old, - store_checksums, + store_row_debug_checksums, row_info.hidden_pk_id); /* @@ -7770,7 +8050,7 @@ int ha_rocksdb::update_sk(const TABLE* table_arg, reinterpret_cast<const char*>(m_sk_packed_tuple_old), old_packed_size); - row_info.tx->get_indexed_write_batch()->SingleDelete(kd->get_cf(), + row_info.tx->get_indexed_write_batch()->SingleDelete(kd.get_cf(), old_key_slice); } @@ -7800,19 +8080,19 @@ int ha_rocksdb::update_sk(const TABLE* table_arg, reinterpret_cast<const char*>(m_sk_tails.ptr()), m_sk_tails.get_current_pos()); - write_batch->Put(kd->get_cf(), new_key_slice, new_value_slice); + write_batch->Put(kd.get_cf(), new_key_slice, new_value_slice); return 0; } int ha_rocksdb::update_indexes(const struct update_row_info& row_info, - bool pk_changed) + const bool &pk_changed) { int rc; for (uint key_id= 0; key_id < m_tbl_def->m_key_count; key_id++) { - const std::shared_ptr<const Rdb_key_def>& kd= m_key_descr_arr[key_id]; + const Rdb_key_def& kd= *m_key_descr_arr[key_id]; if (is_pk(key_id, table, m_tbl_def)) { rc= update_pk(kd, row_info, pk_changed); @@ -7831,8 +8111,8 @@ int ha_rocksdb::update_indexes(const struct update_row_info& row_info, return 0; } -int ha_rocksdb::update_write_row(const uchar *old_data, - const uchar *new_data, +int ha_rocksdb::update_write_row(const uchar* const old_data, + const uchar* const new_data, const bool skip_unique_check) { bool pk_changed= false; @@ -7908,21 +8188,20 @@ int ha_rocksdb::update_write_row(const uchar *old_data, */ void ha_rocksdb::setup_scan_iterator( - const std::shared_ptr<const Rdb_key_def>& kd, - rocksdb::Slice *slice, + const Rdb_key_def& kd, + rocksdb::Slice* const slice, const bool use_all_keys, const bool is_ascending, const uint eq_cond_len) { - DBUG_ASSERT(kd != nullptr); DBUG_ASSERT(slice != nullptr); DBUG_ASSERT(slice->size() >= eq_cond_len); - Rdb_transaction *tx= get_or_create_tx(table->in_use); + Rdb_transaction* const tx= get_or_create_tx(table->in_use); bool skip_bloom= true; - rocksdb::Slice eq_cond(slice->data(), eq_cond_len); + const rocksdb::Slice eq_cond(slice->data(), eq_cond_len); if (can_use_bloom_filter(ha_thd(), kd, eq_cond, use_all_keys, is_ascending)) { skip_bloom= false; @@ -7951,8 +8230,21 @@ void ha_rocksdb::setup_scan_iterator( */ if (!m_scan_it) { - bool fill_cache= !THDVAR(ha_thd(), skip_fill_cache); - m_scan_it= tx->get_iterator(kd->get_cf(), skip_bloom, fill_cache); + const bool fill_cache= !THDVAR(ha_thd(), skip_fill_cache); + if (commit_in_the_middle()) + { + DBUG_ASSERT(m_scan_it_snapshot == nullptr); + m_scan_it_snapshot= rdb->GetSnapshot(); + + auto read_opts = rocksdb::ReadOptions(); + read_opts.total_order_seek = true; // TODO: set based on WHERE conditions + read_opts.snapshot= m_scan_it_snapshot; + m_scan_it= rdb->NewIterator(read_opts, kd.get_cf()); + } + else + { + m_scan_it= tx->get_iterator(kd.get_cf(), skip_bloom, fill_cache); + } m_scan_it_skips_bloom= skip_bloom; } /* @@ -7962,6 +8254,17 @@ void ha_rocksdb::setup_scan_iterator( m_scan_it->Seek(*slice); } +void ha_rocksdb::release_scan_iterator() +{ + delete m_scan_it; + m_scan_it= nullptr; + + if (m_scan_it_snapshot) + { + rdb->ReleaseSnapshot(m_scan_it_snapshot); + m_scan_it_snapshot= nullptr; + } +} void ha_rocksdb::setup_iterator_for_rnd_scan() { @@ -7974,7 +8277,7 @@ void ha_rocksdb::setup_iterator_for_rnd_scan() rocksdb::Slice table_key((const char*)m_pk_packed_tuple, key_size); - setup_scan_iterator(m_pk_descr, &table_key); + setup_scan_iterator(*m_pk_descr, &table_key); m_skip_scan_it_next_call= true; } @@ -7983,7 +8286,7 @@ int ha_rocksdb::rnd_init(bool scan) { DBUG_ENTER("ha_rocksdb::rnd_init"); - Rdb_transaction *tx= get_or_create_tx(table->in_use); + Rdb_transaction* const tx= get_or_create_tx(table->in_use); setup_read_decoders(); if (scan) @@ -8005,7 +8308,7 @@ int ha_rocksdb::rnd_init(bool scan) } -int ha_rocksdb::rnd_next(uchar *buf) +int ha_rocksdb::rnd_next(uchar* const buf) { int rc; ha_statistic_increment(&SSV::ha_read_rnd_next_count); @@ -8033,7 +8336,7 @@ int ha_rocksdb::rnd_next(uchar *buf) /* See also secondary_index_read(). */ -int ha_rocksdb::rnd_next_with_direction(uchar *buf, bool move_forward) +int ha_rocksdb::rnd_next_with_direction(uchar* const buf, bool move_forward) { int rc; DBUG_ENTER("ha_rocksdb::rnd_next"); @@ -8054,65 +8357,79 @@ int ha_rocksdb::rnd_next_with_direction(uchar *buf, bool move_forward) DBUG_RETURN(HA_ERR_END_OF_FILE); } - if (m_skip_scan_it_next_call) + for ( ; ; ) { - m_skip_scan_it_next_call= false; - } - else - { - if (move_forward) - m_scan_it->Next(); /* this call cannot fail */ + if (m_skip_scan_it_next_call) + { + m_skip_scan_it_next_call= false; + } else - m_scan_it->Prev(); /* this call cannot fail */ - } + { + if (move_forward) + m_scan_it->Next(); /* this call cannot fail */ + else + m_scan_it->Prev(); /* this call cannot fail */ + } + + if (!m_scan_it->Valid()) + { + rc= HA_ERR_END_OF_FILE; + break; + } - if (m_scan_it->Valid()) - { /* check if we're out of this table */ - rocksdb::Slice key= m_scan_it->key(); + const rocksdb::Slice key= m_scan_it->key(); if (!m_pk_descr->covers_key(key)) { rc= HA_ERR_END_OF_FILE; + break; } - else + + if (m_lock_rows != RDB_LOCK_NONE) { - if (m_lock_rows != RDB_LOCK_NONE) - { - /* - Lock the row we've just read. + /* + Lock the row we've just read. - Now we call get_for_update which will 1) Take a lock and 2) Will fail - if the row was deleted since the snapshot was taken. - */ - Rdb_transaction *tx= get_or_create_tx(table->in_use); - DEBUG_SYNC(ha_thd(), "rocksdb_concurrent_delete"); - rocksdb::Status s= tx->get_for_update(m_pk_descr->get_cf(), key, + Now we call get_for_update which will 1) Take a lock and 2) Will fail + if the row was deleted since the snapshot was taken. + */ + Rdb_transaction* const tx= get_or_create_tx(table->in_use); + DEBUG_SYNC(ha_thd(), "rocksdb_concurrent_delete"); + const rocksdb::Status s= get_for_update(tx, m_pk_descr->get_cf(), key, &m_retrieved_record); - if (!s.ok()) - { - DBUG_RETURN(tx->set_status_error(table->in_use, s, m_pk_descr, - m_tbl_def)); - } - - // If we called get_for_update() use the value from that call not from - // the iterator as it may be stale since we don't have a snapshot - // when m_lock_rows is not RDB_LOCK_NONE. - m_last_rowkey.copy(key.data(), key.size(), &my_charset_bin); - rc= convert_record_from_storage_format(&key, buf); + if (s.IsNotFound() && + my_core::thd_tx_isolation(ha_thd()) == ISO_READ_COMMITTED) + { + // This occurs if we accessed a row, tried to lock it, failed, + // released and reacquired the snapshot (because of READ COMMITTED + // mode) and the row was deleted by someone else in the meantime. + // If so, we just want to move on to the next row. + continue; } - else + + if (!s.ok()) { - // Use the value from the iterator - rocksdb::Slice value= m_scan_it->value(); - m_last_rowkey.copy(key.data(), key.size(), &my_charset_bin); - rc= convert_record_from_storage_format(&key, &value, buf); + DBUG_RETURN(tx->set_status_error(table->in_use, s, *m_pk_descr, + m_tbl_def)); } - table->status= 0; + // If we called get_for_update() use the value from that call not from + // the iterator as it may be stale since we don't have a snapshot + // when m_lock_rows is not RDB_LOCK_NONE. + m_last_rowkey.copy(key.data(), key.size(), &my_charset_bin); + rc= convert_record_from_storage_format(&key, buf); } + else + { + // Use the value from the iterator + rocksdb::Slice value= m_scan_it->value(); + m_last_rowkey.copy(key.data(), key.size(), &my_charset_bin); + rc= convert_record_from_storage_format(&key, &value, buf); + } + + table->status= 0; + break; } - else - rc= HA_ERR_END_OF_FILE; if (!rc) { stats.rows_read++; @@ -8135,7 +8452,7 @@ int ha_rocksdb::rnd_end() int ha_rocksdb::index_init(uint idx, bool sorted) { - Rdb_transaction *tx= get_or_create_tx(table->in_use); + Rdb_transaction* const tx= get_or_create_tx(table->in_use); DBUG_ASSERT(tx != nullptr); setup_read_decoders(); @@ -8160,7 +8477,7 @@ int ha_rocksdb::index_end() } -int ha_rocksdb::delete_all_rows(ha_rows* nrows) +int ha_rocksdb::delete_all_rows(ha_rows* const nrows) { DBUG_ENTER("ha_rocksdb::delete_all_rows"); /* @@ -8187,7 +8504,7 @@ int ha_rocksdb::truncate() The row is either table->record[0] or table->record[1]. (Check out InnoDB: row_update_for_mysql() has "UT_NOT_USED(mysql_rec)" */ -int ha_rocksdb::delete_row(const uchar *buf) +int ha_rocksdb::delete_row(const uchar* const buf) { DBUG_ENTER("ha_rocksdb::delete_row"); @@ -8195,15 +8512,15 @@ int ha_rocksdb::delete_row(const uchar *buf) set_last_rowkey(buf); rocksdb::Slice key_slice(m_last_rowkey.ptr(), m_last_rowkey.length()); - Rdb_transaction *tx= get_or_create_tx(table->in_use); + Rdb_transaction* const tx= get_or_create_tx(table->in_use); - uint index= pk_index(table, m_tbl_def); + const uint index= pk_index(table, m_tbl_def); rocksdb::Status s= delete_or_singledelete(index, tx, m_pk_descr->get_cf(), key_slice); if (!s.ok()) { - DBUG_RETURN(tx->set_status_error(table->in_use, s, m_pk_descr, m_tbl_def)); + DBUG_RETURN(tx->set_status_error(table->in_use, s, *m_pk_descr, m_tbl_def)); } longlong hidden_pk_id= 0; @@ -8217,13 +8534,13 @@ int ha_rocksdb::delete_row(const uchar *buf) if (!is_pk(i, table, m_tbl_def)) { int packed_size; - const std::shared_ptr<const Rdb_key_def>& kd= m_key_descr_arr[i]; - packed_size= kd->pack_record(table, m_pack_buffer, buf, m_sk_packed_tuple, + const Rdb_key_def& kd= *m_key_descr_arr[i]; + packed_size= kd.pack_record(table, m_pack_buffer, buf, m_sk_packed_tuple, nullptr, false, hidden_pk_id); rocksdb::Slice secondary_key_slice( reinterpret_cast<const char*>(m_sk_packed_tuple), packed_size); /* Deleting on secondary key doesn't need any locks: */ - tx->get_indexed_write_batch()->SingleDelete(kd->get_cf(), + tx->get_indexed_write_batch()->SingleDelete(kd.get_cf(), secondary_key_slice); } } @@ -8241,9 +8558,9 @@ int ha_rocksdb::delete_row(const uchar *buf) rocksdb::Status ha_rocksdb::delete_or_singledelete(uint index, - Rdb_transaction *tx, - rocksdb::ColumnFamilyHandle* column_family, - const rocksdb::Slice& key) + Rdb_transaction* const tx, + rocksdb::ColumnFamilyHandle* const column_family, + const rocksdb::Slice& key) { if (can_use_single_delete(index)) return tx->single_delete(column_family, key); @@ -8339,7 +8656,7 @@ int ha_rocksdb::info(uint flag) { continue; } - KEY* k= &table->key_info[i]; + KEY* const k= &table->key_info[i]; for (uint j = 0; j < k->actual_key_parts; j++) { const Rdb_index_stats& k_stats= m_key_descr_arr[i]->m_stats; @@ -8381,7 +8698,7 @@ int ha_rocksdb::info(uint flag) DBUG_RETURN(0); } -void ha_rocksdb::position(const uchar *record) +void ha_rocksdb::position(const uchar* const record) { DBUG_ENTER("ha_rocksdb::position"); @@ -8402,7 +8719,7 @@ void ha_rocksdb::position(const uchar *record) read set (this may potentially disable index-only access when PK column cannot be restored from its mem-comparable form in the secondary indexes). */ - uint packed_size= m_pk_descr->pack_record(table, m_pack_buffer, record, + const uint packed_size= m_pk_descr->pack_record(table, m_pack_buffer, record, ref, nullptr, false, hidden_pk_id); /* @@ -8416,7 +8733,7 @@ void ha_rocksdb::position(const uchar *record) } -int ha_rocksdb::rnd_pos(uchar *buf, uchar *pos) +int ha_rocksdb::rnd_pos(uchar* const buf, uchar* const pos) { int rc; size_t len; @@ -8462,15 +8779,15 @@ void ha_rocksdb::calc_updated_indexes() for (uint keynr= 0; keynr < table->s->keys; keynr++) { - const std::shared_ptr<const Rdb_key_def>& kd= m_key_descr_arr[keynr]; + const Rdb_key_def& kd= *m_key_descr_arr[keynr]; /* Walk over all key parts, including the "extended key" suffix */ - uint key_parts= kd->get_key_parts(); + const uint key_parts= kd.get_key_parts(); for (uint kp= 0; kp < key_parts; kp++) { if (has_hidden_pk(table) && kp + 1 == key_parts) break; - Field *field= kd->get_table_field_for_part_no(table, kp); + Field* const field= kd.get_table_field_for_part_no(table, kp); if (bitmap_is_set(table->write_set, field->field_index)) { m_update_scope.set_bit(keynr); @@ -8482,7 +8799,7 @@ void ha_rocksdb::calc_updated_indexes() } -int ha_rocksdb::update_row(const uchar *old_data, uchar *new_data) +int ha_rocksdb::update_row(const uchar* const old_data, uchar* const new_data) { DBUG_ENTER("ha_rocksdb::update_row"); DBUG_ASSERT(m_lock_rows == RDB_LOCK_WRITE); @@ -8494,8 +8811,7 @@ int ha_rocksdb::update_row(const uchar *old_data, uchar *new_data) DBUG_ASSERT(new_data == table->record[0]); ha_statistic_increment(&SSV::ha_update_count); - int rv; - rv= update_write_row(old_data, new_data, false); + const int rv= update_write_row(old_data, new_data, false); if (rv == 0) { @@ -8507,7 +8823,7 @@ int ha_rocksdb::update_row(const uchar *old_data, uchar *new_data) /* The following function was copied from ha_blackhole::store_lock: */ -THR_LOCK_DATA **ha_rocksdb::store_lock(THD *thd, +THR_LOCK_DATA **ha_rocksdb::store_lock(THD* const thd, THR_LOCK_DATA **to, enum thr_lock_type lock_type) { @@ -8533,9 +8849,9 @@ THR_LOCK_DATA **ha_rocksdb::store_lock(THD *thd, The following logic was copied directly from ha_innobase::store_lock_with_x_type() in storage/innobase/handler/ha_innodb.cc and causes MyRocks to leave - locks in place on rows that are in a table that is not beint updated. + locks in place on rows that are in a table that is not being updated. */ - uint sql_command = my_core::thd_sql_command(thd); + const uint sql_command = my_core::thd_sql_command(thd); if ((lock_type == TL_READ && in_lock_tables) || (lock_type == TL_READ_HIGH_PRIORITY && in_lock_tables) || can_hold_read_locks_on_select(thd, lock_type)) @@ -8593,14 +8909,29 @@ THR_LOCK_DATA **ha_rocksdb::store_lock(THD *thd, DBUG_RETURN(to); } -void ha_rocksdb::read_thd_vars(THD *thd) +void ha_rocksdb::read_thd_vars(THD* const thd) { - m_store_checksums= THDVAR(thd, store_checksums); - m_verify_checksums= THDVAR(thd, verify_checksums); + m_store_row_debug_checksums= THDVAR(thd, store_row_debug_checksums); + m_verify_row_debug_checksums= THDVAR(thd, verify_row_debug_checksums); m_checksums_pct= THDVAR(thd, checksums_pct); } -int ha_rocksdb::external_lock(THD *thd, int lock_type) +const char* ha_rocksdb::thd_rocksdb_tmpdir() +{ + const char* tmp_dir = THDVAR(ha_thd(), tmpdir); + + /* + We want to treat an empty string as nullptr, in these cases DDL operations + will use the default --tmpdir passed to mysql instead. + */ + if (tmp_dir != nullptr && *tmp_dir == '\0') { + tmp_dir= nullptr; + } + + return(tmp_dir); +} + +int ha_rocksdb::external_lock(THD* const thd, int lock_type) { int res= 0; DBUG_ENTER("ha_rocksdb::external_lock"); @@ -8621,7 +8952,7 @@ int ha_rocksdb::external_lock(THD *thd, int lock_type) if (lock_type == F_UNLCK) { - Rdb_transaction* tx= get_or_create_tx(thd); + Rdb_transaction* const tx= get_or_create_tx(thd); tx->io_perf_end_and_record(&m_io_perf); if (!my_core::thd_test_options(thd, OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN)) @@ -8656,7 +8987,7 @@ int ha_rocksdb::external_lock(THD *thd, int lock_type) is, handler->start_stmt() is not called for INSERTs. So, we put this code here. */ - Rdb_transaction* tx= get_or_create_tx(thd); + Rdb_transaction* const tx= get_or_create_tx(thd); read_thd_vars(thd); m_update_scope_is_valid= false; @@ -8726,9 +9057,9 @@ int ha_rocksdb::external_lock(THD *thd, int lock_type) </quote> */ -int ha_rocksdb::start_stmt(THD *thd, thr_lock_type lock_type) +int ha_rocksdb::start_stmt(THD* const thd, thr_lock_type lock_type) { - Rdb_transaction* tx= get_or_create_tx(thd); + Rdb_transaction* const tx= get_or_create_tx(thd); read_thd_vars(thd); rocksdb_register_tx(ht, thd, tx); tx->io_perf_start(&m_io_perf); @@ -8750,17 +9081,17 @@ rocksdb::Range get_range( } static rocksdb::Range get_range( - const std::shared_ptr<const Rdb_key_def>& kd, + const Rdb_key_def& kd, uchar buf[Rdb_key_def::INDEX_NUMBER_SIZE*2], int offset1, int offset2) { - return get_range(kd->get_index_number(), buf, offset1, offset2); + return get_range(kd.get_index_number(), buf, offset1, offset2); } -rocksdb::Range get_range(const std::shared_ptr<const Rdb_key_def>& kd, +rocksdb::Range get_range(const Rdb_key_def& kd, uchar buf[Rdb_key_def::INDEX_NUMBER_SIZE*2]) { - if (kd->m_is_reverse_cf) + if (kd.m_is_reverse_cf) { return myrocks::get_range(kd, buf, 1, 0); } @@ -8771,9 +9102,9 @@ rocksdb::Range get_range(const std::shared_ptr<const Rdb_key_def>& kd, } rocksdb::Range ha_rocksdb::get_range( - int i, uchar buf[Rdb_key_def::INDEX_NUMBER_SIZE*2]) const + const int &i, uchar buf[Rdb_key_def::INDEX_NUMBER_SIZE*2]) const { - return myrocks::get_range(m_key_descr_arr[i], buf); + return myrocks::get_range(*m_key_descr_arr[i], buf); } @@ -8801,7 +9132,7 @@ void Rdb_drop_index_thread::run() ? 24*60*60 // no filtering : 60; // filtering - auto ret __attribute__((__unused__)) = mysql_cond_timedwait( + const auto ret __attribute__((__unused__)) = mysql_cond_timedwait( &m_signal_cond, &m_signal_mutex, &ts); if (m_stop) { break; @@ -8817,7 +9148,7 @@ void Rdb_drop_index_thread::run() rocksdb::ReadOptions read_opts; read_opts.total_order_seek = true; // disable bloom filter - for (auto d : indices) { + for (const auto d : indices) { uint32 cf_flags= 0; if (!dict_manager.get_cf_flags(d.cf_id, &cf_flags)) { @@ -8828,12 +9159,13 @@ void Rdb_drop_index_thread::run() } rocksdb::ColumnFamilyHandle* cfh= cf_manager.get_cf(d.cf_id); DBUG_ASSERT(cfh); - bool is_reverse_cf= cf_flags & Rdb_key_def::REVERSE_CF_FLAG; + const bool is_reverse_cf= cf_flags & Rdb_key_def::REVERSE_CF_FLAG; bool index_removed= false; uchar key_buf[Rdb_key_def::INDEX_NUMBER_SIZE]= {0}; rdb_netbuf_store_uint32(key_buf, d.index_id); - rocksdb::Slice key = rocksdb::Slice((char*)key_buf, sizeof(key_buf)); + const rocksdb::Slice + key = rocksdb::Slice((char*)key_buf, sizeof(key_buf)); uchar buf[Rdb_key_def::INDEX_NUMBER_SIZE*2]; rocksdb::Range range = get_range(d.index_id, buf, is_reverse_cf?1:0, is_reverse_cf?0:1); @@ -8904,7 +9236,7 @@ void Rdb_drop_index_thread::run() } -Rdb_tbl_def* ha_rocksdb::get_table_if_exists(const char* tablename) +Rdb_tbl_def* ha_rocksdb::get_table_if_exists(const char* const tablename) { DBUG_ASSERT(tablename != nullptr); @@ -8926,14 +9258,14 @@ Rdb_tbl_def* ha_rocksdb::get_table_if_exists(const char* tablename) tablename points to line in form "./dbname/tablename". */ -int ha_rocksdb::delete_table(const char *tablename) +int ha_rocksdb::delete_table(const char* const tablename) { DBUG_ENTER("ha_rocksdb::delete_table"); - std::unique_ptr<rocksdb::WriteBatch> wb= dict_manager.begin(); - rocksdb::WriteBatch *batch= wb.get(); + const std::unique_ptr<rocksdb::WriteBatch> wb= dict_manager.begin(); + rocksdb::WriteBatch* const batch= wb.get(); /* Find the table in the hash */ - Rdb_tbl_def *tbl = get_table_if_exists(tablename); + Rdb_tbl_def* const tbl = get_table_if_exists(tablename); if (!tbl) DBUG_RETURN(HA_ERR_NO_SUCH_TABLE); dict_manager.add_drop_table(tbl->m_key_descr_arr, tbl->m_key_count, batch); @@ -8952,13 +9284,14 @@ int ha_rocksdb::delete_table(const char *tablename) } -void ha_rocksdb::remove_rows(Rdb_tbl_def *tbl) +void ha_rocksdb::remove_rows(Rdb_tbl_def* const tbl) { - rocksdb::WriteOptions wo= rdb_get_rocksdb_write_options(handler::ha_thd()); + const rocksdb::WriteOptions wo= + rdb_get_rocksdb_write_options(handler::ha_thd()); rocksdb::ReadOptions opts; opts.total_order_seek= true; - rocksdb::Iterator* it= rdb->NewIterator(opts); + rocksdb::Iterator* const it= rdb->NewIterator(opts); char key_buf[MAX_KEY_LENGTH]; uint key_len; @@ -8969,15 +9302,15 @@ void ha_rocksdb::remove_rows(Rdb_tbl_def *tbl) */ for (uint i= 0; i < tbl->m_key_count ; i++) { - const std::shared_ptr<const Rdb_key_def>& kd= tbl->m_key_descr_arr[i]; - kd->get_infimum_key(reinterpret_cast<uchar*>(key_buf), &key_len); + const Rdb_key_def& kd= *tbl->m_key_descr_arr[i]; + kd.get_infimum_key(reinterpret_cast<uchar*>(key_buf), &key_len); - rocksdb::Slice table_key(key_buf, key_len); + const rocksdb::Slice table_key(key_buf, key_len); it->Seek(table_key); while (it->Valid()) { - rocksdb::Slice key= it->key(); - if (!kd->covers_key(key)) + const rocksdb::Slice key= it->key(); + if (!kd.covers_key(key)) { break; } @@ -8993,7 +9326,7 @@ void ha_rocksdb::remove_rows(Rdb_tbl_def *tbl) } -int ha_rocksdb::rename_table(const char *from, const char *to) +int ha_rocksdb::rename_table(const char* const from, const char* const to) { DBUG_ASSERT(from != nullptr); DBUG_ASSERT(to != nullptr); @@ -9051,8 +9384,8 @@ int ha_rocksdb::rename_table(const char *from, const char *to) } } - std::unique_ptr<rocksdb::WriteBatch> wb= dict_manager.begin(); - rocksdb::WriteBatch *batch= wb.get(); + const std::unique_ptr<rocksdb::WriteBatch> wb= dict_manager.begin(); + rocksdb::WriteBatch* const batch= wb.get(); dict_manager.lock(); write_err= ddl_manager.rename(from_str, to_str, batch) || dict_manager.commit(batch); @@ -9072,7 +9405,7 @@ int ha_rocksdb::rename_table(const char *from, const char *to) */ -bool ha_rocksdb::check_if_incompatible_data(HA_CREATE_INFO *info, +bool ha_rocksdb::check_if_incompatible_data(HA_CREATE_INFO* const info, uint table_changes) { DBUG_ENTER("ha_rocksdb::check_if_incompatible_data"); @@ -9110,8 +9443,8 @@ int ha_rocksdb::extra(enum ha_extra_function operation) Given a starting key and an ending key, estimate the number of rows that will exist between the two keys. */ -ha_rows ha_rocksdb::records_in_range(uint inx, key_range *min_key, - key_range *max_key) +ha_rows ha_rocksdb::records_in_range(uint inx, key_range* const min_key, + key_range* const max_key) { DBUG_ENTER("ha_rocksdb::records_in_range"); ha_rows ret = THDVAR(ha_thd(), records_in_range); @@ -9119,38 +9452,37 @@ ha_rows ha_rocksdb::records_in_range(uint inx, key_range *min_key, DBUG_RETURN(ret); } if (table->force_index) { - ha_rows force_rows = THDVAR(ha_thd(), force_index_records_in_range); + const ha_rows force_rows = THDVAR(ha_thd(), force_index_records_in_range); if (force_rows) { DBUG_RETURN(force_rows); } } - const std::shared_ptr<const Rdb_key_def>& kd= m_key_descr_arr[inx]; - DBUG_ASSERT(kd); + const Rdb_key_def& kd= *m_key_descr_arr[inx]; uint size1=0; if (min_key) { - size1 = kd->pack_index_tuple(table, m_pack_buffer, m_sk_packed_tuple, + size1 = kd.pack_index_tuple(table, m_pack_buffer, m_sk_packed_tuple, min_key->key, min_key->keypart_map); if (min_key->flag == HA_READ_PREFIX_LAST_OR_PREV || min_key->flag == HA_READ_PREFIX_LAST || min_key->flag == HA_READ_AFTER_KEY) { - kd->successor(m_sk_packed_tuple, size1); + kd.successor(m_sk_packed_tuple, size1); } } else { - kd->get_infimum_key(m_sk_packed_tuple, &size1); + kd.get_infimum_key(m_sk_packed_tuple, &size1); } uint size2=0; if (max_key) { - size2 = kd->pack_index_tuple(table, m_pack_buffer, m_sk_packed_tuple_old, + size2 = kd.pack_index_tuple(table, m_pack_buffer, m_sk_packed_tuple_old, max_key->key, max_key->keypart_map); if (max_key->flag == HA_READ_PREFIX_LAST_OR_PREV || max_key->flag == HA_READ_PREFIX_LAST || max_key->flag == HA_READ_AFTER_KEY) { - kd->successor(m_sk_packed_tuple_old, size2); + kd.successor(m_sk_packed_tuple_old, size2); } // pad the upper key with FFFFs to make sure it is more than the lower if (size1 > size2) { @@ -9158,11 +9490,11 @@ ha_rows ha_rocksdb::records_in_range(uint inx, key_range *min_key, size2 = size1; } } else { - kd->get_supremum_key(m_sk_packed_tuple_old, &size2); + kd.get_supremum_key(m_sk_packed_tuple_old, &size2); } - rocksdb::Slice slice1((const char*) m_sk_packed_tuple, size1); - rocksdb::Slice slice2((const char*) m_sk_packed_tuple_old, size2); + const rocksdb::Slice slice1((const char*) m_sk_packed_tuple, size1); + const rocksdb::Slice slice2((const char*) m_sk_packed_tuple_old, size2); // slice1 >= slice2 means no row will match if (slice1.compare(slice2) >= 0) @@ -9171,15 +9503,15 @@ ha_rows ha_rocksdb::records_in_range(uint inx, key_range *min_key, } rocksdb::Range r( - kd->m_is_reverse_cf ? slice2 : slice1, - kd->m_is_reverse_cf ? slice1 : slice2 + kd.m_is_reverse_cf ? slice2 : slice1, + kd.m_is_reverse_cf ? slice1 : slice2 ); uint64_t sz=0; - auto disk_size = kd->m_stats.m_actual_disk_size; + auto disk_size = kd.m_stats.m_actual_disk_size; if (disk_size == 0) - disk_size = kd->m_stats.m_data_size; - auto rows = kd->m_stats.m_rows; + disk_size = kd.m_stats.m_data_size; + auto rows = kd.m_stats.m_rows; if (rows == 0 || disk_size == 0) { rows= 1; @@ -9187,7 +9519,7 @@ ha_rows ha_rocksdb::records_in_range(uint inx, key_range *min_key, } // Getting statistics, including from Memtables rdb->GetApproximateSizes( - kd->get_cf(), + kd.get_cf(), &r, 1, &sz, true); ret = rows*sz/disk_size; @@ -9217,7 +9549,7 @@ ha_rows ha_rocksdb::records_in_range(uint inx, key_range *min_key, } -void ha_rocksdb::update_create_info(HA_CREATE_INFO *create_info) +void ha_rocksdb::update_create_info(HA_CREATE_INFO* const create_info) { if (!(create_info->used_fields & HA_CREATE_USED_AUTO)) { @@ -9239,7 +9571,7 @@ void ha_rocksdb::update_create_info(HA_CREATE_INFO *create_info) column family, the first row of the next index id should be the last row of the previous index id. */ -int ha_rocksdb::optimize(THD *thd, HA_CHECK_OPT* check_opt) +int ha_rocksdb::optimize(THD* const thd, HA_CHECK_OPT* const check_opt) { int rc= 0; for (uint i= 0; i < table->s->keys; i++) @@ -9257,24 +9589,23 @@ int ha_rocksdb::optimize(THD *thd, HA_CHECK_OPT* check_opt) return rc; } -int ha_rocksdb::analyze(THD* thd, HA_CHECK_OPT* check_opt) +int ha_rocksdb::calculate_stats(const TABLE* const table_arg, THD* const thd, + HA_CHECK_OPT* const check_opt) { - DBUG_ENTER("ha_rocksdb::analyze"); - - if (!table) - DBUG_RETURN(1); + DBUG_ENTER("ha_rocksdb::calculate_stats"); // find per column family key ranges which need to be queried std::unordered_map<rocksdb::ColumnFamilyHandle*, std::vector<rocksdb::Range>> ranges; std::unordered_set<GL_INDEX_ID> ids_to_check; - std::vector<uchar> buf(table->s->keys * 2 * Rdb_key_def::INDEX_NUMBER_SIZE); - for (uint i = 0; i < table->s->keys; i++) + std::vector<uchar> buf(table_arg->s->keys * 2 * + Rdb_key_def::INDEX_NUMBER_SIZE); + for (uint i = 0; i < table_arg->s->keys; i++) { - auto bufp = &buf[i * 2 * Rdb_key_def::INDEX_NUMBER_SIZE]; - const std::shared_ptr<const Rdb_key_def>& kd= m_key_descr_arr[i]; - ranges[kd->get_cf()].push_back(get_range(i, bufp)); - ids_to_check.insert(kd->get_gl_index_id()); + const auto bufp = &buf[i * 2 * Rdb_key_def::INDEX_NUMBER_SIZE]; + const Rdb_key_def& kd= *m_key_descr_arr[i]; + ranges[kd.get_cf()].push_back(get_range(i, bufp)); + ids_to_check.insert(kd.get_gl_index_id()); } // for analyze statements, force flush on memtable to get accurate cardinality @@ -9292,8 +9623,8 @@ int ha_rocksdb::analyze(THD* thd, HA_CHECK_OPT* check_opt) rocksdb::TablePropertiesCollection props; for (auto it : ranges) { - auto old_size __attribute__((__unused__)) = props.size(); - auto status = rdb->GetPropertiesOfTablesInRange( + const auto old_size __attribute__((__unused__)) = props.size(); + const auto status = rdb->GetPropertiesOfTablesInRange( it.first, &it.second[0], it.second.size(), &props); DBUG_ASSERT(props.size() >= old_size); if (!status.ok()) @@ -9303,13 +9634,13 @@ int ha_rocksdb::analyze(THD* thd, HA_CHECK_OPT* check_opt) int num_sst= 0; // group stats per index id std::unordered_map<GL_INDEX_ID, Rdb_index_stats> stats; - for (auto it : ids_to_check) + for (const auto &it : ids_to_check) { // Initialize the stats to 0. If there are no files that contain // this gl_index_id, then 0 should be stored for the cached stats. stats[it] = Rdb_index_stats(it); } - for (auto it : props) + for (const auto &it : props) { std::vector<Rdb_index_stats> sst_stats; Rdb_tbl_prop_coll::read_stats_from_tbl_props(it.second, &sst_stats); @@ -9317,7 +9648,7 @@ int ha_rocksdb::analyze(THD* thd, HA_CHECK_OPT* check_opt) sst_stats is a list of index statistics for indexes that have entries in the current SST file. */ - for (auto it1 : sst_stats) + for (const auto &it1 : sst_stats) { /* Only update statistics for indexes that belong to this SQL table. @@ -9330,8 +9661,8 @@ int ha_rocksdb::analyze(THD* thd, HA_CHECK_OPT* check_opt) if (ids_to_check.find(it1.m_gl_index_id) == ids_to_check.end()) continue; - const std::shared_ptr<const Rdb_key_def>& kd= - ddl_manager.find(it1.m_gl_index_id); + auto kd= ddl_manager.safe_find(it1.m_gl_index_id); + DBUG_ASSERT(kd != nullptr); stats[it1.m_gl_index_id].merge(it1, true, kd->max_storage_fmt_length()); } num_sst++; @@ -9344,10 +9675,20 @@ int ha_rocksdb::analyze(THD* thd, HA_CHECK_OPT* check_opt) DBUG_RETURN(0); } -void ha_rocksdb::get_auto_increment(ulonglong offset, ulonglong increment, +int ha_rocksdb::analyze(THD* const thd, HA_CHECK_OPT* const check_opt) +{ + DBUG_ENTER("ha_rocksdb::analyze"); + + if (!table) + DBUG_RETURN(1); + + DBUG_RETURN(calculate_stats(table, thd, check_opt)); +} + +void ha_rocksdb::get_auto_increment(ulonglong off, ulonglong inc, ulonglong nb_desired_values, - ulonglong *first_value, - ulonglong *nb_reserved_values) + ulonglong* const first_value, + ulonglong* const nb_reserved_values) { /* MySQL has a somewhat complicated way of handling the auto-increment value. @@ -9364,7 +9705,49 @@ void ha_rocksdb::get_auto_increment(ulonglong offset, ulonglong increment, m_auto_incr_val this should be safe - if we had to grab a mutex, doing an actual reserve of some values might be a better solution. */ - *first_value= m_tbl_def->m_auto_incr_val++; + DEBUG_SYNC(ha_thd(), "rocksdb.autoinc_vars"); + + if (off > inc) + { + off= 1; + } + + // Local variable reference to simplify code below + std::atomic<longlong>& auto_incr= m_tbl_def->m_auto_incr_val; + + if (inc == 1 && off == 1) + { + // Optimization for the standard case where we are always simply + // incrementing from the last position + *first_value= auto_incr++; + } + else + { + // The next value can be more complicated if either `inc` or 'off' is not 1 + longlong new_val; + longlong last_val= auto_incr; + + // Loop until we can correctly update the atomic value + do + { + // Calculate the next value in the auto increment series: + // offset + N * increment + // where N is 0, 1, 2, ... + // + // For further information please visit: + // http://dev.mysql.com/doc/refman/5.7/en/replication-options-master.html + new_val= ((last_val + (inc - off) - 1) / inc) * inc + off; + + // Attempt to store the new value (plus 1 since m_auto_incr_val contains + // the next available value) into the atomic value. If the current + // value no longer matches what we have in 'last_val' this will fail and + // we will repeat the loop (`last_val` will automatically get updated + // with the current value). + } while (!auto_incr.compare_exchange_weak(last_val, new_val + 1)); + + *first_value= new_val; + } + *nb_reserved_values= 1; } @@ -9374,9 +9757,9 @@ void ha_rocksdb::get_auto_increment(ulonglong offset, ulonglong increment, /* Debugger help function */ static char dbug_item_print_buf[512]; -const char *dbug_print_item(Item *item) +const char *dbug_print_item(Item* const item) { - char *buf= dbug_item_print_buf; + char* const buf= dbug_item_print_buf; String str(buf, sizeof(dbug_item_print_buf), &my_charset_bin); str.length(0); if (!item) @@ -9412,7 +9795,7 @@ const char *dbug_print_item(Item *item) Part of condition we couldn't check (always nullptr). */ -class Item* ha_rocksdb::idx_cond_push(uint keyno, class Item* idx_cond) +class Item* ha_rocksdb::idx_cond_push(uint keyno, class Item* const idx_cond) { DBUG_ASSERT(keyno != MAX_KEY); DBUG_ASSERT(idx_cond != nullptr); @@ -9444,7 +9827,7 @@ class Item* ha_rocksdb::idx_cond_push(uint keyno, class Item* idx_cond) and return it) */ -enum icp_result ha_rocksdb::check_index_cond() +enum icp_result ha_rocksdb::check_index_cond() const { DBUG_ASSERT(pushed_idx_cond); DBUG_ASSERT(pushed_idx_cond_keyno != MAX_KEY); @@ -9470,16 +9853,14 @@ enum icp_result ha_rocksdb::check_index_cond() @param kd @param find_flag */ -bool ha_rocksdb::is_ascending(const std::shared_ptr<const Rdb_key_def>& kd, +bool ha_rocksdb::is_ascending(const Rdb_key_def& kd, enum ha_rkey_function find_flag) const { - DBUG_ASSERT(kd != nullptr); - bool is_ascending; switch (find_flag) { case HA_READ_KEY_EXACT: { - is_ascending = !kd->m_is_reverse_cf; + is_ascending = !kd.m_is_reverse_cf; break; } case HA_READ_PREFIX: @@ -9490,7 +9871,7 @@ bool ha_rocksdb::is_ascending(const std::shared_ptr<const Rdb_key_def>& kd, case HA_READ_KEY_OR_NEXT: case HA_READ_AFTER_KEY: { - is_ascending = !kd->m_is_reverse_cf; + is_ascending = !kd.m_is_reverse_cf; break; } case HA_READ_KEY_OR_PREV: @@ -9498,7 +9879,7 @@ bool ha_rocksdb::is_ascending(const std::shared_ptr<const Rdb_key_def>& kd, case HA_READ_PREFIX_LAST: case HA_READ_PREFIX_LAST_OR_PREV: { - is_ascending = kd->m_is_reverse_cf; + is_ascending = kd.m_is_reverse_cf; break; } default: @@ -9513,7 +9894,7 @@ bool ha_rocksdb::is_ascending(const std::shared_ptr<const Rdb_key_def>& kd, my_core::enum_alter_inplace_result ha_rocksdb::check_if_supported_inplace_alter( TABLE* /* altered_table */, - my_core::Alter_inplace_info *ha_alter_info) + my_core::Alter_inplace_info* const ha_alter_info) { DBUG_ENTER("check_if_supported_alter"); @@ -9557,24 +9938,24 @@ my_core::enum_alter_inplace_result ha_rocksdb::check_if_supported_inplace_alter( @retval false Success */ bool ha_rocksdb::prepare_inplace_alter_table( - TABLE *altered_table, - my_core::Alter_inplace_info *ha_alter_info) + TABLE* const altered_table, + my_core::Alter_inplace_info* const ha_alter_info) { DBUG_ENTER("prepare_inplace_alter_table"); - uint n_dropped_keys= ha_alter_info->index_drop_count; - uint n_added_keys= ha_alter_info->index_add_count; - uint old_n_keys= m_tbl_def->m_key_count; + const uint n_dropped_keys= ha_alter_info->index_drop_count; + const uint n_added_keys= ha_alter_info->index_add_count; + const uint old_n_keys= m_tbl_def->m_key_count; uint new_n_keys= altered_table->s->keys; if (has_hidden_pk(altered_table)) { new_n_keys += 1; } - std::shared_ptr<Rdb_key_def>* old_key_descr= m_tbl_def->m_key_descr_arr; - std::shared_ptr<Rdb_key_def>* new_key_descr= + std::shared_ptr<Rdb_key_def>* const old_key_descr= m_tbl_def->m_key_descr_arr; + std::shared_ptr<Rdb_key_def>* const new_key_descr= new std::shared_ptr<Rdb_key_def>[new_n_keys]; - Rdb_tbl_def* new_tdef= new Rdb_tbl_def(m_tbl_def->full_tablename()); + Rdb_tbl_def* const new_tdef= new Rdb_tbl_def(m_tbl_def->full_tablename()); new_tdef->m_key_descr_arr= new_key_descr; new_tdef->m_key_count= new_n_keys; new_tdef->m_auto_incr_val= @@ -9610,7 +9991,7 @@ bool ha_rocksdb::prepare_inplace_alter_table( /* Determine which(if any) key definition(s) need to be dropped */ for (i = 0; i < ha_alter_info->index_drop_count; i++) { - const KEY* key = ha_alter_info->index_drop_buffer[i]; + const KEY* const key = ha_alter_info->index_drop_buffer[i]; for (j = 0; j < old_n_keys; j++) { if (!old_key_descr[j]->m_name.compare(key->name)) @@ -9667,14 +10048,14 @@ bool ha_rocksdb::prepare_inplace_alter_table( @retval true Error @retval false Success */ -bool ha_rocksdb::inplace_alter_table(TABLE* altered_table, - my_core::Alter_inplace_info *ha_alter_info) +bool ha_rocksdb::inplace_alter_table(TABLE* const altered_table, + my_core::Alter_inplace_info* const ha_alter_info) { /* Do nothing for now */ DBUG_ENTER("inplace_alter_table"); DBUG_ASSERT(ha_alter_info->handler_ctx); - Rdb_inplace_alter_ctx* ctx= + Rdb_inplace_alter_ctx* const ctx= static_cast<Rdb_inplace_alter_ctx*> (ha_alter_info->handler_ctx); if (ha_alter_info->handler_flags & my_core::Alter_inplace_info::ADD_INDEX) @@ -9703,16 +10084,16 @@ bool ha_rocksdb::inplace_alter_table(TABLE* altered_table, /** Scan the Primary Key index entries and populate the new secondary keys. */ -int ha_rocksdb::inplace_populate_sk(const TABLE* new_table_arg, +int ha_rocksdb::inplace_populate_sk(const TABLE* const new_table_arg, const std::unordered_set<std::shared_ptr<Rdb_key_def>>& indexes) { DBUG_ENTER("ha_rocksdb::inplace_populate_sk"); - std::unique_ptr<rocksdb::WriteBatch> wb= dict_manager.begin(); - rocksdb::WriteBatch *batch= wb.get(); + const std::unique_ptr<rocksdb::WriteBatch> wb= dict_manager.begin(); + rocksdb::WriteBatch* const batch= wb.get(); /* Update the data dictionary */ std::unordered_set<GL_INDEX_ID> create_index_ids; - for (auto& index : indexes) + for (const auto& index : indexes) { create_index_ids.insert(index->get_gl_index_id()); } @@ -9722,15 +10103,54 @@ int ha_rocksdb::inplace_populate_sk(const TABLE* new_table_arg, const bool hidden_pk_exists = has_hidden_pk(table); int res= 0; - Rdb_transaction *tx; - ulonglong rdb_merge_buf_size= THDVAR(ha_thd(), merge_buf_size); - ulonglong rdb_merge_combine_read_size= THDVAR(ha_thd(), + Rdb_transaction *tx= get_or_create_tx(table->in_use); + + /* + There is one specific scenario where m_sst_info may not be nullptr. This + happens if the handler we're using happens to be the handler where the PK + bulk load was done on. The sequence of events that lead to this is as + follows (T1 is PK bulk load, T2 is SK alter table): + + T1: Execute last INSERT statement + T1: Return TABLE and handler object back to Table_cache_manager + T1: Close connection + T2: Execute ALTER statement + T2: Take same TABLE/handler from Table_cache_manager + T2: Call closefrm which will call finalize_bulk_load on every other open + table/handler *except* the one it's on. + T2: Acquire stale snapshot of PK + T1: Call finalize_bulk_load + + This is rare because usually, closefrm will call the destructor (and thus + finalize_bulk_load) on the handler where PK bulk load is done. However, if + the thread ids of the bulk load thread and the alter thread differ by a + multiple of table_cache_instances (8 by default), then they hash to the + same bucket in Table_cache_manager and the alter thread will not not call + the destructor on the handler it is holding. Thus, its m_sst_info will not + be nullptr. + + At this point, it is safe to refresh the snapshot because we know all other + open handlers have been closed at this point, and the one we're on is the + only one left. + */ + if (m_sst_info != nullptr) + { + if ((res= finalize_bulk_load())) + { + DBUG_RETURN(res); + } + tx->commit(); + } + + const ulonglong rdb_merge_buf_size= THDVAR(ha_thd(), merge_buf_size); + const ulonglong rdb_merge_combine_read_size= THDVAR(ha_thd(), merge_combine_read_size); - for (auto& index : indexes) + for (const auto& index : indexes) { const rocksdb::Comparator* index_comp= index->get_cf()->GetComparator(); - Rdb_index_merge rdb_merge(rdb_merge_buf_size, rdb_merge_combine_read_size, + Rdb_index_merge rdb_merge(thd_rocksdb_tmpdir(), rdb_merge_buf_size, + rdb_merge_combine_read_size, index_comp); if ((res= rdb_merge.init())) @@ -9743,7 +10163,7 @@ int ha_rocksdb::inplace_populate_sk(const TABLE* new_table_arg, as the pk index position may have changed in the case of hidden primary keys. */ - uint pk= pk_index(table, m_tbl_def); + const uint pk= pk_index(table, m_tbl_def); ha_index_init(pk, true); /* Scan each record in the primary key in order */ @@ -9761,15 +10181,16 @@ int ha_rocksdb::inplace_populate_sk(const TABLE* new_table_arg, } /* Create new secondary index entry */ - int new_packed_size= index->pack_record(new_table_arg, m_pack_buffer, + const int new_packed_size= index->pack_record( + new_table_arg, m_pack_buffer, table->record[0], m_sk_packed_tuple, &m_sk_tails, - should_store_checksums(), + should_store_row_debug_checksums(), hidden_pk_id); - rocksdb::Slice key= rocksdb::Slice( + const rocksdb::Slice key= rocksdb::Slice( reinterpret_cast<const char*>(m_sk_packed_tuple), new_packed_size); - rocksdb::Slice val= rocksdb::Slice( + const rocksdb::Slice val= rocksdb::Slice( reinterpret_cast<const char*>(m_sk_tails.ptr()), m_sk_tails.get_current_pos()); @@ -9792,7 +10213,6 @@ int ha_rocksdb::inplace_populate_sk(const TABLE* new_table_arg, DBUG_RETURN(res); } - tx= get_or_create_tx(table->in_use); ha_index_end(); /* @@ -9806,7 +10226,7 @@ int ha_rocksdb::inplace_populate_sk(const TABLE* new_table_arg, /* Insert key and slice to SST via SSTFileWriter API. */ - if ((res= bulk_load_key(tx, index, merge_key, merge_val))) + if ((res= bulk_load_key(tx, *index, merge_key, merge_val))) { break; } @@ -9874,11 +10294,11 @@ int ha_rocksdb::inplace_populate_sk(const TABLE* new_table_arg, @retval false Success */ bool ha_rocksdb::commit_inplace_alter_table( - my_core::TABLE *altered_table, - my_core::Alter_inplace_info *ha_alter_info, + my_core::TABLE* const altered_table, + my_core::Alter_inplace_info* const ha_alter_info, bool commit) { - Rdb_inplace_alter_ctx* ctx0= + Rdb_inplace_alter_ctx* const ctx0= static_cast<Rdb_inplace_alter_ctx*> (ha_alter_info->handler_ctx); DBUG_ENTER("commit_inplace_alter_table"); @@ -9954,8 +10374,8 @@ bool ha_rocksdb::commit_inplace_alter_table( my_core::Alter_inplace_info::DROP_UNIQUE_INDEX | my_core::Alter_inplace_info::ADD_INDEX)) { - std::unique_ptr<rocksdb::WriteBatch> wb= dict_manager.begin(); - rocksdb::WriteBatch *batch= wb.get(); + const std::unique_ptr<rocksdb::WriteBatch> wb= dict_manager.begin(); + rocksdb::WriteBatch* const batch= wb.get(); std::unordered_set<GL_INDEX_ID> create_index_ids; m_tbl_def= ctx0->m_new_tdef; @@ -9965,12 +10385,13 @@ bool ha_rocksdb::commit_inplace_alter_table( dict_manager.lock(); for (inplace_alter_handler_ctx** pctx = ctx_array; *pctx; pctx++) { - Rdb_inplace_alter_ctx* ctx= static_cast<Rdb_inplace_alter_ctx*> (*pctx); + Rdb_inplace_alter_ctx* const ctx= + static_cast<Rdb_inplace_alter_ctx*> (*pctx); /* Mark indexes to be dropped */ dict_manager.add_drop_index(ctx->m_dropped_index_ids, batch); - for (auto& index : ctx->m_added_indexes) + for (const auto& index : ctx->m_added_indexes) { create_index_ids.insert(index->get_gl_index_id()); } @@ -9998,6 +10419,20 @@ bool ha_rocksdb::commit_inplace_alter_table( /* Mark ongoing create indexes as finished/remove from data dictionary */ dict_manager.finish_indexes_operation(create_index_ids, Rdb_key_def::DDL_CREATE_INDEX_ONGOING); + + /* + We need to recalculate the index stats here manually. The reason is that + the secondary index does not exist inside + m_index_num_to_keydef until it is committed to the data dictionary, which + prevents us from updating the stats normally as the ddl_manager cannot + find the proper gl_index_ids yet during adjust_stats calls. + */ + if (calculate_stats(altered_table, nullptr, nullptr)) + { + /* Failed to update index statistics, should never happen */ + DBUG_ASSERT(0); + } + rdb_drop_idx_thread.signal(); } @@ -10059,7 +10494,6 @@ struct rocksdb_status_counters_t { uint64_t number_multiget_bytes_read; uint64_t number_deletes_filtered; uint64_t number_merge_failures; - uint64_t sequence_number; uint64_t bloom_filter_prefix_checked; uint64_t bloom_filter_prefix_useful; uint64_t number_reseeks_iteration; @@ -10116,7 +10550,6 @@ DEF_SHOW_FUNC(number_multiget_keys_read, NUMBER_MULTIGET_KEYS_READ) DEF_SHOW_FUNC(number_multiget_bytes_read, NUMBER_MULTIGET_BYTES_READ) DEF_SHOW_FUNC(number_deletes_filtered, NUMBER_FILTERED_DELETES) DEF_SHOW_FUNC(number_merge_failures, NUMBER_MERGE_FAILURES) -DEF_SHOW_FUNC(sequence_number, SEQUENCE_NUMBER) DEF_SHOW_FUNC(bloom_filter_prefix_checked, BLOOM_FILTER_PREFIX_CHECKED) DEF_SHOW_FUNC(bloom_filter_prefix_useful, BLOOM_FILTER_PREFIX_USEFUL) DEF_SHOW_FUNC(number_reseeks_iteration, NUMBER_OF_RESEEKS_IN_ITERATION) @@ -10209,7 +10642,6 @@ static SHOW_VAR rocksdb_status_vars[]= { DEF_STATUS_VAR(number_multiget_bytes_read), DEF_STATUS_VAR(number_deletes_filtered), DEF_STATUS_VAR(number_merge_failures), - DEF_STATUS_VAR(sequence_number), DEF_STATUS_VAR(bloom_filter_prefix_checked), DEF_STATUS_VAR(bloom_filter_prefix_useful), DEF_STATUS_VAR(number_reseeks_iteration), @@ -10232,6 +10664,9 @@ static SHOW_VAR rocksdb_status_vars[]= { DEF_STATUS_VAR_PTR("snapshot_conflict_errors", &rocksdb_snapshot_conflict_errors, SHOW_LONGLONG), + DEF_STATUS_VAR_PTR("wal_group_syncs", + &rocksdb_wal_group_syncs, + SHOW_LONGLONG), DEF_STATUS_VAR_PTR("number_stat_computes", &rocksdb_number_stat_computes, SHOW_LONGLONG), DEF_STATUS_VAR_PTR("number_sst_entry_put", &rocksdb_num_sst_entry_put, SHOW_LONGLONG), @@ -10254,48 +10689,61 @@ static SHOW_VAR rocksdb_status_vars[]= { void Rdb_background_thread::run() { + // How many seconds to wait till flushing the WAL next time. + const int WAKE_UP_INTERVAL = 1; + timespec ts_next_sync; clock_gettime(CLOCK_REALTIME, &ts_next_sync); - ts_next_sync.tv_sec++; + ts_next_sync.tv_sec += WAKE_UP_INTERVAL; for (;;) { - // wait for 1 second or until we received a condition to stop the thread + // Wait until the next timeout or until we receive a signal to stop the + // thread. Request to stop the thread should only be triggered when the + // storage engine is being unloaded. mysql_mutex_lock(&m_signal_mutex); - auto ret __attribute__((__unused__)) = mysql_cond_timedwait( - &m_signal_cond, &m_signal_mutex, &ts_next_sync); - // make sure that no program error is returned + const auto ret __attribute__((__unused__)) = mysql_cond_timedwait( + &m_signal_cond, &m_signal_mutex, &ts_next_sync); + + // Check that we receive only the expected error codes. DBUG_ASSERT(ret == 0 || ret == ETIMEDOUT); - bool local_stop= m_stop; - bool local_save_stats= m_save_stats; + const bool local_stop= m_stop; + const bool local_save_stats= m_save_stats; reset(); mysql_mutex_unlock(&m_signal_mutex); if (local_stop) { + // If we're here then that's because condition variable was signaled by + // another thread and we're shutting down. Break out the loop to make + // sure that shutdown thread can proceed. break; } + // This path should be taken only when the timer expired. + DBUG_ASSERT(ret == ETIMEDOUT); + if (local_save_stats) { ddl_manager.persist_stats(); } - // Flush the WAL if need be but don't do it more frequent - // than once per second timespec ts; clock_gettime(CLOCK_REALTIME, &ts); - if (ts.tv_sec - ts_next_sync.tv_sec >= 1) + + // Flush the WAL. + if (rdb && rocksdb_background_sync) { - if (rdb && rocksdb_background_sync) - { - DBUG_ASSERT(!rocksdb_db_options.allow_mmap_writes); - rocksdb::Status s= rdb->SyncWAL(); - if (!s.ok()) - rdb_handle_io_error(s, RDB_IO_ERROR_BG_THREAD); + DBUG_ASSERT(!rocksdb_db_options.allow_mmap_writes); + const rocksdb::Status s= rdb->SyncWAL(); + if (!s.ok()) { + rdb_handle_io_error(s, RDB_IO_ERROR_BG_THREAD); } - ts_next_sync.tv_sec= ts.tv_sec + 1; } + + // Set the next timestamp for mysql_cond_timedwait() (which ends up calling + // pthread_cond_timedwait()) to wait on. + ts_next_sync.tv_sec= ts.tv_sec + WAKE_UP_INTERVAL; } // save remaining stats which might've left unsaved @@ -10322,7 +10770,7 @@ void Rdb_background_thread::run() This is aware of extended keys. */ bool can_use_bloom_filter(THD *thd, - const std::shared_ptr<const Rdb_key_def>& kd, + const Rdb_key_def& kd, const rocksdb::Slice &eq_cond, const bool use_all_keys, bool is_ascending) @@ -10334,7 +10782,7 @@ bool can_use_bloom_filter(THD *thd, return can_use; } - rocksdb::Options opt = rdb->GetOptions(kd->get_cf()); + rocksdb::Options opt = rdb->GetOptions(kd.get_cf()); if (opt.prefix_extractor) { /* @@ -10379,7 +10827,7 @@ bool can_use_bloom_filter(THD *thd, } /* For modules that need access to the global data structures */ -rocksdb::DB *rdb_get_rocksdb_db() +rocksdb::TransactionDB *rdb_get_rocksdb_db() { return rdb; } @@ -10395,8 +10843,8 @@ rocksdb::BlockBasedTableOptions& rdb_get_table_options() } -int rdb_get_table_perf_counters(const char *tablename, - Rdb_perf_counters *counters) +int rdb_get_table_perf_counters(const char* const tablename, + Rdb_perf_counters* const counters) { DBUG_ASSERT(counters != nullptr); DBUG_ASSERT(tablename != nullptr); @@ -10436,6 +10884,14 @@ void rdb_handle_io_error(rocksdb::Status status, RDB_IO_ERROR_TYPE err_type) status.ToString().c_str()); break; } + case RDB_IO_ERROR_GENERAL: + { + sql_print_error("RocksDB: Failed on I/O - status %d, %s", + status.code(), status.ToString().c_str()); + sql_print_error("RocksDB: Aborting on I/O error."); + abort_with_stack_traces(); + break; + } default: DBUG_ASSERT(0); break; @@ -10463,7 +10919,7 @@ void rdb_handle_io_error(rocksdb::Status status, RDB_IO_ERROR_TYPE err_type) break; } default: - sql_print_warning("RocksDB: Failed to write to RocksDB " + sql_print_warning("RocksDB: Failed to read/write in RocksDB " "- status %d, %s", status.code(), status.ToString().c_str()); break; @@ -10489,15 +10945,15 @@ Rdb_binlog_manager *rdb_get_binlog_manager(void) void rocksdb_set_compaction_options( - my_core::THD* thd __attribute__((__unused__)), - my_core::st_mysql_sys_var* var __attribute__((__unused__)), - void* var_ptr, - const void* save) + my_core::THD* const thd __attribute__((__unused__)), + my_core::st_mysql_sys_var* const var __attribute__((__unused__)), + void* const var_ptr, + const void* const save) { if (var_ptr && save) { *(uint64_t*)var_ptr = *(const uint64_t*) save; } - Rdb_compact_params params = { + const Rdb_compact_params params = { (uint64_t)rocksdb_compaction_sequential_deletes, (uint64_t)rocksdb_compaction_sequential_deletes_window, (uint64_t)rocksdb_compaction_sequential_deletes_file_size @@ -10508,14 +10964,14 @@ rocksdb_set_compaction_options( } void rocksdb_set_table_stats_sampling_pct( - my_core::THD* thd __attribute__((__unused__)), - my_core::st_mysql_sys_var* var __attribute__((__unused__)), - void* var_ptr __attribute__((__unused__)), - const void* save) + my_core::THD* const thd __attribute__((__unused__)), + my_core::st_mysql_sys_var* const var __attribute__((__unused__)), + void* const var_ptr __attribute__((__unused__)), + const void* const save) { mysql_mutex_lock(&rdb_sysvars_mutex); - uint32_t new_val= *static_cast<const uint32_t*>(save); + const uint32_t new_val= *static_cast<const uint32_t*>(save); if (new_val != rocksdb_table_stats_sampling_pct) { rocksdb_table_stats_sampling_pct = new_val; @@ -10540,12 +10996,12 @@ void rocksdb_set_table_stats_sampling_pct( */ void rocksdb_set_rate_limiter_bytes_per_sec( - my_core::THD* thd, - my_core::st_mysql_sys_var* var __attribute__((__unused__)), - void* var_ptr __attribute__((__unused__)), - const void* save) + my_core::THD* const thd, + my_core::st_mysql_sys_var* const var __attribute__((__unused__)), + void* const var_ptr __attribute__((__unused__)), + const void* const save) { - uint64_t new_val= *static_cast<const uint64_t*>(save); + const uint64_t new_val= *static_cast<const uint64_t*>(save); if (new_val == 0 || rocksdb_rate_limiter_bytes_per_sec == 0) { /* @@ -10567,7 +11023,7 @@ rocksdb_set_rate_limiter_bytes_per_sec( } } -void rdb_set_collation_exception_list(const char *exception_list) +void rdb_set_collation_exception_list(const char* const exception_list) { DBUG_ASSERT(rdb_collation_exceptions != nullptr); @@ -10579,12 +11035,12 @@ void rdb_set_collation_exception_list(const char *exception_list) } void -rocksdb_set_collation_exception_list(THD* thd, - struct st_mysql_sys_var* var, - void* var_ptr, - const void* save) +rocksdb_set_collation_exception_list(THD* const thd, + struct st_mysql_sys_var* const var, + void* const var_ptr, + const void* const save) { - const char* val = *static_cast<const char*const*>(save); + const char* const val = *static_cast<const char*const*>(save); rdb_set_collation_exception_list(val); @@ -10592,16 +11048,16 @@ rocksdb_set_collation_exception_list(THD* thd, } void -rocksdb_set_bulk_load(THD* thd, - struct st_mysql_sys_var* var __attribute__((__unused__)), - void* var_ptr, - const void* save) +rocksdb_set_bulk_load(THD* const thd, + struct st_mysql_sys_var* const var __attribute__((__unused__)), + void* const var_ptr, + const void* const save) { Rdb_transaction*& tx= get_tx_from_thd(thd); if (tx != nullptr) { - int rc= tx->finish_bulk_load(); + const int rc= tx->finish_bulk_load(); if (rc != 0) { // NO_LINT_DEBUG @@ -10695,5 +11151,7 @@ myrocks::rdb_i_s_perf_context_global, myrocks::rdb_i_s_cfoptions, myrocks::rdb_i_s_global_info, myrocks::rdb_i_s_ddl, -myrocks::rdb_i_s_index_file_map +myrocks::rdb_i_s_index_file_map, +myrocks::rdb_i_s_lock_info, +myrocks::rdb_i_s_trx_info mysql_declare_plugin_end; diff --git a/storage/rocksdb/ha_rocksdb.h b/storage/rocksdb/ha_rocksdb.h index 8f63050e74c..a15ae6dd443 100644 --- a/storage/rocksdb/ha_rocksdb.h +++ b/storage/rocksdb/ha_rocksdb.h @@ -63,6 +63,30 @@ namespace myrocks { /* + * class for exporting transaction information for + * information_schema.rocksdb_trx + */ +struct Rdb_trx_info { + std::string name; + ulonglong trx_id; + ulonglong write_count; + ulonglong lock_count; + int timeout_sec; + std::string state; + std::string waiting_key; + ulonglong waiting_cf_id; + int is_replication; + int skip_trx_api; + int read_only; + int deadlock_detect; + int num_ongoing_bulk_load; + ulong thread_id; + std::string query_str; +}; + +std::vector<Rdb_trx_info> rdb_get_all_trx_info(); + +/* This is - the name of the default Column Family (the CF which stores indexes which didn't explicitly specify which CF they are in) @@ -160,7 +184,7 @@ const char * const PER_INDEX_CF_NAME = "$per_index_cf"; #define HA_ERR_ROCKSDB_TOO_MANY_LOCKS (HA_ERR_LAST + 3) #define HA_ERR_ROCKSDB_LAST HA_ERR_ROCKSDB_TOO_MANY_LOCKS -inline bool looks_like_per_index_cf_typo(const char *name) +inline bool looks_like_per_index_cf_typo(const char* const name) { return (name && name[0]=='$' && strcmp(name, PER_INDEX_CF_NAME)); } @@ -237,13 +261,19 @@ enum operation_type { ROWS_MAX }; +#if defined(HAVE_SCHED_GETCPU) +#define RDB_INDEXER get_sched_indexer_t +#else +#define RDB_INDEXER thread_id_indexer_t +#endif + /* Global statistics struct used inside MyRocks */ struct st_global_stats { - ib_counter_t<ulonglong, 64> rows[ROWS_MAX]; + ib_counter_t<ulonglong, 64, RDB_INDEXER> rows[ROWS_MAX]; // system_rows_ stats are only for system // tables. They are not counted in rows_* stats. - ib_counter_t<ulonglong, 64> system_rows[ROWS_MAX]; + ib_counter_t<ulonglong, 64, RDB_INDEXER> system_rows[ROWS_MAX]; }; /* Struct used for exporting status to MySQL */ @@ -270,8 +300,8 @@ namespace std { { std::size_t operator()(const myrocks::GL_INDEX_ID& gl_index_id) const { - uint64_t val = ((uint64_t) gl_index_id.cf_id << 32 | - (uint64_t) gl_index_id.index_id); + const uint64_t val = ((uint64_t) gl_index_id.cf_id << 32 | + (uint64_t) gl_index_id.index_id); return std::hash<uint64_t>()(val); } }; @@ -296,6 +326,8 @@ class ha_rocksdb: public my_core::handler /* Whether m_scan_it was created with skip_bloom=true */ bool m_scan_it_skips_bloom; + const rocksdb::Snapshot *m_scan_it_snapshot; + Rdb_tbl_def *m_tbl_def; /* Primary Key encoder from KeyTupleFormat to StorageFormat */ @@ -407,42 +439,46 @@ class ha_rocksdb: public my_core::handler /* SST information used for bulk loading the primary key */ std::shared_ptr<Rdb_sst_info> m_sst_info; + Rdb_transaction *m_bulk_load_tx; + /* Mutex to protect finalizing bulk load */ + mysql_mutex_t m_bulk_load_mutex; /* MySQL index number for duplicate key error */ int m_dupp_errkey; - int create_key_defs(const TABLE *table_arg, Rdb_tbl_def *tbl_def_arg, - const TABLE *old_table_arg= nullptr, - const Rdb_tbl_def *old_tbl_def_arg= nullptr) + int create_key_defs(const TABLE* const table_arg, + Rdb_tbl_def* const tbl_def_arg, + const TABLE* const old_table_arg= nullptr, + const Rdb_tbl_def* const old_tbl_def_arg= nullptr) const __attribute__((__nonnull__(2, 3), __warn_unused_result__)); - int secondary_index_read(const int keyno, uchar *buf) + int secondary_index_read(const int keyno, uchar* const buf) __attribute__((__nonnull__, __warn_unused_result__)); void setup_iterator_for_rnd_scan(); - void setup_scan_iterator(const std::shared_ptr<const Rdb_key_def>& kd, - rocksdb::Slice *slice) + void setup_scan_iterator(const Rdb_key_def& kd, rocksdb::Slice* const slice) __attribute__((__nonnull__)) { setup_scan_iterator(kd, slice, false, false, 0); } - bool is_ascending(const std::shared_ptr<const Rdb_key_def>& keydef, + bool is_ascending(const Rdb_key_def& keydef, enum ha_rkey_function find_flag) const __attribute__((__nonnull__, __warn_unused_result__)); - void setup_scan_iterator(const std::shared_ptr<const Rdb_key_def>& kd, + void setup_scan_iterator(const Rdb_key_def& kd, rocksdb::Slice *slice, const bool use_all_keys, const bool is_ascending, const uint eq_cond_len) __attribute__((__nonnull__)); - void release_scan_iterator(void) - { - delete m_scan_it; - m_scan_it= nullptr; - } + void release_scan_iterator(void); + + rocksdb::Status get_for_update(Rdb_transaction* const tx, + rocksdb::ColumnFamilyHandle* const column_family, + const rocksdb::Slice& key, + std::string* const value) const; - int get_row_by_rowid(uchar *buf, const char *rowid, + int get_row_by_rowid(uchar* const buf, const char* const rowid, const uint rowid_size) __attribute__((__nonnull__, __warn_unused_result__)); - int get_row_by_rowid(uchar *buf, const uchar *rowid, + int get_row_by_rowid(uchar* const buf, const uchar* const rowid, const uint rowid_size) __attribute__((__nonnull__, __warn_unused_result__)) { @@ -454,21 +490,21 @@ class ha_rocksdb: public my_core::handler void load_auto_incr_value(); longlong update_hidden_pk_val(); int load_hidden_pk_value() __attribute__((__warn_unused_result__)); - int read_hidden_pk_id_from_rowkey(longlong *hidden_pk_id) + int read_hidden_pk_id_from_rowkey(longlong* const hidden_pk_id) __attribute__((__nonnull__, __warn_unused_result__)); - bool can_use_single_delete(uint index) + bool can_use_single_delete(const uint &index) const __attribute__((__warn_unused_result__)); - bool skip_unique_check() __attribute__((__warn_unused_result__)); + bool skip_unique_check() const __attribute__((__warn_unused_result__)); void set_force_skip_unique_check(bool skip) override; bool commit_in_the_middle() __attribute__((__warn_unused_result__)); - bool do_bulk_commit(Rdb_transaction *tx) + bool do_bulk_commit(Rdb_transaction* const tx) __attribute__((__nonnull__, __warn_unused_result__)); - bool has_hidden_pk(const TABLE* table) + bool has_hidden_pk(const TABLE* const table) const __attribute__((__nonnull__, __warn_unused_result__)); - void update_row_stats(operation_type type); + void update_row_stats(const operation_type &type); - void set_last_rowkey(const uchar *old_data); + void set_last_rowkey(const uchar* const old_data); /* Array of table->s->fields elements telling how to store fields in the @@ -504,14 +540,15 @@ class ha_rocksdb: public my_core::handler */ uint m_null_bytes_in_rec; - void get_storage_type(Rdb_field_encoder *encoder, uint kp); + void get_storage_type(Rdb_field_encoder* const encoder, const uint &kp); void setup_field_converters(); - int alloc_key_buffers(const TABLE* table_arg, const Rdb_tbl_def* tbl_def_arg) + int alloc_key_buffers(const TABLE* const table_arg, + const Rdb_tbl_def* const tbl_def_arg) __attribute__((__nonnull__, __warn_unused_result__)); void free_key_buffers(); // the buffer size should be at least 2*Rdb_key_def::INDEX_NUMBER_SIZE - rocksdb::Range get_range(int i, uchar buf[]) const; + rocksdb::Range get_range(const int &i, uchar buf[]) const; /* Perf timers for data reads @@ -534,18 +571,20 @@ public: Controls whether writes include checksums. This is updated from the session variable at the start of each query. */ - bool m_store_checksums; + bool m_store_row_debug_checksums; /* Same as above but for verifying checksums when reading */ - bool m_verify_checksums; + bool m_verify_row_debug_checksums; int m_checksums_pct; - ha_rocksdb(my_core::handlerton *hton, my_core::TABLE_SHARE *table_arg); + ha_rocksdb(my_core::handlerton* const hton, + my_core::TABLE_SHARE* const table_arg); ~ha_rocksdb() { int err __attribute__((__unused__)); err= finalize_bulk_load(); DBUG_ASSERT(err == 0); + mysql_mutex_destroy(&m_bulk_load_mutex); } /** @brief @@ -614,41 +653,44 @@ public: return true; } - bool should_store_checksums() const + bool should_store_row_debug_checksums() const { - return m_store_checksums && (rand() % 100 < m_checksums_pct); + return m_store_row_debug_checksums && (rand() % 100 < m_checksums_pct); } - int rename_table(const char *from, const char *to) + int rename_table(const char* const from, const char* const to) __attribute__((__nonnull__, __warn_unused_result__)); - int convert_record_from_storage_format(const rocksdb::Slice *key, - const rocksdb::Slice *value, - uchar *buf) + int convert_record_from_storage_format(const rocksdb::Slice* const key, + const rocksdb::Slice* const value, + uchar* const buf) __attribute__((__nonnull__, __warn_unused_result__)); - int convert_record_from_storage_format(const rocksdb::Slice *key, - uchar *buf) + int convert_record_from_storage_format(const rocksdb::Slice* const key, + uchar* const buf) __attribute__((__nonnull__, __warn_unused_result__)); void convert_record_to_storage_format(const rocksdb::Slice& pk_packed_slice, - Rdb_string_writer *pk_unpack_info, - rocksdb::Slice *packed_rec) + Rdb_string_writer* const pk_unpack_info, + rocksdb::Slice* const packed_rec) __attribute__((__nonnull__)); - static const char* get_key_name(const uint index, const TABLE* table_arg, - const Rdb_tbl_def* tbl_def_arg) + static const char* get_key_name(const uint index, + const TABLE* const table_arg, + const Rdb_tbl_def* const tbl_def_arg) __attribute__((__nonnull__, __warn_unused_result__)); - static const char* get_key_comment(const uint index, const TABLE* table_arg, - const Rdb_tbl_def* tbl_def_arg) + static const char* get_key_comment(const uint index, + const TABLE* const table_arg, + const Rdb_tbl_def* const tbl_def_arg) __attribute__((__nonnull__, __warn_unused_result__)); - static bool is_hidden_pk(const uint index, const TABLE* table_arg, - const Rdb_tbl_def* tbl_def_arg) + static bool is_hidden_pk(const uint index, const TABLE* const table_arg, + const Rdb_tbl_def* const tbl_def_arg) __attribute__((__nonnull__, __warn_unused_result__)); - static uint pk_index(const TABLE* table_arg, const Rdb_tbl_def* tbl_def_arg) + static uint pk_index(const TABLE* const table_arg, + const Rdb_tbl_def* const tbl_def_arg) __attribute__((__nonnull__, __warn_unused_result__)); static bool is_pk(const uint index, const TABLE* table_arg, @@ -686,52 +728,53 @@ public: ha_rows estimate_rows_upper_bound() { return HA_POS_ERROR; } /* At the moment, we're ok with default handler::index_init() implementation. */ - int index_read_map(uchar * buf, const uchar * key, + int index_read_map(uchar* const buf, const uchar* const key, key_part_map keypart_map, enum ha_rkey_function find_flag); __attribute__((__warn_unused_result__)); - int index_read_map_impl(uchar * buf, const uchar * key, + int index_read_map_impl(uchar* const buf, const uchar* const key, key_part_map keypart_map, enum ha_rkey_function find_flag, - const key_range *end_key) + const key_range* end_key) __attribute__((__warn_unused_result__));; - int index_read_last_map(uchar * buf, const uchar * key, + int index_read_last_map(uchar* const buf, const uchar* const key, key_part_map keypart_map) __attribute__((__warn_unused_result__));; - int read_range_first(const key_range *start_key, - const key_range *end_key, + int read_range_first(const key_range* const start_key, + const key_range* const end_key, bool eq_range, bool sorted) __attribute__((__warn_unused_result__));; virtual double scan_time() { return (double) (stats.records+stats.deleted) / 20.0+10; } virtual double read_time(uint, uint, ha_rows rows) override; - int open(const char *name, int mode, uint test_if_locked) + int open(const char* const name, int mode, uint test_if_locked) __attribute__((__warn_unused_result__)); int close(void) __attribute__((__warn_unused_result__)); - int write_row(uchar *buf) __attribute__((__warn_unused_result__)); - int update_row(const uchar *old_data, uchar *new_data) + int write_row(uchar* const buf) __attribute__((__warn_unused_result__)); + int update_row(const uchar* const old_data, uchar* const new_data) + __attribute__((__warn_unused_result__)); + int delete_row(const uchar* const buf) __attribute__((__warn_unused_result__)); - int delete_row(const uchar *buf) __attribute__((__warn_unused_result__)); rocksdb::Status delete_or_singledelete(uint index, - Rdb_transaction *tx, - rocksdb::ColumnFamilyHandle* cf, + Rdb_transaction* const tx, + rocksdb::ColumnFamilyHandle* const cf, const rocksdb::Slice& key) __attribute__((__warn_unused_result__)); - int index_next(uchar *buf) __attribute__((__warn_unused_result__)); - int index_next_with_direction(uchar *buf, bool move_forward) + int index_next(uchar* const buf) __attribute__((__warn_unused_result__)); + int index_next_with_direction(uchar* const buf, bool move_forward) __attribute__((__warn_unused_result__)); - int index_prev(uchar *buf) __attribute__((__warn_unused_result__)); + int index_prev(uchar* const buf) __attribute__((__warn_unused_result__)); - int index_first(uchar *buf) __attribute__((__warn_unused_result__)); - int index_last(uchar *buf) __attribute__((__warn_unused_result__)); + int index_first(uchar* const buf) __attribute__((__warn_unused_result__)); + int index_last(uchar* const buf) __attribute__((__warn_unused_result__)); - class Item* idx_cond_push(uint keyno, class Item* idx_cond); + class Item* idx_cond_push(uint keyno, class Item* const idx_cond); /* Default implementation from cancel_pushed_idx_cond() suits us */ @@ -758,31 +801,32 @@ private: bool skip_unique_check; }; - int create_cfs(const TABLE *table_arg, Rdb_tbl_def *tbl_def_arg, - std::array<struct key_def_cf_info, MAX_INDEXES + 1>* cfs); + int create_cfs(const TABLE* const table_arg, Rdb_tbl_def* const tbl_def_arg, + std::array<struct key_def_cf_info, MAX_INDEXES + 1>* const cfs) const; __attribute__((__nonnull__, __warn_unused_result__)); - int create_key_def(const TABLE *table_arg, uint i, - const Rdb_tbl_def* tbl_def_arg, - std::shared_ptr<Rdb_key_def>* new_key_def, - const struct key_def_cf_info& cf_info); + int create_key_def(const TABLE* const table_arg, const uint &i, + const Rdb_tbl_def* const tbl_def_arg, + std::shared_ptr<Rdb_key_def>* const new_key_def, + const struct key_def_cf_info& cf_info) const; __attribute__((__nonnull__, __warn_unused_result__)); - int create_inplace_key_defs(const TABLE *table_arg, - Rdb_tbl_def *tbl_def_arg, - const TABLE *old_table_arg, - const Rdb_tbl_def *old_tbl_def_arg, - const std::array<key_def_cf_info, MAX_INDEXES + 1>& cfs); + int create_inplace_key_defs(const TABLE* const table_arg, + Rdb_tbl_def* vtbl_def_arg, + const TABLE* const old_table_arg, + const Rdb_tbl_def* const old_tbl_def_arg, + const std::array<key_def_cf_info, MAX_INDEXES + 1>& cfs) const; __attribute__((__nonnull__, __warn_unused_result__)); std::unordered_map<std::string, uint> get_old_key_positions( const TABLE* table_arg, const Rdb_tbl_def* tbl_def_arg, const TABLE* old_table_arg, - const Rdb_tbl_def* old_tbl_def_arg) + const Rdb_tbl_def* old_tbl_def_arg) const __attribute__((__nonnull__)); - int compare_key_parts(const KEY* old_key, const KEY* new_key); + int compare_key_parts(const KEY* const old_key, + const KEY* const new_key) const; __attribute__((__nonnull__, __warn_unused_result__)); int index_first_intern(uchar *buf) @@ -790,89 +834,99 @@ private: int index_last_intern(uchar *buf) __attribute__((__nonnull__, __warn_unused_result__)); - enum icp_result check_index_cond(); - int find_icp_matching_index_rec(bool move_forward, uchar *buf) + enum icp_result check_index_cond() const; + int find_icp_matching_index_rec(const bool &move_forward, uchar* const buf) __attribute__((__nonnull__, __warn_unused_result__)); void calc_updated_indexes(); - int update_write_row(const uchar *old_data, const uchar *new_data, + int update_write_row(const uchar* const old_data, const uchar* const new_data, const bool skip_unique_check) __attribute__((__warn_unused_result__)); - int get_pk_for_update(struct update_row_info* row_info); - int check_and_lock_unique_pk(uint key_id, + int get_pk_for_update(struct update_row_info* const row_info); + int check_and_lock_unique_pk(const uint &key_id, const struct update_row_info& row_info, - bool* found, bool* pk_changed) + bool* const found, bool* const pk_changed) __attribute__((__warn_unused_result__)); - int check_and_lock_sk(uint key_id, const struct update_row_info& row_info, - bool* found) const + int check_and_lock_sk(const uint &key_id, + const struct update_row_info& row_info, + bool* const found) const __attribute__((__warn_unused_result__)); int check_uniqueness_and_lock(const struct update_row_info& row_info, - bool* pk_changed) + bool* const pk_changed) __attribute__((__warn_unused_result__)); bool over_bulk_load_threshold(int* err) __attribute__((__warn_unused_result__)); - int bulk_load_key(Rdb_transaction* tx, - const std::shared_ptr<const Rdb_key_def>& kd, + int bulk_load_key(Rdb_transaction* const tx, + const Rdb_key_def& kd, const rocksdb::Slice& key, const rocksdb::Slice& value) __attribute__((__nonnull__, __warn_unused_result__)); - int update_pk(const std::shared_ptr<const Rdb_key_def>& kd, + int update_pk(const Rdb_key_def& kd, const struct update_row_info& row_info, - bool pk_changed) + const bool &pk_changed) __attribute__((__warn_unused_result__)); - int update_sk(const TABLE* table_arg, - const std::shared_ptr<const Rdb_key_def>& kd, + int update_sk(const TABLE* const table_arg, + const Rdb_key_def& kd, const struct update_row_info& row_info) __attribute__((__warn_unused_result__)); - int update_indexes(const struct update_row_info& row_info, bool pk_changed) + int update_indexes(const struct update_row_info& row_info, + const bool &pk_changed) __attribute__((__warn_unused_result__)); - int read_key_exact(const std::shared_ptr<const Rdb_key_def>& kd, - rocksdb::Iterator* iter, bool using_full_key, + int read_key_exact(const Rdb_key_def& kd, + rocksdb::Iterator* const iter, const bool &using_full_key, const rocksdb::Slice& key_slice) const __attribute__((__nonnull__, __warn_unused_result__)); - int read_before_key(const std::shared_ptr<const Rdb_key_def>& kd, - bool using_full_key, const rocksdb::Slice& key_slice) + int read_before_key(const Rdb_key_def& kd, + const bool &using_full_key, + const rocksdb::Slice& key_slice) __attribute__((__nonnull__, __warn_unused_result__)); - int read_after_key(const std::shared_ptr<const Rdb_key_def>& kd, - bool using_full_key, const rocksdb::Slice& key_slice) + int read_after_key(const Rdb_key_def& kd, + const bool &using_full_key, + const rocksdb::Slice& key_slice) __attribute__((__nonnull__, __warn_unused_result__)); - int position_to_correct_key(const std::shared_ptr<const Rdb_key_def>& kd, - enum ha_rkey_function find_flag, - bool full_key_match, const uchar* key, - key_part_map keypart_map, + int position_to_correct_key(const Rdb_key_def& kd, + const enum ha_rkey_function &find_flag, + const bool &full_key_match, + const uchar* const key, + const key_part_map &keypart_map, const rocksdb::Slice& key_slice, - bool* move_forward) + bool* const move_forward) __attribute__((__warn_unused_result__)); - int read_row_from_primary_key(uchar* buf) + int read_row_from_primary_key(uchar* const buf) __attribute__((__nonnull__, __warn_unused_result__)); - int read_row_from_secondary_key(uchar* buf, - const std::shared_ptr<const Rdb_key_def>& kd, + int read_row_from_secondary_key(uchar* const buf, + const Rdb_key_def& kd, bool move_forward) __attribute__((__nonnull__, __warn_unused_result__)); int try_keyonly_read_from_sk(uchar* buf, - const std::shared_ptr<const Rdb_key_def>& kd, + const Rdb_key_def& kd, const rocksdb::Slice& key, const rocksdb::Slice& value, uint rowid_size) __attribute__((__nonnull__, __warn_unused_result__)); - int calc_eq_cond_len(const std::shared_ptr<const Rdb_key_def>& kd, - enum ha_rkey_function find_flag, - const rocksdb::Slice& slice, int bytes_changed_by_succ, - const key_range *end_key, uint* end_key_packed_size) + int calc_eq_cond_len(const Rdb_key_def& kd, + const enum ha_rkey_function &find_flag, + const rocksdb::Slice& slice, + const int &bytes_changed_by_succ, + const key_range* const end_key, + uint* const end_key_packed_size) __attribute__((__warn_unused_result__)); - Rdb_tbl_def* get_table_if_exists(const char* tablename) + Rdb_tbl_def* get_table_if_exists(const char* const tablename) __attribute__((__nonnull__, __warn_unused_result__)); - void read_thd_vars(THD *thd) + void read_thd_vars(THD* const thd) __attribute__((__nonnull__)); - bool contains_foreign_key(THD* thd) + const char* thd_rocksdb_tmpdir() __attribute__((__nonnull__, __warn_unused_result__)); - int inplace_populate_sk(const TABLE* table_arg, + bool contains_foreign_key(THD* const thd) + __attribute__((__nonnull__, __warn_unused_result__)); + + int inplace_populate_sk(const TABLE* const table_arg, const std::unordered_set<std::shared_ptr<Rdb_key_def>>& indexes); public: @@ -892,12 +946,13 @@ public: int rnd_init(bool scan) __attribute__((__warn_unused_result__)); int rnd_end() __attribute__((__warn_unused_result__)); - int rnd_next(uchar *buf) __attribute__((__warn_unused_result__)); - int rnd_next_with_direction(uchar *buf, bool move_forward) + int rnd_next(uchar* const buf) __attribute__((__warn_unused_result__)); + int rnd_next_with_direction(uchar* const buf, bool move_forward) __attribute__((__warn_unused_result__)); - int rnd_pos(uchar *buf, uchar *pos) __attribute__((__warn_unused_result__)); - void position(const uchar *record); + int rnd_pos(uchar* const buf, uchar* const pos) + __attribute__((__warn_unused_result__)); + void position(const uchar* const record); int info(uint) override; /* This function will always return success, therefore no annotation related @@ -905,11 +960,11 @@ public: * required by the interface. */ int extra(enum ha_extra_function operation); - int start_stmt(THD *thd, thr_lock_type lock_type) + int start_stmt(THD* const thd, thr_lock_type lock_type) __attribute__((__warn_unused_result__)); - int external_lock(THD *thd, int lock_type) + int external_lock(THD* const thd, int lock_type) __attribute__((__warn_unused_result__)); - int delete_all_rows(ha_rows* nrows) __attribute__((__warn_unused_result__)); + int delete_all_rows(ha_rows* const nrows) __attribute__((__warn_unused_result__)); int truncate() __attribute__((__warn_unused_result__)); int reset() override @@ -919,64 +974,67 @@ public: return 0; } - int check(THD* thd, HA_CHECK_OPT* check_opt) + int check(THD* const thd, HA_CHECK_OPT* const check_opt) __attribute__((__warn_unused_result__)); - void remove_rows(Rdb_tbl_def *tbl); - ha_rows records_in_range(uint inx, key_range *min_key, - key_range *max_key) + void remove_rows(Rdb_tbl_def* const tbl); + ha_rows records_in_range(uint inx, key_range* const min_key, + key_range* const max_key) __attribute__((__warn_unused_result__)); - int delete_table(const char *from) __attribute__((__warn_unused_result__)); - int create(const char *name, TABLE *form, HA_CREATE_INFO *create_info) + int delete_table(const char* const from) __attribute__((__warn_unused_result__)); + int create(const char* const name, TABLE* const form, + HA_CREATE_INFO* const create_info) __attribute__((__warn_unused_result__)); - bool check_if_incompatible_data(HA_CREATE_INFO *info, + bool check_if_incompatible_data(HA_CREATE_INFO* const info, uint table_changes) __attribute__((__warn_unused_result__)); - THR_LOCK_DATA **store_lock(THD *thd, THR_LOCK_DATA **to, + THR_LOCK_DATA **store_lock(THD* const thd, THR_LOCK_DATA **to, enum thr_lock_type lock_type) __attribute__((__warn_unused_result__)); - my_bool register_query_cache_table(THD *thd, char *table_key, + my_bool register_query_cache_table(THD* const thd, char* const table_key, uint key_length, - qc_engine_callback - *engine_callback, - ulonglong *engine_data) + qc_engine_callback* const engine_callback, + ulonglong* const engine_data) { /* Currently, we don't support query cache */ return FALSE; } - bool get_error_message(const int error, String *buf) + bool get_error_message(const int error, String* const buf) __attribute__((__nonnull__)); void get_auto_increment(ulonglong offset, ulonglong increment, ulonglong nb_desired_values, - ulonglong *first_value, - ulonglong *nb_reserved_values); - void update_create_info(HA_CREATE_INFO *create_info); - int optimize(THD *thd, HA_CHECK_OPT *check_opt) + ulonglong* const first_value, + ulonglong* const nb_reserved_values); + void update_create_info(HA_CREATE_INFO* const create_info); + int optimize(THD* const thd, HA_CHECK_OPT* const check_opt) + __attribute__((__warn_unused_result__)); + int analyze(THD* const thd, HA_CHECK_OPT* const check_opt) __attribute__((__warn_unused_result__)); - int analyze(THD* thd, HA_CHECK_OPT* check_opt) + int calculate_stats(const TABLE* const table_arg, THD* const thd, + HA_CHECK_OPT* const check_opt) __attribute__((__warn_unused_result__)); enum_alter_inplace_result check_if_supported_inplace_alter( TABLE *altered_table, - my_core::Alter_inplace_info *ha_alter_info) override; + my_core::Alter_inplace_info* const ha_alter_info) override; - bool prepare_inplace_alter_table(TABLE *altered_table, - my_core::Alter_inplace_info *ha_alter_info); + bool prepare_inplace_alter_table(TABLE* const altered_table, + my_core::Alter_inplace_info* const ha_alter_info); - bool inplace_alter_table(TABLE *altered_table, - my_core::Alter_inplace_info *ha_alter_info); + bool inplace_alter_table(TABLE* const altered_table, + my_core::Alter_inplace_info* const ha_alter_info); - bool commit_inplace_alter_table(TABLE *altered_table, - my_core::Alter_inplace_info *ha_alter_info, - bool commit); + bool commit_inplace_alter_table(TABLE* const altered_table, + my_core::Alter_inplace_info* const ha_alter_info, + bool commit); int finalize_bulk_load() __attribute__((__warn_unused_result__)); - void set_use_read_free_rpl(const char* whitelist); - void set_skip_unique_check_tables(const char* whitelist); + void set_use_read_free_rpl(const char* const whitelist); + void set_skip_unique_check_tables(const char* const whitelist); public: virtual void rpl_before_delete_rows() override; @@ -1000,10 +1058,10 @@ public: struct Rdb_inplace_alter_ctx : public my_core::inplace_alter_handler_ctx { /* The new table definition */ - Rdb_tbl_def* m_new_tdef; + Rdb_tbl_def* const m_new_tdef; /* Stores the original key definitions */ - std::shared_ptr<Rdb_key_def>* m_old_key_descr; + std::shared_ptr<Rdb_key_def>* const m_old_key_descr; /* Stores the new key definitions */ std::shared_ptr<Rdb_key_def>* m_new_key_descr; @@ -1015,10 +1073,10 @@ struct Rdb_inplace_alter_ctx : public my_core::inplace_alter_handler_ctx const uint m_new_n_keys; /* Stores the added key glids */ - std::unordered_set<std::shared_ptr<Rdb_key_def>> m_added_indexes; + const std::unordered_set<std::shared_ptr<Rdb_key_def>> m_added_indexes; /* Stores the dropped key glids */ - std::unordered_set<GL_INDEX_ID> m_dropped_index_ids; + const std::unordered_set<GL_INDEX_ID> m_dropped_index_ids; /* Stores number of keys to add */ const uint m_n_added_keys; diff --git a/storage/rocksdb/ha_rocksdb_proto.h b/storage/rocksdb/ha_rocksdb_proto.h index df41d2de619..be7b5518423 100644 --- a/storage/rocksdb/ha_rocksdb_proto.h +++ b/storage/rocksdb/ha_rocksdb_proto.h @@ -24,13 +24,15 @@ /* RocksDB includes */ #include "rocksdb/table.h" +#include "rocksdb/utilities/transaction_db.h" namespace myrocks { enum RDB_IO_ERROR_TYPE { RDB_IO_ERROR_TX_COMMIT, RDB_IO_ERROR_DICT_COMMIT, - RDB_IO_ERROR_BG_THREAD + RDB_IO_ERROR_BG_THREAD, + RDB_IO_ERROR_GENERAL }; void rdb_handle_io_error(rocksdb::Status status, RDB_IO_ERROR_TYPE err_type); @@ -58,7 +60,7 @@ void rdb_queue_save_stats_request(); Access to singleton objects. */ -rocksdb::DB *rdb_get_rocksdb_db(); +rocksdb::TransactionDB *rdb_get_rocksdb_db(); class Rdb_cf_manager; Rdb_cf_manager& rdb_get_cf_manager(); diff --git a/storage/rocksdb/logger.h b/storage/rocksdb/logger.h index a1333e8dfd4..dd6dd138e56 100644 --- a/storage/rocksdb/logger.h +++ b/storage/rocksdb/logger.h @@ -61,7 +61,7 @@ class Rdb_logger : public rocksdb::Logger Logv(rocksdb::InfoLogLevel::INFO_LEVEL, format, ap); } - void SetRocksDBLogger(std::shared_ptr<rocksdb::Logger> logger) + void SetRocksDBLogger(const std::shared_ptr<rocksdb::Logger> logger) { m_logger = logger; } diff --git a/storage/rocksdb/mysql-test/rocksdb/include/locking_issues_case3.inc b/storage/rocksdb/mysql-test/rocksdb/include/locking_issues_case3.inc index bd9af241e5c..c23717c4fda 100644 --- a/storage/rocksdb/mysql-test/rocksdb/include/locking_issues_case3.inc +++ b/storage/rocksdb/mysql-test/rocksdb/include/locking_issues_case3.inc @@ -59,8 +59,9 @@ eval SET SESSION TRANSACTION ISOLATION LEVEL $isolation_level; UPDATE t0 SET VALUE=VALUE+1 WHERE id=190000; connection con1; ---error ER_LOCK_DEADLOCK +--error 0,ER_LOCK_DEADLOCK reap; +--echo ERROR: $mysql_errno connection default; disconnect con1; diff --git a/storage/rocksdb/mysql-test/rocksdb/include/locking_issues_case5.inc b/storage/rocksdb/mysql-test/rocksdb/include/locking_issues_case5.inc index 3e4f6350b79..b77a54e4360 100644 --- a/storage/rocksdb/mysql-test/rocksdb/include/locking_issues_case5.inc +++ b/storage/rocksdb/mysql-test/rocksdb/include/locking_issues_case5.inc @@ -64,8 +64,9 @@ DELETE FROM t0 WHERE id=190000; COMMIT; connection con1; ---error ER_LOCK_DEADLOCK +--error 0,ER_LOCK_DEADLOCK reap; +--echo ERROR: $mysql_errno COMMIT; connection default; diff --git a/storage/rocksdb/mysql-test/rocksdb/include/locking_issues_case6.inc b/storage/rocksdb/mysql-test/rocksdb/include/locking_issues_case6.inc index 4cb5cae15aa..9494146ba5c 100644 --- a/storage/rocksdb/mysql-test/rocksdb/include/locking_issues_case6.inc +++ b/storage/rocksdb/mysql-test/rocksdb/include/locking_issues_case6.inc @@ -64,8 +64,9 @@ UPDATE t0 SET id=200001 WHERE id=190000; COMMIT; connection con1; ---error ER_LOCK_DEADLOCK +--error 0,ER_LOCK_DEADLOCK reap; +--echo ERROR: $mysql_errno COMMIT; connection default; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/2pc_group_commit.result b/storage/rocksdb/mysql-test/rocksdb/r/2pc_group_commit.result new file mode 100644 index 00000000000..bfa06f88011 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/r/2pc_group_commit.result @@ -0,0 +1,44 @@ +# Disable for valgrind because this takes too long +DROP DATABASE IF EXISTS mysqlslap; +CREATE DATABASE mysqlslap; +USE mysqlslap; +CREATE TABLE t1(id BIGINT AUTO_INCREMENT, value BIGINT, PRIMARY KEY(id)) ENGINE=rocksdb; +# 2PC enabled, MyRocks durability enabled +SET GLOBAL rocksdb_disable_2pc=0; +SET GLOBAL rocksdb_write_sync=1; +## 2PC + durability + single thread +select variable_value into @c from information_schema.global_status where variable_name='rocksdb_wal_group_syncs'; +select case when variable_value-@c = 1000 then 'true' else 'false' end from information_schema.global_status where variable_name='rocksdb_wal_group_syncs'; +case when variable_value-@c = 1000 then 'true' else 'false' end +true +## 2PC + durability + group commit +select variable_value into @c from information_schema.global_status where variable_name='rocksdb_wal_group_syncs'; +select case when variable_value-@c > 0 and variable_value-@c < 10000 then 'true' else 'false' end from information_schema.global_status where variable_name='rocksdb_wal_group_syncs'; +case when variable_value-@c > 0 and variable_value-@c < 10000 then 'true' else 'false' end +true +# 2PC enabled, MyRocks durability disabled +SET GLOBAL rocksdb_disable_2pc=0; +SET GLOBAL rocksdb_write_sync=0; +select variable_value into @c from information_schema.global_status where variable_name='rocksdb_wal_group_syncs'; +select case when variable_value-@c = 0 then 'true' else 'false' end from information_schema.global_status where variable_name='rocksdb_wal_group_syncs'; +case when variable_value-@c = 0 then 'true' else 'false' end +true +select variable_value into @c from information_schema.global_status where variable_name='rocksdb_wal_group_syncs'; +select case when variable_value-@c = 0 then 'true' else 'false' end from information_schema.global_status where variable_name='rocksdb_wal_group_syncs'; +case when variable_value-@c = 0 then 'true' else 'false' end +true +# 2PC disabled, MyRocks durability enabled +SET GLOBAL rocksdb_disable_2pc=1; +SET GLOBAL rocksdb_write_sync=1; +select variable_value into @c from information_schema.global_status where variable_name='rocksdb_wal_group_syncs'; +select case when variable_value-@c = 0 then 'true' else 'false' end from information_schema.global_status where variable_name='rocksdb_wal_group_syncs'; +case when variable_value-@c = 0 then 'true' else 'false' end +true +select variable_value into @c from information_schema.global_status where variable_name='rocksdb_wal_group_syncs'; +select case when variable_value-@c = 0 then 'true' else 'false' end from information_schema.global_status where variable_name='rocksdb_wal_group_syncs'; +case when variable_value-@c = 0 then 'true' else 'false' end +true +SET GLOBAL rocksdb_disable_2pc=1; +SET GLOBAL rocksdb_write_sync=0; +DROP TABLE t1; +DROP DATABASE mysqlslap; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/add_index_inplace.result b/storage/rocksdb/mysql-test/rocksdb/r/add_index_inplace.result index 4a707d3a6f4..a7d381fbdb1 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/add_index_inplace.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/add_index_inplace.result @@ -276,103 +276,121 @@ SELECT COUNT(*) FROM t1; COUNT(*) 100 DROP TABLE t1; -CREATE TABLE t1 (a INT, b INT, KEY ka(a), KEY kab(a,b)) ENGINE=RocksDB; -INSERT INTO t1 (a, b) VALUES (1, 5); -INSERT INTO t1 (a, b) VALUES (2, 6); -INSERT INTO t1 (a, b) VALUES (3, 7); -# crash_during_online_index_creation -flush logs; -SET SESSION debug="+d,crash_during_online_index_creation"; -ALTER TABLE t1 ADD INDEX kb(b), ALGORITHM=INPLACE; -ERROR HY000: Lost connection to MySQL server during query -SET SESSION debug="-d,crash_during_online_index_creation"; -SHOW CREATE TABLE t1; -Table Create Table -t1 CREATE TABLE `t1` ( - `a` int(11) DEFAULT NULL, - `b` int(11) DEFAULT NULL, - KEY `ka` (`a`), - KEY `kab` (`a`,`b`) -) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 -CHECK TABLE t1; -Table Op Msg_type Msg_text -test.t1 check status OK +CREATE TABLE t1 (a INT, b TEXT); +ALTER TABLE t1 ADD KEY kb(b(10)); +ERROR HY000: Unsupported collation on string indexed column test.t1.b Use binary collation (binary, latin1_bin, utf8_bin). +ALTER TABLE t1 ADD PRIMARY KEY(a); DROP TABLE t1; -CREATE TABLE t1 (i INT, j INT, k INT, PRIMARY KEY (i), KEY(j)) ENGINE = ROCKSDB PARTITION BY KEY(i) PARTITIONS 4; -# crash_during_index_creation_partition -flush logs; -SET SESSION debug="+d,crash_during_index_creation_partition"; -ALTER TABLE t1 ADD INDEX kij(i,j), ALGORITHM=INPLACE; -ERROR HY000: Lost connection to MySQL server during query -SET SESSION debug="-d,crash_during_index_creation_partition"; -SHOW CREATE TABLE t1; -Table Create Table -t1 CREATE TABLE `t1` ( - `i` int(11) NOT NULL DEFAULT '0', - `j` int(11) DEFAULT NULL, - `k` int(11) DEFAULT NULL, - PRIMARY KEY (`i`), - KEY `j` (`j`) -) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 -/*!50100 PARTITION BY KEY (i) -PARTITIONS 4 */ -ALTER TABLE t1 ADD INDEX kij(i,j), ALGORITHM=INPLACE; -SELECT * FROM t1 ORDER BY i LIMIT 10; -i j k -1 1 1 -2 2 2 -3 3 3 -4 4 4 -5 5 5 -6 6 6 -7 7 7 -8 8 8 -9 9 9 -10 10 10 -SELECT COUNT(*) FROM t1; +set global rocksdb_bulk_load=1; +# Establish connection con1 (user=root) +# Switch to connection con1 +show global variables like 'rocksdb_bulk_load'; +Variable_name Value +rocksdb_bulk_load ON +show session variables like 'rocksdb_bulk_load'; +Variable_name Value +rocksdb_bulk_load ON +CREATE TABLE t1 (i INT, j INT, PRIMARY KEY (i)) ENGINE = ROCKSDB; +INSERT INTO t1 VALUES (1,1); +# Disconnecting on con1 +# Establish connection con2 (user=root) +# Switch to connection con2 +ALTER TABLE t1 ADD INDEX kj(j), ALGORITHM=INPLACE; +SELECT COUNT(*) FROM t1 FORCE INDEX(PRIMARY); COUNT(*) -100 +1 +SELECT COUNT(*) FROM t1 FORCE INDEX(kj); +COUNT(*) +1 DROP TABLE t1; -CREATE TABLE t1 (i INT, j INT, k INT, PRIMARY KEY (i), KEY(j)) ENGINE = ROCKSDB PARTITION BY KEY(i) PARTITIONS 4; -# crash_during_index_creation_partition -flush logs; -SET SESSION debug="+d,myrocks_simulate_index_create_rollback"; -# expected assertion failure from sql layer here for alter rollback -call mtr.add_suppression("Assertion `0' failed."); -call mtr.add_suppression("Attempting backtrace. You can use the following information to find out"); -ALTER TABLE t1 ADD INDEX kij(i,j), ALGORITHM=INPLACE; -ERROR HY000: Lost connection to MySQL server during query -SET SESSION debug="-d,myrocks_simulate_index_create_rollback"; -SHOW CREATE TABLE t1; -Table Create Table -t1 CREATE TABLE `t1` ( - `i` int(11) NOT NULL DEFAULT '0', - `j` int(11) DEFAULT NULL, - `k` int(11) DEFAULT NULL, - PRIMARY KEY (`i`), - KEY `j` (`j`) -) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 -/*!50100 PARTITION BY KEY (i) -PARTITIONS 4 */ -ALTER TABLE t1 ADD INDEX kij(i,j), ALGORITHM=INPLACE; +# Establish connection con1 (user=root) +# Establish connection con2 (user=root) +# Switch to connection con1 +CREATE TABLE t1 (i INT, j INT, PRIMARY KEY (i)) ENGINE = ROCKSDB; +set rocksdb_bulk_load=1; +INSERT INTO t1 VALUES (1,1); +# Switch to connection con2 +SELECT COUNT(*) FROM t1 FORCE INDEX(PRIMARY); +COUNT(*) +0 +ALTER TABLE t1 ADD INDEX kj(j), ALGORITHM=INPLACE; +SELECT COUNT(*) FROM t1 FORCE INDEX(PRIMARY); +COUNT(*) +1 +SELECT COUNT(*) FROM t1 FORCE INDEX(kj); +COUNT(*) +1 +set global rocksdb_bulk_load=0; +DROP TABLE t1; +SET @prior_rocksdb_merge_combine_read_size= @@rocksdb_merge_combine_read_size; +SET @prior_rocksdb_strict_collation_check= @@rocksdb_strict_collation_check; +SET @prior_rocksdb_merge_buf_size = @@rocksdb_merge_buf_size; +SET global rocksdb_strict_collation_check = off; +SET session rocksdb_merge_combine_read_size = 566; +SET session rocksdb_merge_buf_size = 336; +show variables like '%rocksdb_bulk_load%'; +Variable_name Value +rocksdb_bulk_load OFF +rocksdb_bulk_load_size 1000 +CREATE TABLE t1 (a VARCHAR(80)) ENGINE=RocksDB; +INSERT INTO t1 (a) VALUES (REPEAT("a", 80)); +INSERT INTO t1 (a) VALUES (REPEAT("a", 80)); +INSERT INTO t1 (a) VALUES (REPEAT("a", 80)); +INSERT INTO t1 (a) VALUES (REPEAT("a", 80)); +ALTER TABLE t1 ADD INDEX ka(a), ALGORITHM=INPLACE; SHOW CREATE TABLE t1; Table Create Table t1 CREATE TABLE `t1` ( - `i` int(11) NOT NULL DEFAULT '0', - `j` int(11) DEFAULT NULL, - `k` int(11) DEFAULT NULL, - PRIMARY KEY (`i`), - KEY `j` (`j`), - KEY `kij` (`i`,`j`) + `a` varchar(80) DEFAULT NULL, + KEY `ka` (`a`) ) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 -/*!50100 PARTITION BY KEY (i) -PARTITIONS 4 */ -SELECT COUNT(*) FROM t1; -COUNT(*) -100 +CHECK TABLE t1; +Table Op Msg_type Msg_text +test.t1 check status OK +SELECT * FROM t1 FORCE INDEX(ka) WHERE a > ""; +a +aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa DROP TABLE t1; -CREATE TABLE t1 (a INT, b TEXT); -ALTER TABLE t1 ADD KEY kb(b(10)); -ERROR HY000: Unsupported collation on string indexed column test.t1.b Use binary collation (binary, latin1_bin, utf8_bin). -ALTER TABLE t1 ADD PRIMARY KEY(a); +SET session rocksdb_merge_buf_size = @prior_rocksdb_merge_buf_size; +SET session rocksdb_merge_combine_read_size = @prior_rocksdb_merge_combine_read_size; +SET global rocksdb_strict_collation_check = @prior_rocksdb_strict_collation_check; +CREATE TABLE t1 (i INT, j INT, PRIMARY KEY (i)) ENGINE = ROCKSDB; +set global rocksdb_force_flush_memtable_now=1; +ALTER TABLE t1 ADD INDEX kj(j), ALGORITHM=INPLACE; +larger +1 +larger +1 +Table Op Msg_type Msg_text +test.t1 analyze status OK +larger +1 +larger +1 +Table Op Msg_type Msg_text +test.t1 analyze status OK +Table Op Msg_type Msg_text +test.t1 analyze status OK +Table Op Msg_type Msg_text +test.t1 analyze status OK +Table Op Msg_type Msg_text +test.t1 analyze status OK +Table Op Msg_type Msg_text +test.t1 analyze status OK +Table Op Msg_type Msg_text +test.t1 analyze status OK +Table Op Msg_type Msg_text +test.t1 analyze status OK +Table Op Msg_type Msg_text +test.t1 analyze status OK +Table Op Msg_type Msg_text +test.t1 analyze status OK +Table Op Msg_type Msg_text +test.t1 analyze status OK +select 1300 < 1300 * 1.5 as "same"; +same +1 DROP TABLE t1; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/add_index_inplace_crash.result b/storage/rocksdb/mysql-test/rocksdb/r/add_index_inplace_crash.result new file mode 100644 index 00000000000..987b34948e8 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/r/add_index_inplace_crash.result @@ -0,0 +1,96 @@ +drop table if exists t1; +CREATE TABLE t1 (a INT, b INT, KEY ka(a), KEY kab(a,b)) ENGINE=RocksDB; +INSERT INTO t1 (a, b) VALUES (1, 5); +INSERT INTO t1 (a, b) VALUES (2, 6); +INSERT INTO t1 (a, b) VALUES (3, 7); +# crash_during_online_index_creation +flush logs; +SET SESSION debug="+d,crash_during_online_index_creation"; +ALTER TABLE t1 ADD INDEX kb(b), ALGORITHM=INPLACE; +ERROR HY000: Lost connection to MySQL server during query +SET SESSION debug="-d,crash_during_online_index_creation"; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` int(11) DEFAULT NULL, + KEY `ka` (`a`), + KEY `kab` (`a`,`b`) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 +CHECK TABLE t1; +Table Op Msg_type Msg_text +test.t1 check status OK +DROP TABLE t1; +CREATE TABLE t1 (i INT, j INT, k INT, PRIMARY KEY (i), KEY(j)) ENGINE = ROCKSDB PARTITION BY KEY(i) PARTITIONS 4; +# crash_during_index_creation_partition +flush logs; +SET SESSION debug="+d,crash_during_index_creation_partition"; +ALTER TABLE t1 ADD INDEX kij(i,j), ALGORITHM=INPLACE; +ERROR HY000: Lost connection to MySQL server during query +SET SESSION debug="-d,crash_during_index_creation_partition"; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `i` int(11) NOT NULL DEFAULT '0', + `j` int(11) DEFAULT NULL, + `k` int(11) DEFAULT NULL, + PRIMARY KEY (`i`), + KEY `j` (`j`) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 +/*!50100 PARTITION BY KEY (i) +PARTITIONS 4 */ +ALTER TABLE t1 ADD INDEX kij(i,j), ALGORITHM=INPLACE; +SELECT * FROM t1 ORDER BY i LIMIT 10; +i j k +1 1 1 +2 2 2 +3 3 3 +4 4 4 +5 5 5 +6 6 6 +7 7 7 +8 8 8 +9 9 9 +10 10 10 +SELECT COUNT(*) FROM t1; +COUNT(*) +100 +DROP TABLE t1; +CREATE TABLE t1 (i INT, j INT, k INT, PRIMARY KEY (i), KEY(j)) ENGINE = ROCKSDB PARTITION BY KEY(i) PARTITIONS 4; +# crash_during_index_creation_partition +flush logs; +SET SESSION debug="+d,myrocks_simulate_index_create_rollback"; +# expected assertion failure from sql layer here for alter rollback +call mtr.add_suppression("Assertion `0' failed."); +call mtr.add_suppression("Attempting backtrace. You can use the following information to find out"); +ALTER TABLE t1 ADD INDEX kij(i,j), ALGORITHM=INPLACE; +ERROR HY000: Lost connection to MySQL server during query +SET SESSION debug="-d,myrocks_simulate_index_create_rollback"; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `i` int(11) NOT NULL DEFAULT '0', + `j` int(11) DEFAULT NULL, + `k` int(11) DEFAULT NULL, + PRIMARY KEY (`i`), + KEY `j` (`j`) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 +/*!50100 PARTITION BY KEY (i) +PARTITIONS 4 */ +ALTER TABLE t1 ADD INDEX kij(i,j), ALGORITHM=INPLACE; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `i` int(11) NOT NULL DEFAULT '0', + `j` int(11) DEFAULT NULL, + `k` int(11) DEFAULT NULL, + PRIMARY KEY (`i`), + KEY `j` (`j`), + KEY `kij` (`i`,`j`) +) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 +/*!50100 PARTITION BY KEY (i) +PARTITIONS 4 */ +SELECT COUNT(*) FROM t1; +COUNT(*) +100 +DROP TABLE t1; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/allow_os_buffer.result b/storage/rocksdb/mysql-test/rocksdb/r/allow_os_buffer.result deleted file mode 100644 index d15566f5a2c..00000000000 --- a/storage/rocksdb/mysql-test/rocksdb/r/allow_os_buffer.result +++ /dev/null @@ -1 +0,0 @@ - RocksDB: Can't disable allow_os_buffer if allow_mmap_reads is enabled diff --git a/storage/rocksdb/mysql-test/rocksdb/r/autoinc_vars_thread.result b/storage/rocksdb/mysql-test/rocksdb/r/autoinc_vars_thread.result new file mode 100644 index 00000000000..aefef4fbf38 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/r/autoinc_vars_thread.result @@ -0,0 +1,24 @@ +#--------------------------- +# two threads inserting simultaneously with increment > 1 +# Issue #390 +#--------------------------- +CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY) ENGINE=rocksdb; +SET auto_increment_increment = 2; +SET auto_increment_offset = 1; +INSERT INTO t1 VALUES(NULL); +SET auto_increment_increment = 2; +SET auto_increment_offset = 1; +SET debug_sync='rocksdb.autoinc_vars SIGNAL parked1 WAIT_FOR go NO_CLEAR_EVENT'; +INSERT INTO t1 VALUES(NULL); +SET debug_sync='rocksdb.autoinc_vars SIGNAL parked2 WAIT_FOR go NO_CLEAR_EVENT'; +INSERT INTO t1 VALUES(NULL); +SET debug_sync='now WAIT_FOR parked1'; +SET debug_sync='now WAIT_FOR parked2'; +SET debug_sync='now SIGNAL go'; +SET debug_sync='RESET'; +SELECT * FROM t1; +a +1 +3 +5 +DROP TABLE t1; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/autoinc_vars_thread_2.result b/storage/rocksdb/mysql-test/rocksdb/r/autoinc_vars_thread_2.result new file mode 100644 index 00000000000..652515c6c09 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/r/autoinc_vars_thread_2.result @@ -0,0 +1,53 @@ +#--------------------------- +# ten threads inserting simultaneously with increment > 1 +# Issue #390 +#--------------------------- +CREATE TABLE t1 (pk INT AUTO_INCREMENT PRIMARY KEY, thr INT) ENGINE=rocksdb; +SET auto_increment_increment = 100; +SET auto_increment_offset = 9 + 1; +SET auto_increment_increment = 100; +SET auto_increment_offset = 8 + 1; +SET auto_increment_increment = 100; +SET auto_increment_offset = 7 + 1; +SET auto_increment_increment = 100; +SET auto_increment_offset = 6 + 1; +SET auto_increment_increment = 100; +SET auto_increment_offset = 5 + 1; +SET auto_increment_increment = 100; +SET auto_increment_offset = 4 + 1; +SET auto_increment_increment = 100; +SET auto_increment_offset = 3 + 1; +SET auto_increment_increment = 100; +SET auto_increment_offset = 2 + 1; +SET auto_increment_increment = 100; +SET auto_increment_offset = 1 + 1; +SET auto_increment_increment = 100; +SET auto_increment_offset = 0 + 1; +LOAD DATA INFILE <input_file> INTO TABLE t1; +LOAD DATA INFILE <input_file> INTO TABLE t1; +LOAD DATA INFILE <input_file> INTO TABLE t1; +LOAD DATA INFILE <input_file> INTO TABLE t1; +LOAD DATA INFILE <input_file> INTO TABLE t1; +LOAD DATA INFILE <input_file> INTO TABLE t1; +LOAD DATA INFILE <input_file> INTO TABLE t1; +LOAD DATA INFILE <input_file> INTO TABLE t1; +LOAD DATA INFILE <input_file> INTO TABLE t1; +LOAD DATA INFILE <input_file> INTO TABLE t1; +SELECT COUNT(*) FROM t1; +COUNT(*) +1000000 +SELECT thr, COUNT(pk) FROM t1 GROUP BY thr; +thr COUNT(pk) +0 100000 +1 100000 +2 100000 +3 100000 +4 100000 +5 100000 +6 100000 +7 100000 +8 100000 +9 100000 +SELECT * FROM t1 ORDER BY pk INTO OUTFILE <output_file>; +All pk values matched their expected values +DROP TABLE t1; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/bulk_load.result b/storage/rocksdb/mysql-test/rocksdb/r/bulk_load.result index 4440cb3ea8d..50b73a98111 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/bulk_load.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/bulk_load.result @@ -17,6 +17,21 @@ LOAD DATA INFILE <input_file> INTO TABLE t1; LOAD DATA INFILE <input_file> INTO TABLE t2; LOAD DATA INFILE <input_file> INTO TABLE t3; set rocksdb_bulk_load=0; +SHOW TABLE STATUS WHERE name LIKE 't%'; +Name Engine Version Row_format Rows Avg_row_length Data_length Max_data_length Index_length Data_free Auto_increment Create_time Update_time Check_time Collation Checksum Create_options Comment +t1 ROCKSDB 10 Fixed 10000000 # # # # 0 NULL NULL NULL NULL latin1_bin NULL +t2 ROCKSDB 10 Fixed 10000000 # # # # 0 NULL NULL NULL NULL latin1_bin NULL +t3 ROCKSDB 10 Fixed 10000000 # # # # 0 NULL NULL NULL NULL latin1_bin NULL partitioned +ANALYZE TABLE t1, t2, t3; +Table Op Msg_type Msg_text +test.t1 analyze status OK +test.t2 analyze status OK +test.t3 analyze status OK +SHOW TABLE STATUS WHERE name LIKE 't%'; +Name Engine Version Row_format Rows Avg_row_length Data_length Max_data_length Index_length Data_free Auto_increment Create_time Update_time Check_time Collation Checksum Create_options Comment +t1 ROCKSDB 10 Fixed 10000000 # # # # 0 NULL NULL NULL NULL latin1_bin NULL +t2 ROCKSDB 10 Fixed 10000000 # # # # 0 NULL NULL NULL NULL latin1_bin NULL +t3 ROCKSDB 10 Fixed 10000000 # # # # 0 NULL NULL NULL NULL latin1_bin NULL partitioned select count(pk) from t1; count(pk) 10000000 diff --git a/storage/rocksdb/mysql-test/rocksdb/r/commit_in_the_middle_ddl.result b/storage/rocksdb/mysql-test/rocksdb/r/commit_in_the_middle_ddl.result new file mode 100644 index 00000000000..4d64d12816f --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/r/commit_in_the_middle_ddl.result @@ -0,0 +1,14 @@ +DROP TABLE IF EXISTS a; +create table a (id int, value int, primary key (id) comment 'cf_a') engine=rocksdb; +set rocksdb_bulk_load=1; +set rocksdb_commit_in_the_middle=1; +alter table a add index v (value) COMMENT 'cf_a'; +set rocksdb_bulk_load=0; +set rocksdb_commit_in_the_middle=0; +select count(*) from a force index(primary); +count(*) +100000 +select count(*) from a force index(v); +count(*) +100000 +DROP TABLE a; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/corrupted_data_reads_debug.result b/storage/rocksdb/mysql-test/rocksdb/r/corrupted_data_reads_debug.result index 28c200ebf30..20ac751f582 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/corrupted_data_reads_debug.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/corrupted_data_reads_debug.result @@ -16,13 +16,13 @@ pk col1 1 1 2 2 3 3 -set @tmp1=@@rocksdb_verify_checksums; -set rocksdb_verify_checksums=1; +set @tmp1=@@rocksdb_verify_row_debug_checksums; +set rocksdb_verify_row_debug_checksums=1; set session debug= "+d,myrocks_simulate_bad_row_read1"; select * from t1 where pk=1; ERROR HY000: Got error 122 from storage engine set session debug= "-d,myrocks_simulate_bad_row_read1"; -set rocksdb_verify_checksums=@tmp1; +set rocksdb_verify_row_debug_checksums=@tmp1; select * from t1 where pk=1; pk col1 1 1 diff --git a/storage/rocksdb/mysql-test/rocksdb/r/index.result b/storage/rocksdb/mysql-test/rocksdb/r/index.result index f61bad7c4a9..99390c8ceb2 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/index.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/index.result @@ -40,3 +40,23 @@ t1 0 PRIMARY 1 pk A # NULL NULL LSMTREE t1 1 a 1 a A # NULL NULL YES LSMTREE simple index on a ALTER TABLE t1 DROP KEY a; DROP TABLE t1; +# +# Issue #376: MyRocks: ORDER BY optimizer is unable to use the index extension +# +create table t0 (a int); +insert into t0 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); +create table t1(a int); +insert into t1 select A.a + B.a* 10 + C.a * 100 from t0 A, t0 B, t0 C; +create table t2 ( +pk int not null, +a int not null, +b int not null, +primary key(pk), +key(a) +) engine=rocksdb; +insert into t2 select A.a, FLOOR(A.a/10), A.a from t1 A; +# This must have type=range, index=a, and must not have 'Using filesort': +explain select * from t2 force index (a) where a=0 and pk>=3 order by pk; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t2 range a a 8 NULL # Using index condition +drop table t0,t1,t2; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/lock_info.result b/storage/rocksdb/mysql-test/rocksdb/r/lock_info.result new file mode 100644 index 00000000000..d0f1221e472 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/r/lock_info.result @@ -0,0 +1,31 @@ +DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t2; +create table t1 (a int, primary key(a) comment 'lock_into_cf1') engine=rocksdb; +insert into t1 values (1); +insert into t1 values (2); +create table t2 (a int, primary key(a) comment 'lock_info_cf2') engine=rocksdb; +insert into t2 values (1); +insert into t2 values (2); +set autocommit=0; +select * from t1 for update; +a +1 +2 +select * from t2 for update; +a +1 +2 +use information_schema; +select rocksdb_ddl.cf, rocksdb_locks.transaction_id, rocksdb_locks.key +from rocksdb_locks +left join rocksdb_ddl +on rocksdb_locks.column_family_id=rocksdb_ddl.column_family +order by rocksdb_ddl.cf; +cf transaction_id key +lock_info_cf2 _txn_id_ _key_ +lock_info_cf2 _txn_id_ _key_ +lock_into_cf1 _txn_id_ _key_ +lock_into_cf1 _txn_id_ _key_ +use test; +DROP TABLE t1; +DROP TABLE t2; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/locking_issues.result b/storage/rocksdb/mysql-test/rocksdb/r/locking_issues.result index 4b237dcb7aa..6df4d44f72b 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/locking_issues.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/locking_issues.result @@ -229,7 +229,7 @@ SET SESSION TRANSACTION ISOLATION LEVEL REPEATABLE READ; SELECT * FROM t0 WHERE value > 0 FOR UPDATE; SET SESSION TRANSACTION ISOLATION LEVEL REPEATABLE READ; UPDATE t0 SET VALUE=VALUE+1 WHERE id=190000; -ERROR 40001: Deadlock found when trying to get lock; try restarting transaction +ERROR: 1213 DROP TABLE t0; ----------------------------------------------------------------------- @@ -244,7 +244,9 @@ SET SESSION TRANSACTION ISOLATION LEVEL READ COMMITTED; SELECT * FROM t0 WHERE value > 0 FOR UPDATE; SET SESSION TRANSACTION ISOLATION LEVEL READ COMMITTED; UPDATE t0 SET VALUE=VALUE+1 WHERE id=190000; -ERROR 40001: Deadlock found when trying to get lock; try restarting transaction +id value +190000 1 +ERROR: 0 DROP TABLE t0; ----------------------------------------------------------------------- @@ -293,7 +295,7 @@ SET SESSION TRANSACTION ISOLATION LEVEL REPEATABLE READ; BEGIN; DELETE FROM t0 WHERE id=190000; COMMIT; -ERROR 40001: Deadlock found when trying to get lock; try restarting transaction +ERROR: 1213 COMMIT; DROP TABLE t0; @@ -313,7 +315,8 @@ SET SESSION TRANSACTION ISOLATION LEVEL READ COMMITTED; BEGIN; DELETE FROM t0 WHERE id=190000; COMMIT; -ERROR 40001: Deadlock found when trying to get lock; try restarting transaction +id value +ERROR: 0 COMMIT; DROP TABLE t0; @@ -333,7 +336,7 @@ SET SESSION TRANSACTION ISOLATION LEVEL REPEATABLE READ; BEGIN; UPDATE t0 SET id=200001 WHERE id=190000; COMMIT; -ERROR 40001: Deadlock found when trying to get lock; try restarting transaction +ERROR: 1213 COMMIT; DROP TABLE t0; @@ -353,7 +356,8 @@ SET SESSION TRANSACTION ISOLATION LEVEL READ COMMITTED; BEGIN; UPDATE t0 SET id=200001 WHERE id=190000; COMMIT; -ERROR 40001: Deadlock found when trying to get lock; try restarting transaction +id value +ERROR: 0 COMMIT; DROP TABLE t0; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/optimizer_loose_index_scans.result b/storage/rocksdb/mysql-test/rocksdb/r/optimizer_loose_index_scans.result new file mode 100644 index 00000000000..27b1779627b --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/r/optimizer_loose_index_scans.result @@ -0,0 +1,281 @@ +set optimizer_switch='index_merge_sort_union=off'; +create table t (a int, b int, c int, d int, e int, primary key(a, b, c, d), key(b, d)) engine=rocksdb; +analyze table t; +Table Op Msg_type Msg_text +test.t analyze status OK +show indexes from t; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +t 0 PRIMARY 1 a A 100 NULL NULL LSMTREE +t 0 PRIMARY 2 b A 500 NULL NULL LSMTREE +t 0 PRIMARY 3 c A 2500 NULL NULL LSMTREE +t 0 PRIMARY 4 d A 2500 NULL NULL LSMTREE +t 1 b 1 b A 50 NULL NULL LSMTREE +t 1 b 2 d A 500 NULL NULL LSMTREE +set optimizer_switch = 'skip_scan=off'; +explain select b, d from t where d < 2; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t index NULL b 8 NULL # Using where; Using index +rows_read +2500 +set optimizer_switch = 'skip_scan=on,skip_scan_cost_based=off'; +explain select b, d from t where d < 2; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t range PRIMARY,b b 8 NULL # Using where; Using index for skip scan +rows_read +260 +include/diff_tables.inc [temp_orig, temp_skip] +set optimizer_switch = 'skip_scan=off,skip_scan_cost_based=on'; +set optimizer_switch = 'skip_scan=off'; +explain select b, d from t where d > 4; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t index NULL b 8 NULL # Using where; Using index +rows_read +2500 +set optimizer_switch = 'skip_scan=on,skip_scan_cost_based=off'; +explain select b, d from t where d > 4; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t range PRIMARY,b b 8 NULL # Using where; Using index for skip scan +rows_read +1509 +include/diff_tables.inc [temp_orig, temp_skip] +set optimizer_switch = 'skip_scan=off,skip_scan_cost_based=on'; +set optimizer_switch = 'skip_scan=off'; +explain select a, b, c, d from t where a = 5 and d <= 3; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t ref PRIMARY PRIMARY 4 const # Using where; Using index +rows_read +251 +set optimizer_switch = 'skip_scan=on,skip_scan_cost_based=off'; +explain select a, b, c, d from t where a = 5 and d <= 3; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t range PRIMARY,b PRIMARY 16 NULL # Using where; Using index for skip scan +rows_read +126 +include/diff_tables.inc [temp_orig, temp_skip] +set optimizer_switch = 'skip_scan=off,skip_scan_cost_based=on'; +set optimizer_switch = 'skip_scan=off'; +explain select e from t where a = 5 and d <= 3; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t ref PRIMARY PRIMARY 4 const # Using where +rows_read +251 +set optimizer_switch = 'skip_scan=on,skip_scan_cost_based=off'; +explain select e from t where a = 5 and d <= 3; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t ref PRIMARY,b PRIMARY 4 const # Using where +rows_read +251 +include/diff_tables.inc [temp_orig, temp_skip] +set optimizer_switch = 'skip_scan=off,skip_scan_cost_based=on'; +set optimizer_switch = 'skip_scan=off'; +explain select a, b, c, d from t where a = 5 and d >= 98; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t ref PRIMARY PRIMARY 4 const # Using where; Using index +rows_read +251 +set optimizer_switch = 'skip_scan=on,skip_scan_cost_based=off'; +explain select a, b, c, d from t where a = 5 and d >= 98; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t range PRIMARY,b PRIMARY 16 NULL # Using where; Using index for skip scan +rows_read +51 +include/diff_tables.inc [temp_orig, temp_skip] +set optimizer_switch = 'skip_scan=off,skip_scan_cost_based=on'; +set optimizer_switch = 'skip_scan=off'; +explain select e from t where a = 5 and d >= 98; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t ref PRIMARY PRIMARY 4 const # Using where +rows_read +251 +set optimizer_switch = 'skip_scan=on,skip_scan_cost_based=off'; +explain select e from t where a = 5 and d >= 98; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t ref PRIMARY,b PRIMARY 4 const # Using where +rows_read +251 +include/diff_tables.inc [temp_orig, temp_skip] +set optimizer_switch = 'skip_scan=off,skip_scan_cost_based=on'; +set optimizer_switch = 'skip_scan=off'; +explain select a, b, c, d from t where a in (1, 5) and d >= 98; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t range PRIMARY PRIMARY 4 NULL # Using where; Using index +rows_read +502 +set optimizer_switch = 'skip_scan=on,skip_scan_cost_based=off'; +explain select a, b, c, d from t where a in (1, 5) and d >= 98; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t range PRIMARY,b PRIMARY 16 NULL # Using where; Using index for skip scan +rows_read +102 +include/diff_tables.inc [temp_orig, temp_skip] +set optimizer_switch = 'skip_scan=off,skip_scan_cost_based=on'; +set optimizer_switch = 'skip_scan=off'; +explain select a, b, c, d from t where a in (1, 3, 5) and d >= 98; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t range PRIMARY PRIMARY 4 NULL # Using where; Using index +rows_read +753 +set optimizer_switch = 'skip_scan=on,skip_scan_cost_based=off'; +explain select a, b, c, d from t where a in (1, 3, 5) and d >= 98; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t range PRIMARY,b PRIMARY 16 NULL # Using where; Using index for skip scan +rows_read +153 +include/diff_tables.inc [temp_orig, temp_skip] +set optimizer_switch = 'skip_scan=off,skip_scan_cost_based=on'; +set optimizer_switch = 'skip_scan=off'; +explain select a, b, c, d from t where a in (1, 5) and b in (1, 2) and d >= 98; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t range PRIMARY,b PRIMARY 8 NULL # Using where; Using index +rows_read +204 +set optimizer_switch = 'skip_scan=on,skip_scan_cost_based=off'; +explain select a, b, c, d from t where a in (1, 5) and b in (1, 2) and d >= 98; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t range PRIMARY,b PRIMARY 16 NULL # Using where; Using index for skip scan +rows_read +44 +include/diff_tables.inc [temp_orig, temp_skip] +set optimizer_switch = 'skip_scan=off,skip_scan_cost_based=on'; +set optimizer_switch = 'skip_scan=off'; +explain select a, b, c, d from t where a in (1, 2, 3, 4, 5) and b in (1, 2, 3) and d >= 98; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t range PRIMARY,b PRIMARY 8 NULL # Using where; Using index +rows_read +765 +set optimizer_switch = 'skip_scan=on,skip_scan_cost_based=off'; +explain select a, b, c, d from t where a in (1, 2, 3, 4, 5) and b in (1, 2, 3) and d >= 98; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t range PRIMARY,b PRIMARY 16 NULL # Using where; Using index for skip scan +rows_read +165 +include/diff_tables.inc [temp_orig, temp_skip] +set optimizer_switch = 'skip_scan=off,skip_scan_cost_based=on'; +set optimizer_switch = 'skip_scan=off'; +explain select a, b, c, d from t where a = 5 and b = 2 and d >= 98; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t ref PRIMARY,b PRIMARY 8 const,const # Using where; Using index +rows_read +51 +set optimizer_switch = 'skip_scan=on,skip_scan_cost_based=off'; +explain select a, b, c, d from t where a = 5 and b = 2 and d >= 98; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t range PRIMARY,b PRIMARY 16 NULL # Using where; Using index for skip scan +rows_read +11 +include/diff_tables.inc [temp_orig, temp_skip] +set optimizer_switch = 'skip_scan=off,skip_scan_cost_based=on'; +set optimizer_switch = 'skip_scan=off'; +explain select a+1, b, c, d from t where a = 5 and d < 3; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t ref PRIMARY PRIMARY 4 const # Using where; Using index +rows_read +251 +set optimizer_switch = 'skip_scan=on,skip_scan_cost_based=off'; +explain select a+1, b, c, d from t where a = 5 and d < 3; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t range PRIMARY,b PRIMARY 16 NULL # Using where; Using index for skip scan +rows_read +101 +include/diff_tables.inc [temp_orig, temp_skip] +set optimizer_switch = 'skip_scan=off,skip_scan_cost_based=on'; +set optimizer_switch = 'skip_scan=off'; +explain select b, c, d from t where a = 5 and d < 3; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t ref PRIMARY PRIMARY 4 const # Using where; Using index +rows_read +251 +set optimizer_switch = 'skip_scan=on,skip_scan_cost_based=off'; +explain select b, c, d from t where a = 5 and d < 3; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t range PRIMARY,b PRIMARY 16 NULL # Using where; Using index for skip scan +rows_read +101 +include/diff_tables.inc [temp_orig, temp_skip] +set optimizer_switch = 'skip_scan=off,skip_scan_cost_based=on'; +set optimizer_switch = 'skip_scan=off'; +explain select a, b, c, d from t where a = b and d >= 98; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t index NULL b 8 NULL # Using where; Using index +rows_read +2500 +set optimizer_switch = 'skip_scan=on,skip_scan_cost_based=off'; +explain select a, b, c, d from t where a = b and d >= 98; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t range PRIMARY,b b 8 NULL # Using where; Using index for skip scan +rows_read +9 +include/diff_tables.inc [temp_orig, temp_skip] +set optimizer_switch = 'skip_scan=off,skip_scan_cost_based=on'; +set optimizer_switch = 'skip_scan=on'; +set optimizer_trace = 'enabled=on'; +explain select a, b, c, d from t where a = 5 and d < 3 order by b, c, d; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t ref PRIMARY,b PRIMARY 4 const # Using where; Using index +select count(*) from information_schema.optimizer_trace where trace like '%order_attribute_not_prefix_in_index%'; +count(*) +1 +explain select a, b, c, d from t where a = 2 and d >= 98 and e = 2; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t ref PRIMARY PRIMARY 4 const # Using where +select count(*) from information_schema.optimizer_trace where trace like '%query_references_nonkey_column%'; +count(*) +1 +explain select a, b, c, d from t where a = 5 or b = 2 and d >= 98; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t index PRIMARY,b b 8 NULL # Using where; Using index +select count(*) from information_schema.optimizer_trace where trace like '%no_range_tree%'; +count(*) +1 +explain select a, b, c, d from t where a = 5 or b = 2 or d >= 98; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t index PRIMARY,b b 8 NULL # Using where; Using index +select count(*) from information_schema.optimizer_trace where trace like '%no_range_tree%'; +count(*) +1 +explain select a, b, c, d from t where a = 5 or d >= 98; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t index PRIMARY,b b 8 NULL # Using where; Using index +select count(*) from information_schema.optimizer_trace where trace like '%no_range_tree%'; +count(*) +1 +explain select a, b, c, d from t where ((a = 5 and b = 2) or a = 2) and d >= 98; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t range PRIMARY,b b 8 NULL # Using where; Using index for skip scan +select count(*) from information_schema.optimizer_trace where trace like '%keypart_in_disjunctive_query%'; +count(*) +1 +explain select a, b, c, d from t where a > 2 and d >= 98; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t range PRIMARY,b PRIMARY 4 NULL # Using where; Using index +select count(*) from information_schema.optimizer_trace where trace like '%prefix_not_const_equality%'; +count(*) +1 +explain select a, b, c, d from t where a = 2 and (d >= 98 or d < 2); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t ref PRIMARY,b PRIMARY 4 const # Using where; Using index +select count(*) from information_schema.optimizer_trace where trace like '%range_predicate_too_complex%'; +count(*) +1 +explain select a, b, c, d from t where a = 2 and b = 2; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t ref PRIMARY,b PRIMARY 8 const,const # Using index +select count(*) from information_schema.optimizer_trace where trace like '%no_range_predicate%'; +count(*) +1 +explain select a, b, c, d from t where a = 2 and c > 2 and d < 2; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t ref PRIMARY,b PRIMARY 4 const # Using where; Using index +select count(*) from information_schema.optimizer_trace where trace like '%too_many_range_predicates%'; +count(*) +1 +explain select a, b, c, d from t where (a < 1 or a = 4 or a = 5) and b in (1, 2, 3) and d >= 98; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t range PRIMARY,b PRIMARY 8 NULL # Using where; Using index +select count(*) from information_schema.optimizer_trace where trace like '%prefix_not_const_equality%'; +count(*) +1 +set optimizer_trace = 'enabled=off'; +set optimizer_switch= 'skip_scan=off'; +drop table t; +set optimizer_switch='index_merge_sort_union=on'; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/rocksdb.result b/storage/rocksdb/mysql-test/rocksdb/r/rocksdb.result index 55388c65b99..b6a17d90221 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/rocksdb.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/rocksdb.result @@ -862,10 +862,9 @@ rocksdb_advise_random_on_open ON rocksdb_allow_concurrent_memtable_write OFF rocksdb_allow_mmap_reads OFF rocksdb_allow_mmap_writes OFF -rocksdb_allow_os_buffer ON rocksdb_background_sync OFF rocksdb_base_background_compactions 1 -rocksdb_block_cache_size 8388608 +rocksdb_block_cache_size 536870912 rocksdb_block_restart_interval 16 rocksdb_block_size 4096 rocksdb_block_size_deviation 10 @@ -887,6 +886,7 @@ rocksdb_create_if_missing ON rocksdb_create_missing_column_families OFF rocksdb_datadir ./.rocksdb rocksdb_db_write_buffer_size 0 +rocksdb_deadlock_detect OFF rocksdb_debug_optimizer_no_zero_cardinality ON rocksdb_default_cf_options rocksdb_delete_obsolete_files_period_micros 21600000000 @@ -925,6 +925,7 @@ rocksdb_paranoid_checks ON rocksdb_pause_background_work ON rocksdb_perf_context_level 0 rocksdb_pin_l0_filter_and_index_blocks_in_cache ON +rocksdb_print_snapshot_conflict_queries OFF rocksdb_rate_limiter_bytes_per_sec 0 rocksdb_read_free_rpl_tables rocksdb_records_in_range 50 @@ -936,16 +937,20 @@ rocksdb_skip_fill_cache OFF rocksdb_skip_unique_check OFF rocksdb_skip_unique_check_tables .* rocksdb_stats_dump_period_sec 600 -rocksdb_store_checksums OFF +rocksdb_store_row_debug_checksums OFF rocksdb_strict_collation_check OFF rocksdb_strict_collation_exceptions rocksdb_table_cache_numshardbits 6 rocksdb_table_stats_sampling_pct 10 +rocksdb_tmpdir +rocksdb_trace_sst_api OFF rocksdb_unsafe_for_binlog OFF rocksdb_use_adaptive_mutex OFF +rocksdb_use_direct_reads OFF +rocksdb_use_direct_writes OFF rocksdb_use_fsync OFF rocksdb_validate_tables 1 -rocksdb_verify_checksums OFF +rocksdb_verify_row_debug_checksums OFF rocksdb_wal_bytes_per_sync 0 rocksdb_wal_dir rocksdb_wal_recovery_mode 2 @@ -985,7 +990,7 @@ insert into t49 values (1,10),(2,20); begin; update t49 set a = 100 where pk = 1; connect con1,localhost,root,,; -set rocksdb_lock_wait_timeout=5000; +set rocksdb_lock_wait_timeout=60; set @var1= to_seconds(now()); update t49 set a = 1000 where pk = 1; connect con2,localhost,root,,; @@ -993,9 +998,7 @@ kill query $con1_id; connection con1; ERROR 70100: Query execution was interrupted set @var2= to_seconds(now()); -"[Jay Edgar] I've updated this query to help determine why it is sometimes failing" -"(t13541934). If you get an error here (i.e. not 'passed') notify me." -select if ((@var2 - @var1) < 1000, "passed", (@var2 - @var1)) as 'result'; +select if ((@var2 - @var1) < 60, "passed", (@var2 - @var1)) as 'result'; result passed connection default; @@ -1297,7 +1300,7 @@ insert into t1 select (@a:=@a+1), 1234 from information_schema.session_variables set @tmp1= @@rocksdb_max_row_locks; set rocksdb_max_row_locks= 20; update t1 set a=a+10; -ERROR HY000: Internal error: Operation aborted: Number of locks held by the transaction exceeded @@rocksdb_max_row_locks +ERROR HY000: Got error 196 'Number of locks held reached @@rocksdb_max_row_locks.' from ROCKSDB DROP TABLE t1; # # Test AUTO_INCREMENT behavior problem, @@ -1465,9 +1468,9 @@ rocksdb_number_superversion_acquires # rocksdb_number_superversion_cleanups # rocksdb_number_superversion_releases # rocksdb_rate_limit_delay_millis # -rocksdb_sequence_number # rocksdb_snapshot_conflict_errors # rocksdb_wal_bytes # +rocksdb_wal_group_syncs # rocksdb_wal_synced # rocksdb_write_other # rocksdb_write_self # @@ -1537,9 +1540,9 @@ ROCKSDB_NUMBER_SUPERVERSION_ACQUIRES ROCKSDB_NUMBER_SUPERVERSION_CLEANUPS ROCKSDB_NUMBER_SUPERVERSION_RELEASES ROCKSDB_RATE_LIMIT_DELAY_MILLIS -ROCKSDB_SEQUENCE_NUMBER ROCKSDB_SNAPSHOT_CONFLICT_ERRORS ROCKSDB_WAL_BYTES +ROCKSDB_WAL_GROUP_SYNCS ROCKSDB_WAL_SYNCED ROCKSDB_WRITE_OTHER ROCKSDB_WRITE_SELF @@ -1611,9 +1614,9 @@ ROCKSDB_NUMBER_SUPERVERSION_ACQUIRES ROCKSDB_NUMBER_SUPERVERSION_CLEANUPS ROCKSDB_NUMBER_SUPERVERSION_RELEASES ROCKSDB_RATE_LIMIT_DELAY_MILLIS -ROCKSDB_SEQUENCE_NUMBER ROCKSDB_SNAPSHOT_CONFLICT_ERRORS ROCKSDB_WAL_BYTES +ROCKSDB_WAL_GROUP_SYNCS ROCKSDB_WAL_SYNCED ROCKSDB_WRITE_OTHER ROCKSDB_WRITE_SELF @@ -2453,4 +2456,24 @@ a 10 11 DROP TABLE t1; +# +# Issue #411: Setting rocksdb_commit_in_the_middle commits transaction +# without releasing iterator +# +CREATE TABLE t1 (id1 bigint(20), +id2 bigint(20), +id3 bigint(20), +PRIMARY KEY (id1, id2, id3)) +DEFAULT CHARSET=latin1; +CREATE TABLE t2 (id1 bigint(20), +id2 bigint(20), +PRIMARY KEY (id1, id2)) +DEFAULT CHARSET=latin1; +set rocksdb_commit_in_the_middle=1; +SET @save_rocksdb_bulk_load_size= @@rocksdb_bulk_load_size; +set rocksdb_bulk_load_size = 100; +DELETE t2, t1 FROM t2 LEFT JOIN t1 ON t2.id2 = t1.id2 AND t2.id1 = t1.id1 WHERE t2.id1 = 0; +SET rocksdb_bulk_load_size= @save_rocksdb_bulk_load_size; +SET rocksdb_commit_in_the_middle=0; +DROP TABLE t1, t2; SET GLOBAL ROCKSDB_PAUSE_BACKGROUND_WORK = @ORIG_PAUSE_BACKGROUND_WORK; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_cf_options.result b/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_cf_options.result index 09d251ccbe6..6c3d85b760c 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_cf_options.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_cf_options.result @@ -17,19 +17,19 @@ where option_type in ('WRITE_BUFFER_SIZE', 'MAX_BYTES_FOR_LEVEL_MULTIPLIER') order by cf_name, option_type; cf_name option_type value -cf1 MAX_BYTES_FOR_LEVEL_MULTIPLIER 10 +cf1 MAX_BYTES_FOR_LEVEL_MULTIPLIER 10.000000 cf1 TARGET_FILE_SIZE_BASE 1048576 cf1 WRITE_BUFFER_SIZE 12582912 -cf2 MAX_BYTES_FOR_LEVEL_MULTIPLIER 10 +cf2 MAX_BYTES_FOR_LEVEL_MULTIPLIER 10.000000 cf2 TARGET_FILE_SIZE_BASE 1048576 cf2 WRITE_BUFFER_SIZE 12582912 -default MAX_BYTES_FOR_LEVEL_MULTIPLIER 10 +default MAX_BYTES_FOR_LEVEL_MULTIPLIER 10.000000 default TARGET_FILE_SIZE_BASE 1048576 default WRITE_BUFFER_SIZE 12582912 -z MAX_BYTES_FOR_LEVEL_MULTIPLIER 10 +z MAX_BYTES_FOR_LEVEL_MULTIPLIER 10.000000 z TARGET_FILE_SIZE_BASE 1048576 z WRITE_BUFFER_SIZE 12582912 -__system__ MAX_BYTES_FOR_LEVEL_MULTIPLIER 10 +__system__ MAX_BYTES_FOR_LEVEL_MULTIPLIER 10.000000 __system__ TARGET_FILE_SIZE_BASE 1048576 __system__ WRITE_BUFFER_SIZE 12582912 @@ -42,19 +42,19 @@ where option_type in ('WRITE_BUFFER_SIZE', 'MAX_BYTES_FOR_LEVEL_MULTIPLIER') order by cf_name, option_type; cf_name option_type value -cf1 MAX_BYTES_FOR_LEVEL_MULTIPLIER 10 +cf1 MAX_BYTES_FOR_LEVEL_MULTIPLIER 10.000000 cf1 TARGET_FILE_SIZE_BASE 2097152 cf1 WRITE_BUFFER_SIZE 8388608 -cf2 MAX_BYTES_FOR_LEVEL_MULTIPLIER 8 +cf2 MAX_BYTES_FOR_LEVEL_MULTIPLIER 8.000000 cf2 TARGET_FILE_SIZE_BASE 1048576 cf2 WRITE_BUFFER_SIZE 16777216 -default MAX_BYTES_FOR_LEVEL_MULTIPLIER 10 +default MAX_BYTES_FOR_LEVEL_MULTIPLIER 10.000000 default TARGET_FILE_SIZE_BASE 1048576 default WRITE_BUFFER_SIZE 12582912 -z MAX_BYTES_FOR_LEVEL_MULTIPLIER 10 +z MAX_BYTES_FOR_LEVEL_MULTIPLIER 10.000000 z TARGET_FILE_SIZE_BASE 4194304 z WRITE_BUFFER_SIZE 12582912 -__system__ MAX_BYTES_FOR_LEVEL_MULTIPLIER 10 +__system__ MAX_BYTES_FOR_LEVEL_MULTIPLIER 10.000000 __system__ TARGET_FILE_SIZE_BASE 1048576 __system__ WRITE_BUFFER_SIZE 12582912 diff --git a/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_checksums.result b/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_checksums.result index a8908edada5..87243c05e2a 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_checksums.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_checksums.result @@ -1,12 +1,12 @@ -set @save_rocksdb_store_checksums=@@global.rocksdb_store_checksums; -set @save_rocksdb_verify_checksums=@@global.rocksdb_verify_checksums; +set @save_rocksdb_store_row_debug_checksums=@@global.rocksdb_store_row_debug_checksums; +set @save_rocksdb_verify_row_debug_checksums=@@global.rocksdb_verify_row_debug_checksums; set @save_rocksdb_checksums_pct=@@global.rocksdb_checksums_pct; drop table if exists t1,t2,t3; show variables like 'rocksdb_%checksum%'; Variable_name Value rocksdb_checksums_pct 100 -rocksdb_store_checksums OFF -rocksdb_verify_checksums OFF +rocksdb_store_row_debug_checksums OFF +rocksdb_verify_row_debug_checksums OFF create table t1 (pk int primary key, a int, b int, key(a), key(b)) engine=rocksdb; insert into t1 values (1,1,1),(2,2,2),(3,3,3); check table t1; @@ -19,7 +19,7 @@ test.t1 check status OK CHECKTABLE t1: ... 3 index entries checked (0 had checksums) CHECKTABLE t1: 0 table records had checksums drop table t1; -set session rocksdb_store_checksums=on; +set session rocksdb_store_row_debug_checksums=on; create table t2 (pk int primary key, a int, b int, key(a), key(b)) engine=rocksdb; insert into t2 values (1,1,1),(2,2,2),(3,3,3); check table t2; @@ -34,9 +34,9 @@ test.t2 check status OK # Now, make a table that has both rows with checksums and without create table t3 (pk int primary key, a int, b int, key(a), key(b)) engine=rocksdb; insert into t3 values (1,1,1),(2,2,2),(3,3,3); -set session rocksdb_store_checksums=off; +set session rocksdb_store_row_debug_checksums=off; update t3 set b=3 where a=2; -set session rocksdb_store_checksums=on; +set session rocksdb_store_row_debug_checksums=on; check table t3; Table Op Msg_type Msg_text test.t3 check status OK @@ -46,7 +46,7 @@ test.t3 check status OK CHECKTABLE t3: Checking index b CHECKTABLE t3: ... 3 index entries checked (2 had checksums) CHECKTABLE t3: 2 table records had checksums -set session rocksdb_store_checksums=on; +set session rocksdb_store_row_debug_checksums=on; set session rocksdb_checksums_pct=5; create table t4 (pk int primary key, a int, b int, key(a), key(b)) engine=rocksdb; check table t4; @@ -65,13 +65,13 @@ insert into mtr.test_suppressions values ('Data with incorrect checksum'); # 1. Start with mismatch in key checksum of the PK. set session debug= "+d,myrocks_simulate_bad_pk_checksum1"; -set session rocksdb_verify_checksums=off; +set session rocksdb_verify_row_debug_checksums=off; select * from t3; pk a b 1 1 1 2 2 3 3 3 3 -set session rocksdb_verify_checksums=on; +set session rocksdb_verify_row_debug_checksums=on; select * from t3; ERROR HY000: Internal error: Record checksum mismatch select * from t4; @@ -79,13 +79,13 @@ ERROR HY000: Internal error: Record checksum mismatch set session debug= "-d,myrocks_simulate_bad_pk_checksum1"; # 2. Continue with mismatch in pk value checksum. set session debug= "+d,myrocks_simulate_bad_pk_checksum2"; -set session rocksdb_verify_checksums=off; +set session rocksdb_verify_row_debug_checksums=off; select * from t3; pk a b 1 1 1 2 2 3 3 3 3 -set session rocksdb_verify_checksums=on; +set session rocksdb_verify_row_debug_checksums=on; select * from t3; ERROR HY000: Internal error: Record checksum mismatch select * from t4; @@ -123,7 +123,7 @@ ERROR HY000: Internal error: Record checksum mismatch select a from t4 force index(a) where a<1000000; ERROR HY000: Internal error: Record checksum mismatch set session debug= "-d,myrocks_simulate_bad_key_checksum1"; -set @@global.rocksdb_store_checksums=@save_rocksdb_store_checksums; -set @@global.rocksdb_verify_checksums=@save_rocksdb_verify_checksums; +set @@global.rocksdb_store_row_debug_checksums=@save_rocksdb_store_row_debug_checksums; +set @@global.rocksdb_verify_row_debug_checksums=@save_rocksdb_verify_row_debug_checksums; set @@global.rocksdb_checksums_pct=@save_rocksdb_checksums_pct; drop table t2,t3,t4; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_deadlock_detect_rc.result b/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_deadlock_detect_rc.result new file mode 100644 index 00000000000..043750f94f6 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_deadlock_detect_rc.result @@ -0,0 +1,54 @@ +set @prior_rocksdb_lock_wait_timeout = @@rocksdb_lock_wait_timeout; +set @prior_rocksdb_deadlock_detect = @@rocksdb_deadlock_detect; +set global rocksdb_lock_wait_timeout = 100000; +set global rocksdb_deadlock_detect = ON; +create table t (i int primary key); +create table r1 (id int primary key, value int); +insert into r1 values (1,1),(2,2),(3,3),(4,4),(5,5),(6,6),(7,7),(8,8),(9,9),(10,10); +create table r2 like r1; +insert into r2 select * from r1; +begin; +update r2 set value=100 where id=9; +begin; +update r1 set value=100 where id=8; +select * from r2 for update;; +select * from r1 for update; +ERROR 40001: Deadlock found when trying to get lock; try restarting transaction +rollback; +id value +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +7 7 +8 8 +9 9 +10 10 +rollback; +begin; +insert into t values (1); +begin; +insert into t values (2); +begin; +insert into t values (3); +select * from t where i = 2 for update; +select * from t where i = 3 for update; +select * from t; +i +3 +insert into t values (4), (1); +ERROR 40001: Deadlock found when trying to get lock; try restarting transaction +# Statement should be rolled back +select * from t; +i +3 +rollback; +i +rollback; +i +rollback; +set global rocksdb_lock_wait_timeout = @prior_rocksdb_lock_wait_timeout; +set global rocksdb_deadlock_detect = @prior_rocksdb_deadlock_detect; +drop table t,r1,r2; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_deadlock_detect_rr.result b/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_deadlock_detect_rr.result new file mode 100644 index 00000000000..043750f94f6 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_deadlock_detect_rr.result @@ -0,0 +1,54 @@ +set @prior_rocksdb_lock_wait_timeout = @@rocksdb_lock_wait_timeout; +set @prior_rocksdb_deadlock_detect = @@rocksdb_deadlock_detect; +set global rocksdb_lock_wait_timeout = 100000; +set global rocksdb_deadlock_detect = ON; +create table t (i int primary key); +create table r1 (id int primary key, value int); +insert into r1 values (1,1),(2,2),(3,3),(4,4),(5,5),(6,6),(7,7),(8,8),(9,9),(10,10); +create table r2 like r1; +insert into r2 select * from r1; +begin; +update r2 set value=100 where id=9; +begin; +update r1 set value=100 where id=8; +select * from r2 for update;; +select * from r1 for update; +ERROR 40001: Deadlock found when trying to get lock; try restarting transaction +rollback; +id value +1 1 +2 2 +3 3 +4 4 +5 5 +6 6 +7 7 +8 8 +9 9 +10 10 +rollback; +begin; +insert into t values (1); +begin; +insert into t values (2); +begin; +insert into t values (3); +select * from t where i = 2 for update; +select * from t where i = 3 for update; +select * from t; +i +3 +insert into t values (4), (1); +ERROR 40001: Deadlock found when trying to get lock; try restarting transaction +# Statement should be rolled back +select * from t; +i +3 +rollback; +i +rollback; +i +rollback; +set global rocksdb_lock_wait_timeout = @prior_rocksdb_lock_wait_timeout; +set global rocksdb_deadlock_detect = @prior_rocksdb_deadlock_detect; +drop table t,r1,r2; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_deadlock_stress_rc.result b/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_deadlock_stress_rc.result new file mode 100644 index 00000000000..f97da0099fe --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_deadlock_stress_rc.result @@ -0,0 +1,8 @@ +create table t1 (a int primary key, b int) engine=rocksdb; +set @prior_rocksdb_lock_wait_timeout = @@rocksdb_lock_wait_timeout; +set @prior_rocksdb_deadlock_detect = @@rocksdb_deadlock_detect; +set global rocksdb_lock_wait_timeout = 100000; +set global rocksdb_deadlock_detect = ON; +set global rocksdb_lock_wait_timeout = @prior_rocksdb_lock_wait_timeout; +set global rocksdb_deadlock_detect = @prior_rocksdb_deadlock_detect; +drop table t1; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_deadlock_stress_rr.result b/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_deadlock_stress_rr.result new file mode 100644 index 00000000000..f97da0099fe --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_deadlock_stress_rr.result @@ -0,0 +1,8 @@ +create table t1 (a int primary key, b int) engine=rocksdb; +set @prior_rocksdb_lock_wait_timeout = @@rocksdb_lock_wait_timeout; +set @prior_rocksdb_deadlock_detect = @@rocksdb_deadlock_detect; +set global rocksdb_lock_wait_timeout = 100000; +set global rocksdb_deadlock_detect = ON; +set global rocksdb_lock_wait_timeout = @prior_rocksdb_lock_wait_timeout; +set global rocksdb_deadlock_detect = @prior_rocksdb_deadlock_detect; +drop table t1; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_locks.result b/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_locks.result index e4d11960e6e..a39fa0a429e 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_locks.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_locks.result @@ -7,6 +7,7 @@ pk 1 connect con1,localhost,root,,; connection con1; +call mtr.add_suppression("Got snapshot conflict errors"); ### Connection con1 set @@rocksdb_lock_wait_timeout=500; set autocommit=0; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/rollback_savepoint.result b/storage/rocksdb/mysql-test/rocksdb/r/rollback_savepoint.result new file mode 100644 index 00000000000..18c014b9b07 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/r/rollback_savepoint.result @@ -0,0 +1,22 @@ +DROP TABLE IF EXISTS t1, t2; +CREATE TABLE t1 (a INT, b CHAR(8), pk INT AUTO_INCREMENT PRIMARY KEY) ENGINE=rocksdb; +INSERT INTO t1 (a,b) VALUES (1,'a'),(2,'b'),(3,'a'); +CREATE TABLE t2 LIKE t1; +INSERT INTO t2 SELECT * FROM t1; +START TRANSACTION WITH CONSISTENT SNAPSHOT; +SAVEPOINT a; +SELECT * FROM t1 ORDER BY pk; +a b pk +1 a 1 +2 b 2 +3 a 3 +ROLLBACK TO SAVEPOINT a; +SAVEPOINT a; +SELECT * FROM t2 ORDER BY pk; +a b pk +1 a 1 +2 b 2 +3 a 3 +ROLLBACK TO SAVEPOINT a; +ALTER TABLE t1 RENAME TO t3; +DROP TABLE t2, t3; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/select_lock_in_share_mode.result b/storage/rocksdb/mysql-test/rocksdb/r/select_lock_in_share_mode.result index e6433dcbeef..286ab7178cc 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/select_lock_in_share_mode.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/select_lock_in_share_mode.result @@ -13,11 +13,10 @@ SELECT a,b FROM t1 WHERE b='a'; a b 1 a 3 a -# -# Currently, SELECT ... LOCK IN SHARE MODE works like -# SELECT FOR UPDATE SELECT a,b FROM t1 WHERE b='a' LOCK IN SHARE MODE; -ERROR HY000: Lock wait timeout exceeded; try restarting transaction: Timeout on index: test.t1.PRIMARY +a b +1 a +3 a UPDATE t1 SET b='c' WHERE b='a'; ERROR HY000: Lock wait timeout exceeded; try restarting transaction: Timeout on index: test.t1.PRIMARY connection con1; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/show_engine.result b/storage/rocksdb/mysql-test/rocksdb/r/show_engine.result index 69b927ba5a8..7f1584938d6 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/show_engine.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/show_engine.result @@ -407,7 +407,8 @@ SNAPSHOTS --------- LIST OF SNAPSHOTS FOR EACH SESSION: ---SNAPSHOT, ACTIVE NUM sec -MySQL thread id TID, OS thread handle PTR +MySQL thread id TID, OS thread handle PTR, query id QID localhost root ACTION +SHOW ENGINE rocksdb TRANSACTION STATUS lock count 0, write count 0 ----------------------------------------- END OF ROCKSDB TRANSACTION MONITOR OUTPUT diff --git a/storage/rocksdb/mysql-test/rocksdb/r/statistics.result b/storage/rocksdb/mysql-test/rocksdb/r/statistics.result index 1798563f328..78344991360 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/statistics.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/statistics.result @@ -21,8 +21,8 @@ index t3_1(b) comment 'rev:cf_t4' ) engine=rocksdb; SELECT table_name, table_rows FROM information_schema.tables WHERE table_schema = DATABASE() and table_name <> 't1'; table_name table_rows -t2 4999 -t3 4999 +t2 1000 +t3 1000 SELECT CASE WHEN table_rows < 100000 then 'true' else 'false' end from information_schema.tables where table_name = 't1'; CASE WHEN table_rows < 100000 then 'true' else 'false' end true diff --git a/storage/rocksdb/mysql-test/rocksdb/r/tmpdir.result b/storage/rocksdb/mysql-test/rocksdb/r/tmpdir.result new file mode 100644 index 00000000000..e07d750c413 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/r/tmpdir.result @@ -0,0 +1,26 @@ +# If rocksdb_tmpdir is NULL or "", temporary file will be created in +# server configuration variable location(--tmpdir) +CREATE TABLE t1 (a INT PRIMARY KEY, b INT) ENGINE=RocksDB; +show session variables like 'rocksdb_tmpdir'; +Variable_name Value +rocksdb_tmpdir +# Connection con1 +show session variables like 'rocksdb_tmpdir'; +Variable_name Value +rocksdb_tmpdir +ALTER TABLE t1 ADD INDEX kb(b), ALGORITHM=INPLACE; +DROP TABLE t1; +# rocksdb_tmpdir with valid location. +CREATE TABLE t1 (a INT PRIMARY KEY, b INT) ENGINE=RocksDB; +set @tmpdir = @@global.tmpdir; +set global rocksdb_tmpdir = @tmpdir; +show session variables like 'rocksdb_tmpdir'; +Variable_name Value +rocksdb_tmpdir +# Connection con3 +show session variables like 'rocksdb_tmpdir'; +Variable_name Value +rocksdb_tmpdir MYSQL_TMP_DIR/mysqld.1 +ALTER TABLE t1 ADD INDEX kb(b), ALGORITHM=INPLACE; +set global rocksdb_tmpdir=NULL; +DROP TABLE t1; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/trx_info.result b/storage/rocksdb/mysql-test/rocksdb/r/trx_info.result new file mode 100644 index 00000000000..ada2e127021 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/r/trx_info.result @@ -0,0 +1,13 @@ +DROP TABLE IF EXISTS t1; +create table t1 (a int) engine=rocksdb; +insert into t1 values (1); +insert into t1 values (2); +set autocommit=0; +select * from t1 for update; +a +1 +2 +select * from information_schema.rocksdb_trx; +TRANSACTION_ID STATE NAME WRITE_COUNT LOCK_COUNT TIMEOUT_SEC WAITING_KEY WAITING_COLUMN_FAMILY_ID IS_REPLICATION SKIP_TRX_API READ_ONLY HAS_DEADLOCK_DETECTION NUM_ONGOING_BULKLOAD THREAD_ID QUERY +_TRX_ID_ STARTED _NAME_ 0 2 1 _KEY_ 0 0 0 0 0 0 _THREAD_ID_ select * from information_schema.rocksdb_trx +DROP TABLE t1; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/trx_info_rpl.result b/storage/rocksdb/mysql-test/rocksdb/r/trx_info_rpl.result new file mode 100644 index 00000000000..1e0c7a5adbf --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/r/trx_info_rpl.result @@ -0,0 +1,15 @@ +include/master-slave.inc +Warnings: +Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. +Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. +[connection master] +DROP TABLE IF EXISTS t1; +include/stop_slave.inc +create table t1 (a int) engine=rocksdb; +show variables like 'rocksdb_rpl_skip_tx_api'; +Variable_name Value +rocksdb_rpl_skip_tx_api ON +include/start_slave.inc +found +DROP TABLE t1; +include/rpl_end.inc diff --git a/storage/rocksdb/mysql-test/rocksdb/r/type_char_indexes_collation.result b/storage/rocksdb/mysql-test/rocksdb/r/type_char_indexes_collation.result index 9ab00243b6b..4b720271a4e 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/type_char_indexes_collation.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/type_char_indexes_collation.result @@ -1,21 +1,3 @@ -set session debug= '+d,MYROCKS_FORMAT_VERSION_INITIAL'; -create table t (id int not null auto_increment primary key, -c varchar(1) CHARACTER SET latin1 COLLATE latin1_swedish_ci, -key sk (c)); -set session debug= '-d,MYROCKS_FORMAT_VERSION_INITIAL'; -insert into t (c) values ('A'), ('b'), ('C'); -explain select c from t; -id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t index NULL sk 4 NULL # Using index -select c from t; -c -A -b -C -select c from t where c = 'a'; -c -A -drop table t; set session debug= "+d,myrocks_enable_unknown_collation_index_only_scans"; create table t (id int not null auto_increment primary key, c varchar(8) CHARACTER SET utf8 COLLATE utf8_general_ci, @@ -91,7 +73,7 @@ c1 Asdf bbbb drop table t; -set session rocksdb_verify_checksums = on; +set session rocksdb_verify_row_debug_checksums = on; create table t (id int primary key, email varchar(100), KEY email_i (email(30))) engine=rocksdb default charset=latin1; insert into t values (1, ' a'); explain select 'email_i' as index_name, count(*) AS count from t force index(email_i); diff --git a/storage/rocksdb/mysql-test/rocksdb/r/type_decimal.result b/storage/rocksdb/mysql-test/rocksdb/r/type_decimal.result index 7a7a5c7638c..7397ff64ab1 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/type_decimal.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/type_decimal.result @@ -1,80 +1,4 @@ drop table if exists t1, t2; -# -# Check that DECIMAL PK -# -create table t0(a int); -insert into t0 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); -# First, make the server to create a dataset in the old format: -set session debug= '+d,MYROCKS_FORMAT_VERSION_INITIAL'; -create table t1 ( -pk1 decimal(32,16), -pk2 decimal(32,16), -pk3 decimal(32,16), -a smallint not null, -primary key(pk1, pk2, pk3) -); -insert into t1 -select -A.a, B.a, C.a, 1234 -from t0 A, t0 B, t0 C; -# -# Looking at the table size, one can tell that the data is stored using -# old format: -# -set global rocksdb_force_flush_memtable_now=1; -# Check the format version: -select table_name,index_name,kv_format_version -from information_schema.ROCKSDB_DDL -where TABLE_SCHEMA=database() AND table_name='t1'; -table_name index_name kv_format_version -t1 PRIMARY 10 -flush tables; -set session debug= '-d,MYROCKS_FORMAT_VERSION_INITIAL'; -# Check that the new server reads the data in the old format: -select * from t1 order by pk1,pk2,pk3 limit 5; -pk1 pk2 pk3 a -0.0000000000000000 0.0000000000000000 0.0000000000000000 1234 -0.0000000000000000 0.0000000000000000 1.0000000000000000 1234 -0.0000000000000000 0.0000000000000000 2.0000000000000000 1234 -0.0000000000000000 0.0000000000000000 3.0000000000000000 1234 -0.0000000000000000 0.0000000000000000 4.0000000000000000 1234 -# -# Ok, now, enable the new data format: -# -create table t2 ( -pk1 decimal(32,16), -pk2 decimal(32,16), -pk3 decimal(32,16), -a smallint not null, -primary key(pk1, pk2, pk3) -); -insert into t2 -select -A.a, B.a, C.a, 1234 -from t0 A, t0 B, t0 C; -set global rocksdb_force_flush_memtable_now=1; -larger -1 -# This should show the new PK data fromat -select table_name,index_name,kv_format_version from information_schema.ROCKSDB_DDL -where TABLE_SCHEMA=database() AND table_name='t2'; -table_name index_name kv_format_version -t2 PRIMARY 11 -# -# Check that the server is able to read BOTH the old and the new formats: -# -select * from t2 limit 3; -pk1 pk2 pk3 a -0.0000000000000000 0.0000000000000000 0.0000000000000000 1234 -0.0000000000000000 0.0000000000000000 1.0000000000000000 1234 -0.0000000000000000 0.0000000000000000 2.0000000000000000 1234 -select * from t1 limit 3; -pk1 pk2 pk3 a -0.0000000000000000 0.0000000000000000 0.0000000000000000 1234 -0.0000000000000000 0.0000000000000000 1.0000000000000000 1234 -0.0000000000000000 0.0000000000000000 2.0000000000000000 1234 -drop table t1,t2; -drop table t0; # # Check that DECIMAL datatype supports 'index-only' scans and is decoded correctly. # (Decoding happens from the mem-comparable image in the index, regardless diff --git a/storage/rocksdb/mysql-test/rocksdb/r/type_varchar.result b/storage/rocksdb/mysql-test/rocksdb/r/type_varchar.result index 5c449da2b8f..3cb06bc3c9c 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/type_varchar.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/type_varchar.result @@ -727,9 +727,9 @@ index_name count email_i 1 drop table t; set @save_rocksdb_checksums_pct = @@global.rocksdb_checksums_pct; -set @save_rocksdb_verify_checksums = @@session.rocksdb_verify_checksums; +set @save_rocksdb_verify_row_debug_checksums = @@session.rocksdb_verify_row_debug_checksums; set global rocksdb_checksums_pct = 100; -set session rocksdb_verify_checksums = on; +set session rocksdb_verify_row_debug_checksums = on; create table t (id int primary key, email varchar(100), KEY email_i (email(30))); insert into t values (1, 'a'); explain select 'email_i' as index_name, count(*) AS count from t force index(email_i); @@ -740,4 +740,4 @@ index_name count email_i 1 drop table t; set global rocksdb_checksums_pct = @save_rocksdb_checksums_pct; -set session rocksdb_verify_checksums = @save_rocksdb_verify_checksums; +set session rocksdb_verify_row_debug_checksums = @save_rocksdb_verify_row_debug_checksums; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/type_varchar_debug.result b/storage/rocksdb/mysql-test/rocksdb/r/type_varchar_debug.result deleted file mode 100644 index de7608ebb1c..00000000000 --- a/storage/rocksdb/mysql-test/rocksdb/r/type_varchar_debug.result +++ /dev/null @@ -1,254 +0,0 @@ -drop table if exists t1,t2; -set session debug= "+d,myrocks_enable_unknown_collation_index_only_scans"; -# -# Issue 257: Sort order for varchars is different between -# MyISAM/InnoDB vs MyRocks -# -create table t1 ( -pk varchar(64) CHARACTER SET utf8 COLLATE utf8_general_ci, -col1 varchar(64), -primary key (pk) -); -insert into t1 values ('a','a'); -insert into t1 values ('a ', 'a-space'); -ERROR 23000: Duplicate entry 'a ' for key 'PRIMARY' -insert into t1 values('b ', 'b-2x-space'); -insert into t1 values ('b', 'b'); -ERROR 23000: Duplicate entry 'b' for key 'PRIMARY' -select pk, hex(pk), col1 from t1; -pk hex(pk) col1 -a 61 a -b 622020 b-2x-space -insert into t1 values ('a\t', 'a-tab'); -insert into t1 values ('a \t', 'a-space-tab'); -select pk, hex(pk), col1 from t1 order by pk; -pk hex(pk) col1 -a 6109 a-tab -a 612009 a-space-tab -a 61 a -b 622020 b-2x-space -# Try longer values -insert into t1 values (concat('a', repeat(' ',10)), 'a-10-x-space'); -ERROR 23000: Duplicate entry 'a ' for key 'PRIMARY' -insert into t1 values (concat('c', repeat(' ',10)), 'c-10-x-space'); -select * from t1; -pk col1 -a a-tab -a a-space-tab -a a -b b-2x-space -c c-10-x-space -drop table t1; -# Secondary index -create table t1 ( -pk int not null primary key, -col1 varchar(64) CHARACTER SET utf8 COLLATE utf8_general_ci, -col2 varchar(64), -key (col1) -); -insert into t1 values (0, 'ab', 'a-b'); -insert into t1 values (1, 'a ', 'a-space'); -insert into t1 values (2, 'a', 'a'); -insert into t1 values (3, 'a \t', 'a-tab'); -# Must show 'using index' for latin1_bin and utf8_bin: -explain -select col1, hex(col1) from t1; -id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 index NULL col1 195 NULL # Using index -select col1, hex(col1) from t1; -col1 hex(col1) -a 61202009 -a 6120 -a 61 -ab 6162 -# Must show 'using index' for latin1_bin and utf8_bin: -explain -select col1, hex(col1) from t1 where col1 < 'b'; -id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 # col1 col1 195 NULL # Using where; Using index -select col1, hex(col1) from t1 where col1 < 'b'; -col1 hex(col1) -a 61202009 -a 6120 -a 61 -ab 6162 -delete from t1; -insert into t1 values(10, '', 'empty'); -insert into t1 values(11, repeat(' ', 8), '8x-space'); -insert into t1 values(12, repeat(' ', 16), '16x-space'); -insert into t1 values(13, repeat(' ', 24), '24x-space'); -insert into t1 values(14, concat(repeat(' ', 16),'a'), '16x-space-a'); -insert into t1 values(21, repeat(' ', 9), '9x-space'); -insert into t1 values(22, repeat(' ',17), '17x-space'); -insert into t1 values(23, repeat(' ',18), '18x-space'); -explain -select pk, col1, hex(col1), length(col1) from t1; -id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 # NULL col1 195 NULL # Using index -select pk, col1, hex(col1), length(col1) from t1; -pk col1 hex(col1) length(col1) -10 0 -11 2020202020202020 8 -12 20202020202020202020202020202020 16 -13 202020202020202020202020202020202020202020202020 24 -21 202020202020202020 9 -22 2020202020202020202020202020202020 17 -23 202020202020202020202020202020202020 18 -14 a 2020202020202020202020202020202061 17 -drop table t1; -create table t1 (pk int primary key, a varchar(512), key(a)) engine=rocksdb; -insert into t1 values (1, concat('a', repeat(' ', 300))); -insert into t1 values (2, concat('b', repeat(' ', 300))); -select pk,length(a) from t1 force index(a) where a < 'zz'; -pk length(a) -1 301 -2 301 -select pk,length(a),rtrim(a) from t1 force index(a) where a < 'zz'; -pk length(a) rtrim(a) -1 301 a -2 301 b -select pk,length(a),rtrim(a) from t1 ignore index(a) where a < 'zz'; -pk length(a) rtrim(a) -1 301 a -2 301 b -drop table t1; -set session debug= "-d,myrocks_enable_unknown_collation_index_only_scans"; -# -# Check backwards compatibility: -# -set session debug= '+d,MYROCKS_FORMAT_VERSION_INITIAL'; -# Create the tables in the old format -create table t1 ( -pk varchar(64) collate latin1_bin, -col1 varchar(64), -primary key (pk) -); -insert into t1 values ('a','a'); -# The following will not produce an error: -insert into t1 values ('a ', 'a-space'); -select pk, hex(pk), col1 from t1; -pk hex(pk) col1 -a 61 a -a 6120 a-space -create table t2 ( -pk int not null primary key, -col1 varchar(64) collate latin1_bin, -col2 varchar(64), -unique key (col1) -); -insert into t2 values (0, 'ab', 'a-b'); -# The following will not produce an error: -insert into t2 values (1, 'a ', 'a-space'); -insert into t2 values (2, 'a', 'a'); -select pk, col1, hex(col1), col2 from t2; -pk col1 hex(col1) col2 -0 ab 6162 a-b -1 a 6120 a-space -2 a 61 a -# Check the format version: -select table_name,index_name,kv_format_version -from information_schema.ROCKSDB_DDL -where TABLE_SCHEMA=database() AND table_name in ('t1','t2'); -table_name index_name kv_format_version -t1 PRIMARY 10 -t2 PRIMARY 10 -t2 col1 10 -flush tables; -set session debug= '-d,MYROCKS_FORMAT_VERSION_INITIAL'; -select pk, hex(pk), col1 from t1; -pk hex(pk) col1 -a 61 a -a 6120 a-space -select pk, col1, hex(col1), col2 from t2; -pk col1 hex(col1) col2 -0 ab 6162 a-b -1 a 6120 a-space -2 a 61 a -select pk, hex(pk), col1 from t1; -pk hex(pk) col1 -a 61 a -a 6120 a-space -select pk, col1, hex(col1), col2 from t2; -pk col1 hex(col1) col2 -0 ab 6162 a-b -1 a 6120 a-space -2 a 61 a -drop table t1,t2; -# -# General upgrade tests to see that they work. -# -set session debug= '+d,MYROCKS_FORMAT_VERSION_INITIAL'; -create table t2 ( -id int primary key, -col1 varchar(64) collate latin1_swedish_ci, -unique key (col1) -) engine=rocksdb; -set session debug= '-d,MYROCKS_FORMAT_VERSION_INITIAL'; -insert into t2 values (1, 'a'); -insert into t2 values (2, 'b'); -insert into t2 values (3, 'c'); -insert into t2 values (4, 'c '); -select col1 from t2; -col1 -a -b -c -c -delete from t2 where id = 4; -alter table t2 engine=rocksdb; -select col1 from t2; -col1 -a -b -c -insert into t2 values (4, 'c '); -ERROR 23000: Duplicate entry 'c ' for key 'col1' -drop table t2; -set session debug= '+d,MYROCKS_FORMAT_VERSION_INITIAL'; -create table t2 ( -id int primary key, -col1 varchar(64) collate latin1_bin, -unique key (col1) -) engine=rocksdb; -set session debug= '-d,MYROCKS_FORMAT_VERSION_INITIAL'; -insert into t2 values (1, 'a'); -insert into t2 values (2, 'b'); -insert into t2 values (3, 'c'); -insert into t2 values (4, 'c '); -select col1 from t2; -col1 -a -b -c -c -delete from t2 where id = 4; -alter table t2 engine=rocksdb; -select col1 from t2; -col1 -a -b -c -insert into t2 values (4, 'c '); -ERROR 23000: Duplicate entry 'c ' for key 'col1' -drop table t2; -# -# Check what happens when one tries to 'upgrade' to the new data format -# and causes a unique key violation: -# -set session debug= '+d,MYROCKS_FORMAT_VERSION_INITIAL'; -create table t2 ( -pk int not null primary key, -col1 varchar(64) collate latin1_bin, -col2 varchar(64), -unique key (col1) -); -insert into t2 values (1, 'a ', 'a-space'); -insert into t2 values (2, 'a', 'a'); -select * from t2; -pk col1 col2 -1 a a-space -2 a a -set session debug= '-d,MYROCKS_FORMAT_VERSION_INITIAL'; -alter table t2 engine=rocksdb; -ERROR 23000: Duplicate entry 'a' for key 'col1' -drop table t2; diff --git a/storage/rocksdb/mysql-test/rocksdb/r/use_direct_reads_writes.result b/storage/rocksdb/mysql-test/rocksdb/r/use_direct_reads_writes.result new file mode 100644 index 00000000000..d5cfdee4f07 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/r/use_direct_reads_writes.result @@ -0,0 +1,2 @@ + RocksDB: Can't enable both use_direct_reads and allow_mmap_reads + RocksDB: Can't enable both use_direct_writes and allow_mmap_writes diff --git a/storage/rocksdb/mysql-test/rocksdb/t/2pc_group_commit-master.opt b/storage/rocksdb/mysql-test/rocksdb/t/2pc_group_commit-master.opt new file mode 100644 index 00000000000..83ed8522e72 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/t/2pc_group_commit-master.opt @@ -0,0 +1 @@ +--binlog-format=row diff --git a/storage/rocksdb/mysql-test/rocksdb/t/2pc_group_commit.test b/storage/rocksdb/mysql-test/rocksdb/t/2pc_group_commit.test new file mode 100644 index 00000000000..c806e46aa4d --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/t/2pc_group_commit.test @@ -0,0 +1,64 @@ +--source include/have_rocksdb.inc +--source include/have_log_bin.inc + +--echo # Disable for valgrind because this takes too long +--source include/not_valgrind.inc + +--disable_warnings +DROP DATABASE IF EXISTS mysqlslap; +--enable_warnings + +CREATE DATABASE mysqlslap; +USE mysqlslap; +CREATE TABLE t1(id BIGINT AUTO_INCREMENT, value BIGINT, PRIMARY KEY(id)) ENGINE=rocksdb; + +--echo # 2PC enabled, MyRocks durability enabled +SET GLOBAL rocksdb_disable_2pc=0; +SET GLOBAL rocksdb_write_sync=1; + +--echo ## 2PC + durability + single thread +select variable_value into @c from information_schema.global_status where variable_name='rocksdb_wal_group_syncs'; +--exec $MYSQL_SLAP --silent --concurrency=1 --number-of-queries=1000 --query="INSERT INTO t1 (id, value) VALUES(NULL, 1)" +select case when variable_value-@c = 1000 then 'true' else 'false' end from information_schema.global_status where variable_name='rocksdb_wal_group_syncs'; + +--echo ## 2PC + durability + group commit +select variable_value into @c from information_schema.global_status where variable_name='rocksdb_wal_group_syncs'; +--exec $MYSQL_SLAP --silent --concurrency=50 --number-of-queries=10000 --query="INSERT INTO t1 (id, value) VALUES(NULL, 1)" +select case when variable_value-@c > 0 and variable_value-@c < 10000 then 'true' else 'false' end from information_schema.global_status where variable_name='rocksdb_wal_group_syncs'; + + +--echo # 2PC enabled, MyRocks durability disabled +SET GLOBAL rocksdb_disable_2pc=0; +SET GLOBAL rocksdb_write_sync=0; + +select variable_value into @c from information_schema.global_status where variable_name='rocksdb_wal_group_syncs'; +--exec $MYSQL_SLAP --silent --concurrency=1 --number-of-queries=1000 --query="INSERT INTO t1 (id, value) VALUES(NULL, 1)" +select case when variable_value-@c = 0 then 'true' else 'false' end from information_schema.global_status where variable_name='rocksdb_wal_group_syncs'; + +select variable_value into @c from information_schema.global_status where variable_name='rocksdb_wal_group_syncs'; +--exec $MYSQL_SLAP --silent --concurrency=50 --number-of-queries=10000 --query="INSERT INTO t1 (id, value) VALUES(NULL, 1)" +select case when variable_value-@c = 0 then 'true' else 'false' end from information_schema.global_status where variable_name='rocksdb_wal_group_syncs'; + + +--echo # 2PC disabled, MyRocks durability enabled +SET GLOBAL rocksdb_disable_2pc=1; +SET GLOBAL rocksdb_write_sync=1; + +select variable_value into @c from information_schema.global_status where variable_name='rocksdb_wal_group_syncs'; +--exec $MYSQL_SLAP --silent --concurrency=1 --number-of-queries=1000 --query="INSERT INTO t1 (id, value) VALUES(NULL, 1)" +select case when variable_value-@c = 0 then 'true' else 'false' end from information_schema.global_status where variable_name='rocksdb_wal_group_syncs'; + +select variable_value into @c from information_schema.global_status where variable_name='rocksdb_wal_group_syncs'; +--exec $MYSQL_SLAP --silent --concurrency=50 --number-of-queries=10000 --query="INSERT INTO t1 (id, value) VALUES(NULL, 1)" +select case when variable_value-@c = 0 then 'true' else 'false' end from information_schema.global_status where variable_name='rocksdb_wal_group_syncs'; + + + + + + + +SET GLOBAL rocksdb_disable_2pc=1; +SET GLOBAL rocksdb_write_sync=0; +DROP TABLE t1; +DROP DATABASE mysqlslap; diff --git a/storage/rocksdb/mysql-test/rocksdb/t/add_index_inplace.test b/storage/rocksdb/mysql-test/rocksdb/t/add_index_inplace.test index e0d7a4465c8..7d3f4091bb4 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/add_index_inplace.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/add_index_inplace.test @@ -1,5 +1,4 @@ --source include/have_rocksdb.inc ---source include/have_debug.inc --disable_warnings drop table if exists t1; @@ -168,123 +167,179 @@ SELECT COUNT(*) FROM t1; DROP TABLE t1; -# -# test crash recovery -# +# test failure in prepare phase (due to collation) +CREATE TABLE t1 (a INT, b TEXT); -CREATE TABLE t1 (a INT, b INT, KEY ka(a), KEY kab(a,b)) ENGINE=RocksDB; -INSERT INTO t1 (a, b) VALUES (1, 5); -INSERT INTO t1 (a, b) VALUES (2, 6); -INSERT INTO t1 (a, b) VALUES (3, 7); +--error 1105 +ALTER TABLE t1 ADD KEY kb(b(10)); +ALTER TABLE t1 ADD PRIMARY KEY(a); +DROP TABLE t1; ---echo # crash_during_online_index_creation -flush logs; +# make sure race condition between connection close and alter on another +# connection is handled ---exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect -SET SESSION debug="+d,crash_during_online_index_creation"; ---error 2013 -ALTER TABLE t1 ADD INDEX kb(b), ALGORITHM=INPLACE; +set global rocksdb_bulk_load=1; ---enable_reconnect ---source include/wait_until_connected_again.inc +--echo # Establish connection con1 (user=root) +connect (con1,localhost,root,,); -SET SESSION debug="-d,crash_during_online_index_creation"; +--echo # Switch to connection con1 +connection con1; -SHOW CREATE TABLE t1; -CHECK TABLE t1; +show global variables like 'rocksdb_bulk_load'; +show session variables like 'rocksdb_bulk_load'; + +CREATE TABLE t1 (i INT, j INT, PRIMARY KEY (i)) ENGINE = ROCKSDB; + +INSERT INTO t1 VALUES (1,1); + +# Disconnect connection 1, this starts the code path that will call +# rocksdb_close_connection, ending the bulk load. +--echo # Disconnecting on con1 +disconnect con1; + +--echo # Establish connection con2 (user=root) +connect (con2,localhost,root,,); +--echo # Switch to connection con2 +connection con2; + +# when alter table happens, it tries to close all other TABLE instances +# when acquiring the exclusive lock for alter table (this happens in SQL layer) +# make sure bulk_load now handles this possible race condition properly +ALTER TABLE t1 ADD INDEX kj(j), ALGORITHM=INPLACE; + +SELECT COUNT(*) FROM t1 FORCE INDEX(PRIMARY); +SELECT COUNT(*) FROM t1 FORCE INDEX(kj); DROP TABLE t1; +disconnect con2; -# -# Test crash recovery with partitioned tables -# -CREATE TABLE t1 (i INT, j INT, k INT, PRIMARY KEY (i), KEY(j)) ENGINE = ROCKSDB PARTITION BY KEY(i) PARTITIONS 4; +# make sure implicilty closing the alter from another session works ---disable_query_log -let $max = 100; -let $i = 1; -while ($i <= $max) { - let $insert = INSERT INTO t1 VALUES ($i, $i, $i); - inc $i; - eval $insert; -} ---enable_query_log +--echo # Establish connection con1 (user=root) +connect (con1,localhost,root,,); +--echo # Establish connection con2 (user=root) +connect (con2,localhost,root,,); ---echo # crash_during_index_creation_partition -flush logs; +--echo # Switch to connection con1 +connection con1; ---exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect -SET SESSION debug="+d,crash_during_index_creation_partition"; ---error 2013 -ALTER TABLE t1 ADD INDEX kij(i,j), ALGORITHM=INPLACE; +CREATE TABLE t1 (i INT, j INT, PRIMARY KEY (i)) ENGINE = ROCKSDB; ---enable_reconnect ---source include/wait_until_connected_again.inc +set rocksdb_bulk_load=1; +INSERT INTO t1 VALUES (1,1); -SET SESSION debug="-d,crash_during_index_creation_partition"; +--echo # Switch to connection con2 +connection con2; -SHOW CREATE TABLE t1; +# here, the bulk load hasn't been completed yet, and we are in conn2 +# therefore select count returns 0 +SELECT COUNT(*) FROM t1 FORCE INDEX(PRIMARY); -# here, the index numbers should be higher because previously 4 index numbers -# were allocated for the partitioned table -ALTER TABLE t1 ADD INDEX kij(i,j), ALGORITHM=INPLACE; +# implicilty close the table from connection 2 +ALTER TABLE t1 ADD INDEX kj(j), ALGORITHM=INPLACE; -SELECT * FROM t1 ORDER BY i LIMIT 10; -SELECT COUNT(*) FROM t1; +SELECT COUNT(*) FROM t1 FORCE INDEX(PRIMARY); +SELECT COUNT(*) FROM t1 FORCE INDEX(kj); + +set global rocksdb_bulk_load=0; DROP TABLE t1; -# -# Test rollback on partitioned tables for inplace alter -# -CREATE TABLE t1 (i INT, j INT, k INT, PRIMARY KEY (i), KEY(j)) ENGINE = ROCKSDB PARTITION BY KEY(i) PARTITIONS 4; +connection default; + + +SET @prior_rocksdb_merge_combine_read_size= @@rocksdb_merge_combine_read_size; +SET @prior_rocksdb_strict_collation_check= @@rocksdb_strict_collation_check; +SET @prior_rocksdb_merge_buf_size = @@rocksdb_merge_buf_size; + +SET global rocksdb_strict_collation_check = off; +SET session rocksdb_merge_combine_read_size = 566; +SET session rocksdb_merge_buf_size = 336; + +show variables like '%rocksdb_bulk_load%'; +CREATE TABLE t1 (a VARCHAR(80)) ENGINE=RocksDB; +INSERT INTO t1 (a) VALUES (REPEAT("a", 80)); +INSERT INTO t1 (a) VALUES (REPEAT("a", 80)); +INSERT INTO t1 (a) VALUES (REPEAT("a", 80)); +INSERT INTO t1 (a) VALUES (REPEAT("a", 80)); +ALTER TABLE t1 ADD INDEX ka(a), ALGORITHM=INPLACE; +SHOW CREATE TABLE t1; +CHECK TABLE t1; +--sorted_result +SELECT * FROM t1 FORCE INDEX(ka) WHERE a > ""; +DROP TABLE t1; + +SET session rocksdb_merge_buf_size = @prior_rocksdb_merge_buf_size; +SET session rocksdb_merge_combine_read_size = @prior_rocksdb_merge_combine_read_size; +SET global rocksdb_strict_collation_check = @prior_rocksdb_strict_collation_check; + +# Test to make sure index statistics are updating properly +CREATE TABLE t1 (i INT, j INT, PRIMARY KEY (i)) ENGINE = ROCKSDB; --disable_query_log let $max = 100; let $i = 1; while ($i <= $max) { - let $insert = INSERT INTO t1 VALUES ($i, $i, $i); + let $insert = INSERT INTO t1 VALUES ($i, $i); inc $i; eval $insert; } --enable_query_log ---echo # crash_during_index_creation_partition -flush logs; +set global rocksdb_force_flush_memtable_now=1; ---exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect -SET SESSION debug="+d,myrocks_simulate_index_create_rollback"; +--let $data_length_old = query_get_value("select INDEX_LENGTH from information_schema.tables where table_schema=database() and table_name='t1'", INDEX_LENGTH, 1) ---echo # expected assertion failure from sql layer here for alter rollback -call mtr.add_suppression("Assertion `0' failed."); -call mtr.add_suppression("Attempting backtrace. You can use the following information to find out"); +## uncomment to see the actual values +#--replace_column 8 # +#SHOW TABLE STATUS WHERE name LIKE 't1'; ---error 2013 +# Now do an alter and see what happens +ALTER TABLE t1 ADD INDEX kj(j), ALGORITHM=INPLACE; -ALTER TABLE t1 ADD INDEX kij(i,j), ALGORITHM=INPLACE; +--let $data_length_new = query_get_value("select INDEX_LENGTH from information_schema.tables where table_schema=database() and table_name='t1'", INDEX_LENGTH, 1) +--disable_query_log +--eval select $data_length_old < $data_length_new as "larger" ---enable_reconnect +--source include/restart_mysqld.inc --source include/wait_until_connected_again.inc +--let $data_length_new = query_get_value("select INDEX_LENGTH from information_schema.tables where table_schema=database() and table_name='t1'", INDEX_LENGTH, 1) +--disable_query_log +--eval select $data_length_old < $data_length_new as "larger" -SET SESSION debug="-d,myrocks_simulate_index_create_rollback"; +analyze table t1; +--let $data_length_new = query_get_value("select INDEX_LENGTH from information_schema.tables where table_schema=database() and table_name='t1'", INDEX_LENGTH, 1) +--disable_query_log +--eval select $data_length_old < $data_length_new as "larger" -SHOW CREATE TABLE t1; +--source include/restart_mysqld.inc +--source include/wait_until_connected_again.inc +--let $data_length_new = query_get_value("select INDEX_LENGTH from information_schema.tables where table_schema=database() and table_name='t1'", INDEX_LENGTH, 1) +--disable_query_log +--eval select $data_length_old < $data_length_new as "larger" -# here, the index numbers should be higher because previously 4 index numbers -# were allocated for the partitioned table -ALTER TABLE t1 ADD INDEX kij(i,j), ALGORITHM=INPLACE; +# verifying multiple analyze table won't change stats +--disable_query_log +let $max = 10; +let $i = 1; +while ($i <= $max) { + let $analyze = ANALYZE TABLE t1; + inc $i; + eval $analyze; +} +--enable_query_log -SHOW CREATE TABLE t1; -SELECT COUNT(*) FROM t1; +--let $data_length_new2 = query_get_value("select INDEX_LENGTH from information_schema.tables where table_schema=database() and table_name='t1'", INDEX_LENGTH, 1) +--eval select $data_length_new2 < $data_length_new * 1.5 as "same" -DROP TABLE t1; -# test failure in prepare phase (due to collation) -CREATE TABLE t1 (a INT, b TEXT); +--enable_query_log + +## uncomment to see the actual values +#--replace_column 8 # +#SHOW TABLE STATUS WHERE name LIKE 't1'; ---error 1105 -ALTER TABLE t1 ADD KEY kb(b(10)); -ALTER TABLE t1 ADD PRIMARY KEY(a); DROP TABLE t1; diff --git a/storage/rocksdb/mysql-test/rocksdb/t/add_index_inplace_crash.test b/storage/rocksdb/mysql-test/rocksdb/t/add_index_inplace_crash.test new file mode 100644 index 00000000000..ca9122bccd7 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/t/add_index_inplace_crash.test @@ -0,0 +1,117 @@ +--source include/have_rocksdb.inc +--source include/have_debug.inc + +--disable_warnings +drop table if exists t1; +--enable_warnings + +# +# test crash recovery +# + +CREATE TABLE t1 (a INT, b INT, KEY ka(a), KEY kab(a,b)) ENGINE=RocksDB; +INSERT INTO t1 (a, b) VALUES (1, 5); +INSERT INTO t1 (a, b) VALUES (2, 6); +INSERT INTO t1 (a, b) VALUES (3, 7); + +--echo # crash_during_online_index_creation +flush logs; + +--exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +SET SESSION debug="+d,crash_during_online_index_creation"; +--error 2013 +ALTER TABLE t1 ADD INDEX kb(b), ALGORITHM=INPLACE; + +--enable_reconnect +--source include/wait_until_connected_again.inc + +SET SESSION debug="-d,crash_during_online_index_creation"; + +SHOW CREATE TABLE t1; +CHECK TABLE t1; + +DROP TABLE t1; + +# +# Test crash recovery with partitioned tables +# +CREATE TABLE t1 (i INT, j INT, k INT, PRIMARY KEY (i), KEY(j)) ENGINE = ROCKSDB PARTITION BY KEY(i) PARTITIONS 4; + +--disable_query_log +let $max = 100; +let $i = 1; +while ($i <= $max) { + let $insert = INSERT INTO t1 VALUES ($i, $i, $i); + inc $i; + eval $insert; +} +--enable_query_log + +--echo # crash_during_index_creation_partition +flush logs; + +--exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +SET SESSION debug="+d,crash_during_index_creation_partition"; +--error 2013 +ALTER TABLE t1 ADD INDEX kij(i,j), ALGORITHM=INPLACE; + +--enable_reconnect +--source include/wait_until_connected_again.inc + +SET SESSION debug="-d,crash_during_index_creation_partition"; + +SHOW CREATE TABLE t1; + +# here, the index numbers should be higher because previously 4 index numbers +# were allocated for the partitioned table +ALTER TABLE t1 ADD INDEX kij(i,j), ALGORITHM=INPLACE; + +SELECT * FROM t1 ORDER BY i LIMIT 10; +SELECT COUNT(*) FROM t1; + +DROP TABLE t1; + +# +# Test rollback on partitioned tables for inplace alter +# +CREATE TABLE t1 (i INT, j INT, k INT, PRIMARY KEY (i), KEY(j)) ENGINE = ROCKSDB PARTITION BY KEY(i) PARTITIONS 4; + +--disable_query_log +let $max = 100; +let $i = 1; +while ($i <= $max) { + let $insert = INSERT INTO t1 VALUES ($i, $i, $i); + inc $i; + eval $insert; +} +--enable_query_log + +--echo # crash_during_index_creation_partition +flush logs; + +--exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +SET SESSION debug="+d,myrocks_simulate_index_create_rollback"; + +--echo # expected assertion failure from sql layer here for alter rollback +call mtr.add_suppression("Assertion `0' failed."); +call mtr.add_suppression("Attempting backtrace. You can use the following information to find out"); + +--error 2013 + +ALTER TABLE t1 ADD INDEX kij(i,j), ALGORITHM=INPLACE; + +--enable_reconnect +--source include/wait_until_connected_again.inc + +SET SESSION debug="-d,myrocks_simulate_index_create_rollback"; + +SHOW CREATE TABLE t1; + +# here, the index numbers should be higher because previously 4 index numbers +# were allocated for the partitioned table +ALTER TABLE t1 ADD INDEX kij(i,j), ALGORITHM=INPLACE; + +SHOW CREATE TABLE t1; +SELECT COUNT(*) FROM t1; + +DROP TABLE t1; diff --git a/storage/rocksdb/mysql-test/rocksdb/t/add_index_inplace_sstfilewriter.test b/storage/rocksdb/mysql-test/rocksdb/t/add_index_inplace_sstfilewriter.test index 2ad2c390d59..7e600224dcc 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/add_index_inplace_sstfilewriter.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/add_index_inplace_sstfilewriter.test @@ -1,5 +1,4 @@ --source include/have_rocksdb.inc ---source include/have_debug.inc --disable_warnings drop table if exists t1; diff --git a/storage/rocksdb/mysql-test/rocksdb/t/allow_os_buffer.test b/storage/rocksdb/mysql-test/rocksdb/t/allow_os_buffer.test deleted file mode 100644 index e3ac4307c54..00000000000 --- a/storage/rocksdb/mysql-test/rocksdb/t/allow_os_buffer.test +++ /dev/null @@ -1,30 +0,0 @@ ---source include/have_rocksdb.inc - -# Issue221 -# Turning on --rocksdb-allow-mmap-reads while having --rocksdb-allow-os-buffer -# off caused an assertion in RocksDB. Now it should not be allowed and the -# server will not start with that configuration - -# Write file to make mysql-test-run.pl expect the "crash", but don't restart -# the serve runtil it is told to ---let $_server_id= `SELECT @@server_id` ---let $_expect_file_name= $MYSQLTEST_VARDIR/tmp/mysqld.$_server_id.expect ---exec echo "wait" >$_expect_file_name -shutdown_server 10; - -# Clear the log ---exec echo "" >$MYSQLTEST_VARDIR/log/mysqld.1.err - -# Attempt to restart the server with invalid options ---exec echo "restart:--rocksdb_allow_os_buffer=0 --rocksdb_allow_mmap_reads=1" >$_expect_file_name ---sleep 0.1 # Wait 100ms - that is how long the sleep is in check_expected_crash_and_restart ---exec echo "restart:" >$_expect_file_name - -# Cleanup ---enable_reconnect ---source include/wait_until_connected_again.inc ---disable_reconnect - -# We should now have an error message ---exec grep "disable allow_os_buffer" $MYSQLTEST_VARDIR/log/mysqld.1.err | cut -d] -f2 - diff --git a/storage/rocksdb/mysql-test/rocksdb/t/autoinc_vars_thread.test b/storage/rocksdb/mysql-test/rocksdb/t/autoinc_vars_thread.test new file mode 100644 index 00000000000..f801b2f683a --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/t/autoinc_vars_thread.test @@ -0,0 +1,53 @@ +--source include/have_rocksdb.inc +--source include/have_debug_sync.inc + +--echo #--------------------------- +--echo # two threads inserting simultaneously with increment > 1 +--echo # Issue #390 +--echo #--------------------------- + +CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY) ENGINE=rocksdb; + +# Set up connections +connect (con1, localhost, root,,); +SET auto_increment_increment = 2; +SET auto_increment_offset = 1; +# Insert one row to set up the conditions that caused the original failure +INSERT INTO t1 VALUES(NULL); + +connect (con2, localhost, root,,); +SET auto_increment_increment = 2; +SET auto_increment_offset = 1; + +# Start each thread on an insert that will block waiting for a signal +connection con1; +SET debug_sync='rocksdb.autoinc_vars SIGNAL parked1 WAIT_FOR go NO_CLEAR_EVENT'; +send INSERT INTO t1 VALUES(NULL); + +connection con2; +SET debug_sync='rocksdb.autoinc_vars SIGNAL parked2 WAIT_FOR go NO_CLEAR_EVENT'; +send INSERT INTO t1 VALUES(NULL); + +# Wait for both threads to be at debug_sync point +connection default; +SET debug_sync='now WAIT_FOR parked1'; +SET debug_sync='now WAIT_FOR parked2'; + +# Signal both threads to continue +SET debug_sync='now SIGNAL go'; + +connection con1; +reap; + +connection con2; +reap; + +connection default; +SET debug_sync='RESET'; + +disconnect con1; +disconnect con2; + +SELECT * FROM t1; +DROP TABLE t1; + diff --git a/storage/rocksdb/mysql-test/rocksdb/t/autoinc_vars_thread_2.test b/storage/rocksdb/mysql-test/rocksdb/t/autoinc_vars_thread_2.test new file mode 100644 index 00000000000..3c7d61aa15b --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/t/autoinc_vars_thread_2.test @@ -0,0 +1,141 @@ +--source include/have_rocksdb.inc + +--echo #--------------------------- +--echo # ten threads inserting simultaneously with increment > 1 +--echo # Issue #390 +--echo #--------------------------- + +# Run 10 simulatenous threads each inserting 10,000 rows +let $num_threads = 10; +let $num_rows_per_thread = 100000; + +# Create the table with an AUTO_INCREMENT primary key and a separate colum +# to store which thread created the row +CREATE TABLE t1 (pk INT AUTO_INCREMENT PRIMARY KEY, thr INT) ENGINE=rocksdb; + +# For each thread... +# 1) set up a connection +# 2) create a file that can be used for LOAD DATA INFILE ... +let $i = `SELECT $num_threads`; +while ($i > 0) +{ + dec $i; + + # Set up connection + connect (con$i, localhost, root,,); + + # Set up the auto_increment_* variables for each thread + eval SET auto_increment_increment = 100; + eval SET auto_increment_offset = $i + 1; + let $file = `SELECT CONCAT(@@datadir, "test_insert_", $i, ".txt")`; + + # Pass variables into perl + let ROCKSDB_INFILE = $file; + let ROCKSDB_THREAD = `SELECT $i`; + let ROCKSDB_ROWS_PER_THREAD = `SELECT $num_rows_per_thread`; + + # Create a file to load + perl; + my $fn = $ENV{'ROCKSDB_INFILE'}; + my $thr = $ENV{'ROCKSDB_THREAD'}; + my $num = $ENV{'ROCKSDB_ROWS_PER_THREAD'}; + open(my $fh, '>>', $fn) || die "perl open($fn): $!"; + for (my $ii = 0; $ii < $num; $ii++) + { + print $fh "\\N\t$thr\n" + } + close($fh); + EOF +} + +# For each connection start the LOAD DATA INFILE in the background +connection default; +let $i = `SELECT $num_threads`; +while ($i > 0) +{ + dec $i; + + connection con$i; + let $file = `SELECT CONCAT(@@datadir, "test_insert_", $i, ".txt")`; + --disable_query_log + --echo LOAD DATA INFILE <input_file> INTO TABLE t1; + send_eval LOAD DATA INFILE '$file' INTO TABLE t1; + --enable_query_log +} + +# Reap each connection's background result +connection default; +let $i = `SELECT $num_threads`; +while ($i > 0) +{ + dec $i; + + connection con$i; + reap; +} + +# Make sure we have the required number of rows +connection default; +SELECT COUNT(*) FROM t1; +SELECT thr, COUNT(pk) FROM t1 GROUP BY thr; + +# Cleanup the connection and file used for LOAD DATA INFILE +let $i = `SELECT $num_threads`; +while ($i > 0) +{ + dec $i; + + disconnect con$i; + let $file = `SELECT CONCAT(@@datadir, "test_insert_", "$i", ".txt")`; + remove_file $file; +} + +# Validate each row. For each row, the created 'thr' column shows which +# thread created the row. The pk that was automatically generated should +# therefore match a certain pattern. For thread 0, the pk should be in +# the sequence [1, 101, 201, 301, ...]; for thread 1, it should be in the +# sequence [2, 102, 202, 302, ...], etc. The pk for each row should be +# smallest value in the sequence for thread 'thr' that is greater than +# the pk in the previous row. +let $file = `SELECT CONCAT(@@datadir, "test_export.txt")`; +--disable_query_log +--echo SELECT * FROM t1 ORDER BY pk INTO OUTFILE <output_file>; +eval SELECT * FROM t1 ORDER BY pk INTO OUTFILE "$file"; +--enable_query_log + +let ROCKSDB_OUTFILE = $file; + +perl; +my $fn = $ENV{'ROCKSDB_OUTFILE'}; +my $last_pk = 0; +open(my $fh, '<', $fn) || die "perl open($fn): $!"; +while (<$fh>) +{ + if ($_ =~ m/^(.*)\t(.*)$/) + { + my $pk = $1; + my $thr = $2; + + my $expected_pk = int($last_pk / 100) * 100 + ($thr + 1); + $expected_pk += 100 if $expected_pk <= $last_pk; + + if ($expected_pk != $pk) + { + die "Incorrect next pk ($pk); expected $expected_pk (previous: $last_pk)" + } + + $last_pk = $pk; + } + else + { + die "output file has incorrect format: $_"; + } +} +print stdout "All pk values matched their expected values\n"; +EOF + +remove_file $file; + +# Drop the table to finally clean up +DROP TABLE t1; + diff --git a/storage/rocksdb/mysql-test/rocksdb/t/bulk_load.test b/storage/rocksdb/mysql-test/rocksdb/t/bulk_load.test index cfa2f6ff747..c80b789ae37 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/bulk_load.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/bulk_load.test @@ -73,6 +73,15 @@ eval LOAD DATA INFILE '$file' INTO TABLE t3; --enable_query_log set rocksdb_bulk_load=0; +# Make sure row count index stats are correct +--replace_column 6 # 7 # 8 # 9 # +SHOW TABLE STATUS WHERE name LIKE 't%'; + +ANALYZE TABLE t1, t2, t3; + +--replace_column 6 # 7 # 8 # 9 # +SHOW TABLE STATUS WHERE name LIKE 't%'; + # Make sure all the data is there. select count(pk) from t1; select count(a) from t1; diff --git a/storage/rocksdb/mysql-test/rocksdb/t/collation.test b/storage/rocksdb/mysql-test/rocksdb/t/collation.test index c3fcaad776c..030c3e21454 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/collation.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/collation.test @@ -155,7 +155,7 @@ DROP TABLE t2; # test invalid regex (missing end bracket) --exec echo "" >$MYSQLTEST_VARDIR/log/mysqld.1.err SET GLOBAL rocksdb_strict_collation_exceptions="[a-b"; ---exec grep -A 1 "Invalid pattern" $MYSQLTEST_VARDIR/log/mysqld.1.err | cut -d] -f2 +--exec grep "Invalid pattern" $MYSQLTEST_VARDIR/log/mysqld.1.err | cut -d] -f2 --error ER_UNKNOWN_ERROR CREATE TABLE a (id INT PRIMARY KEY, value varchar(50), index(value)) engine=rocksdb charset utf8; SET GLOBAL rocksdb_strict_collation_exceptions="[a-b]"; @@ -168,7 +168,7 @@ DROP TABLE a, b; # test invalid regex (trailing escape) --exec echo "" >$MYSQLTEST_VARDIR/log/mysqld.1.err SET GLOBAL rocksdb_strict_collation_exceptions="abc\\"; ---exec grep -A 1 "Invalid pattern" $MYSQLTEST_VARDIR/log/mysqld.1.err | cut -d] -f2 +--exec grep "Invalid pattern" $MYSQLTEST_VARDIR/log/mysqld.1.err | cut -d] -f2 --error ER_UNKNOWN_ERROR CREATE TABLE abc (id INT PRIMARY KEY, value varchar(50), index(value)) engine=rocksdb charset utf8; SET GLOBAL rocksdb_strict_collation_exceptions="abc"; diff --git a/storage/rocksdb/mysql-test/rocksdb/t/commit_in_the_middle_ddl.test b/storage/rocksdb/mysql-test/rocksdb/t/commit_in_the_middle_ddl.test new file mode 100644 index 00000000000..02f45779b9a --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/t/commit_in_the_middle_ddl.test @@ -0,0 +1,27 @@ +--source include/have_rocksdb.inc + +--disable_warnings +DROP TABLE IF EXISTS a; +--enable_warnings + +create table a (id int, value int, primary key (id) comment 'cf_a') engine=rocksdb; + +--disable_query_log +let $i = 1; +while ($i <= 100000) { + let $insert = INSERT INTO a VALUES($i, $i); + inc $i; + eval $insert; +} +--enable_query_log + +set rocksdb_bulk_load=1; +set rocksdb_commit_in_the_middle=1; +alter table a add index v (value) COMMENT 'cf_a'; +set rocksdb_bulk_load=0; +set rocksdb_commit_in_the_middle=0; +select count(*) from a force index(primary); +select count(*) from a force index(v); + +DROP TABLE a; + diff --git a/storage/rocksdb/mysql-test/rocksdb/t/corrupted_data_reads_debug.test b/storage/rocksdb/mysql-test/rocksdb/t/corrupted_data_reads_debug.test index a9ee98dfda5..b0e74221702 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/corrupted_data_reads_debug.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/corrupted_data_reads_debug.test @@ -21,13 +21,13 @@ insert into t1 values (1,1),(2,2),(3,3); select * from t1; -set @tmp1=@@rocksdb_verify_checksums; -set rocksdb_verify_checksums=1; +set @tmp1=@@rocksdb_verify_row_debug_checksums; +set rocksdb_verify_row_debug_checksums=1; set session debug= "+d,myrocks_simulate_bad_row_read1"; --error ER_GET_ERRNO select * from t1 where pk=1; set session debug= "-d,myrocks_simulate_bad_row_read1"; -set rocksdb_verify_checksums=@tmp1; +set rocksdb_verify_row_debug_checksums=@tmp1; select * from t1 where pk=1; diff --git a/storage/rocksdb/mysql-test/rocksdb/t/index.test b/storage/rocksdb/mysql-test/rocksdb/t/index.test index f12e7714bc3..e0c6175b397 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/index.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/index.test @@ -20,4 +20,25 @@ --source index.inc +--echo # +--echo # Issue #376: MyRocks: ORDER BY optimizer is unable to use the index extension +--echo # +create table t0 (a int); +insert into t0 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); +create table t1(a int); +insert into t1 select A.a + B.a* 10 + C.a * 100 from t0 A, t0 B, t0 C; +create table t2 ( + pk int not null, + a int not null, + b int not null, + primary key(pk), + key(a) +) engine=rocksdb; +insert into t2 select A.a, FLOOR(A.a/10), A.a from t1 A; + +--echo # This must have type=range, index=a, and must not have 'Using filesort': +--replace_column 9 # +explain select * from t2 force index (a) where a=0 and pk>=3 order by pk; + +drop table t0,t1,t2; diff --git a/storage/rocksdb/mysql-test/rocksdb/t/insert_optimized_config-master.opt b/storage/rocksdb/mysql-test/rocksdb/t/insert_optimized_config-master.opt index 2d5e33057f6..a0bf5759ec4 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/insert_optimized_config-master.opt +++ b/storage/rocksdb/mysql-test/rocksdb/t/insert_optimized_config-master.opt @@ -3,4 +3,5 @@ --rocksdb_override_cf_options=__system__={memtable=skip_list:16} --rocksdb_compaction_sequential_deletes=0 --rocksdb_compaction_sequential_deletes_window=0 +--rocksdb_allow_concurrent_memtable_write=0 diff --git a/storage/rocksdb/mysql-test/rocksdb/t/lock_info.test b/storage/rocksdb/mysql-test/rocksdb/t/lock_info.test new file mode 100644 index 00000000000..1b624cf38c0 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/t/lock_info.test @@ -0,0 +1,31 @@ +--source include/have_rocksdb.inc + +--disable_warnings +DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t2; +--enable_warnings + +create table t1 (a int, primary key(a) comment 'lock_into_cf1') engine=rocksdb; +insert into t1 values (1); +insert into t1 values (2); + +create table t2 (a int, primary key(a) comment 'lock_info_cf2') engine=rocksdb; +insert into t2 values (1); +insert into t2 values (2); + +set autocommit=0; +select * from t1 for update; +select * from t2 for update; + +use information_schema; +--replace_column 2 _txn_id_ 3 _key_ +select rocksdb_ddl.cf, rocksdb_locks.transaction_id, rocksdb_locks.key +from rocksdb_locks +left join rocksdb_ddl +on rocksdb_locks.column_family_id=rocksdb_ddl.column_family +order by rocksdb_ddl.cf; + +use test; + +DROP TABLE t1; +DROP TABLE t2; diff --git a/storage/rocksdb/mysql-test/rocksdb/t/optimizer_loose_index_scans.test b/storage/rocksdb/mysql-test/rocksdb/t/optimizer_loose_index_scans.test new file mode 100644 index 00000000000..beccc8a6b8e --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/t/optimizer_loose_index_scans.test @@ -0,0 +1,3 @@ + +let $engine=rocksdb; +--source include/loose_index_scans.inc diff --git a/storage/rocksdb/mysql-test/rocksdb/t/rocksdb.test b/storage/rocksdb/mysql-test/rocksdb/t/rocksdb.test index f612cb0997b..7ec15d157a7 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/rocksdb.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/rocksdb.test @@ -818,13 +818,17 @@ update t49 set a = 100 where pk = 1; --connect (con1,localhost,root,,) --let $con1_id = `SELECT CONNECTION_ID()` -set rocksdb_lock_wait_timeout=5000; +set rocksdb_lock_wait_timeout=60; set @var1= to_seconds(now()); send update t49 set a = 1000 where pk = 1; --connect (con2,localhost,root,,) --echo kill query \$con1_id; --disable_query_log +# If we immeditely kill the query - internally the condition broadcast can +# occur before the lock is waiting on the condition, thus the broadcast call +# is lost. Sleep 1 second to avoid this condition. +--sleep 1 eval kill query $con1_id; --enable_query_log --connection con1 @@ -833,10 +837,8 @@ eval kill query $con1_id; set @var2= to_seconds(now()); # We expect the time to kill query in con1 should be below -# rocksdb_lock_wait_timeout (5000). ---echo "[Jay Edgar] I've updated this query to help determine why it is sometimes failing" ---echo "(t13541934). If you get an error here (i.e. not 'passed') notify me." -select if ((@var2 - @var1) < 1000, "passed", (@var2 - @var1)) as 'result'; +# rocksdb_lock_wait_timeout (60). +select if ((@var2 - @var1) < 60, "passed", (@var2 - @var1)) as 'result'; --connection default --disconnect con1 @@ -1119,7 +1121,7 @@ set @a=-1; insert into t1 select (@a:=@a+1), 1234 from information_schema.session_variables limit 100; set @tmp1= @@rocksdb_max_row_locks; set rocksdb_max_row_locks= 20; ---error ER_INTERNAL_ERROR +--error ER_GET_ERRMSG update t1 set a=a+10; DROP TABLE t1; @@ -1922,4 +1924,42 @@ SHOW TABLE STATUS LIKE 't1'; SELECT * FROM t1; DROP TABLE t1; +--echo # +--echo # Issue #411: Setting rocksdb_commit_in_the_middle commits transaction +--echo # without releasing iterator +--echo # + +CREATE TABLE t1 (id1 bigint(20), + id2 bigint(20), + id3 bigint(20), + PRIMARY KEY (id1, id2, id3)) + DEFAULT CHARSET=latin1; + +CREATE TABLE t2 (id1 bigint(20), + id2 bigint(20), + PRIMARY KEY (id1, id2)) + DEFAULT CHARSET=latin1; + + +set rocksdb_commit_in_the_middle=1; +SET @save_rocksdb_bulk_load_size= @@rocksdb_bulk_load_size; +set rocksdb_bulk_load_size = 100; + +--disable_query_log +let $j = 10000; +while ($j) +{ + --eval insert into t1 (id1, id2, id3) values (0, $j, 0); + --eval insert into t2 (id1, id2) values (0, $j); + dec $j; +} +--enable_query_log + +DELETE t2, t1 FROM t2 LEFT JOIN t1 ON t2.id2 = t1.id2 AND t2.id1 = t1.id1 WHERE t2.id1 = 0; + +SET rocksdb_bulk_load_size= @save_rocksdb_bulk_load_size; +SET rocksdb_commit_in_the_middle=0; +DROP TABLE t1, t2; + + SET GLOBAL ROCKSDB_PAUSE_BACKGROUND_WORK = @ORIG_PAUSE_BACKGROUND_WORK; diff --git a/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_checksums.test b/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_checksums.test index 1a0364ebaee..7e5c61386f2 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_checksums.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_checksums.test @@ -5,8 +5,8 @@ # --source include/have_debug.inc -set @save_rocksdb_store_checksums=@@global.rocksdb_store_checksums; -set @save_rocksdb_verify_checksums=@@global.rocksdb_verify_checksums; +set @save_rocksdb_store_row_debug_checksums=@@global.rocksdb_store_row_debug_checksums; +set @save_rocksdb_verify_row_debug_checksums=@@global.rocksdb_verify_row_debug_checksums; set @save_rocksdb_checksums_pct=@@global.rocksdb_checksums_pct; # wiping mysql log for repeatable tests @@ -26,7 +26,7 @@ check table t1; drop table t1; -set session rocksdb_store_checksums=on; +set session rocksdb_store_row_debug_checksums=on; create table t2 (pk int primary key, a int, b int, key(a), key(b)) engine=rocksdb; insert into t2 values (1,1,1),(2,2,2),(3,3,3); check table t2; @@ -35,13 +35,13 @@ check table t2; --echo # Now, make a table that has both rows with checksums and without create table t3 (pk int primary key, a int, b int, key(a), key(b)) engine=rocksdb; insert into t3 values (1,1,1),(2,2,2),(3,3,3); -set session rocksdb_store_checksums=off; +set session rocksdb_store_row_debug_checksums=off; update t3 set b=3 where a=2; -set session rocksdb_store_checksums=on; +set session rocksdb_store_row_debug_checksums=on; check table t3; --exec grep "^[0-9-]* [0-9:]* [0-9]* \[Note\] CHECKTABLE t3" $MYSQLTEST_VARDIR/log/mysqld.1.err | cut -d] -f2 -set session rocksdb_store_checksums=on; +set session rocksdb_store_row_debug_checksums=on; set session rocksdb_checksums_pct=5; create table t4 (pk int primary key, a int, b int, key(a), key(b)) engine=rocksdb; --disable_query_log @@ -71,9 +71,9 @@ insert into mtr.test_suppressions values --echo # 1. Start with mismatch in key checksum of the PK. set session debug= "+d,myrocks_simulate_bad_pk_checksum1"; -set session rocksdb_verify_checksums=off; +set session rocksdb_verify_row_debug_checksums=off; select * from t3; -set session rocksdb_verify_checksums=on; +set session rocksdb_verify_row_debug_checksums=on; --error ER_INTERNAL_ERROR select * from t3; --error ER_INTERNAL_ERROR @@ -82,9 +82,9 @@ set session debug= "-d,myrocks_simulate_bad_pk_checksum1"; --echo # 2. Continue with mismatch in pk value checksum. set session debug= "+d,myrocks_simulate_bad_pk_checksum2"; -set session rocksdb_verify_checksums=off; +set session rocksdb_verify_row_debug_checksums=off; select * from t3; -set session rocksdb_verify_checksums=on; +set session rocksdb_verify_row_debug_checksums=on; --error ER_INTERNAL_ERROR select * from t3; --error ER_INTERNAL_ERROR @@ -105,6 +105,9 @@ select * from t4 force index(a) where a<1000000; set session debug= "-d,myrocks_simulate_bad_key_checksum1"; --echo # 4. The same for index-only reads? +--disable_query_log +set global rocksdb_force_flush_memtable_now=1; +--enable_query_log --replace_column 9 # explain select a from t3 force index(a) where a<4; @@ -117,8 +120,8 @@ select a from t3 force index(a) where a<4; select a from t4 force index(a) where a<1000000; set session debug= "-d,myrocks_simulate_bad_key_checksum1"; -set @@global.rocksdb_store_checksums=@save_rocksdb_store_checksums; -set @@global.rocksdb_verify_checksums=@save_rocksdb_verify_checksums; +set @@global.rocksdb_store_row_debug_checksums=@save_rocksdb_store_row_debug_checksums; +set @@global.rocksdb_verify_row_debug_checksums=@save_rocksdb_verify_row_debug_checksums; set @@global.rocksdb_checksums_pct=@save_rocksdb_checksums_pct; drop table t2,t3,t4; diff --git a/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_deadlock_detect.inc b/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_deadlock_detect.inc new file mode 100644 index 00000000000..01180ea29a8 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_deadlock_detect.inc @@ -0,0 +1,90 @@ +# +# Some basic sanity tests for deadlock detection. +# +--source include/have_rocksdb.inc + +set @prior_rocksdb_lock_wait_timeout = @@rocksdb_lock_wait_timeout; +set @prior_rocksdb_deadlock_detect = @@rocksdb_deadlock_detect; +set global rocksdb_lock_wait_timeout = 100000; +set global rocksdb_deadlock_detect = ON; + +create table t (i int primary key); +create table r1 (id int primary key, value int); +insert into r1 values (1,1),(2,2),(3,3),(4,4),(5,5),(6,6),(7,7),(8,8),(9,9),(10,10); +create table r2 like r1; +insert into r2 select * from r1; + +# deadlock on scanned locking reads +connect (con1,localhost,root,,); +let $con1= `SELECT CONNECTION_ID()`; +begin; +update r2 set value=100 where id=9; + +connect (con2,localhost,root,,); +let $con2= `SELECT CONNECTION_ID()`; +begin; +update r1 set value=100 where id=8; +--send select * from r2 for update; + +connection con1; +let $wait_condition = +`SELECT CONCAT('select count(*) = 1 from information_schema.rocksdb_trx where THREAD_ID = ', '$con2', ' and WAITING_KEY != ""')`; +--source include/wait_condition.inc +--error ER_LOCK_DEADLOCK +select * from r1 for update; +rollback; + +connection con2; +--reap; +rollback; + +connection con1; +begin; +insert into t values (1); + +connection con2; +begin; +insert into t values (2); + +connect (con3,localhost,root,,); +begin; +insert into t values (3); + +connection con1; +--send select * from t where i = 2 for update + +connection con2; +let $wait_condition = +`SELECT CONCAT('select count(*) = 1 from information_schema.rocksdb_trx where THREAD_ID = ', '$con1', ' and WAITING_KEY != ""')`; +--source include/wait_condition.inc + +--send select * from t where i = 3 for update + +connection con3; +let $wait_condition = +`SELECT CONCAT('select count(*) = 1 from information_schema.rocksdb_trx where THREAD_ID = ', '$con2', ' and WAITING_KEY != ""')`; +--source include/wait_condition.inc + +select * from t; +--error ER_LOCK_DEADLOCK +insert into t values (4), (1); +--echo # Statement should be rolled back +select * from t; +rollback; + +connection con2; +--reap +rollback; + +connection con1; +--reap +rollback; + +connection default; +disconnect con1; +disconnect con2; +disconnect con3; + +set global rocksdb_lock_wait_timeout = @prior_rocksdb_lock_wait_timeout; +set global rocksdb_deadlock_detect = @prior_rocksdb_deadlock_detect; +drop table t,r1,r2; diff --git a/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_deadlock_detect_rc-master.opt b/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_deadlock_detect_rc-master.opt new file mode 100644 index 00000000000..25b80282211 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_deadlock_detect_rc-master.opt @@ -0,0 +1 @@ +--transaction-isolation=read-committed diff --git a/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_deadlock_detect_rc.test b/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_deadlock_detect_rc.test new file mode 100644 index 00000000000..c7d69c61d14 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_deadlock_detect_rc.test @@ -0,0 +1 @@ +--source suite/rocksdb/t/rocksdb_deadlock_detect.inc diff --git a/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_deadlock_detect_rr.test b/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_deadlock_detect_rr.test new file mode 100644 index 00000000000..c7d69c61d14 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_deadlock_detect_rr.test @@ -0,0 +1 @@ +--source suite/rocksdb/t/rocksdb_deadlock_detect.inc diff --git a/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_deadlock_stress.inc b/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_deadlock_stress.inc new file mode 100644 index 00000000000..e164591ddec --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_deadlock_stress.inc @@ -0,0 +1,18 @@ +# +# Stress tests deadlock detection +# + +--source include/have_rocksdb.inc + +create table t1 (a int primary key, b int) engine=rocksdb; + +set @prior_rocksdb_lock_wait_timeout = @@rocksdb_lock_wait_timeout; +set @prior_rocksdb_deadlock_detect = @@rocksdb_deadlock_detect; +set global rocksdb_lock_wait_timeout = 100000; +set global rocksdb_deadlock_detect = ON; + +exec python suite/rocksdb/t/rocksdb_deadlock_stress.py root 127.0.0.1 $MASTER_MYPORT test t1 10000 10; + +set global rocksdb_lock_wait_timeout = @prior_rocksdb_lock_wait_timeout; +set global rocksdb_deadlock_detect = @prior_rocksdb_deadlock_detect; +drop table t1; diff --git a/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_deadlock_stress.py b/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_deadlock_stress.py new file mode 100644 index 00000000000..3bc8a3be010 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_deadlock_stress.py @@ -0,0 +1,94 @@ +""" +This script stress tests deadlock detection. + +Usage: rocksdb_deadlock_stress.py user host port db_name table_name + num_iters num_threads +""" +import cStringIO +import hashlib +import MySQLdb +from MySQLdb.constants import ER +import os +import random +import signal +import sys +import threading +import time +import string +import traceback + +def is_deadlock_error(exc): + error_code = exc.args[0] + return (error_code == MySQLdb.constants.ER.LOCK_DEADLOCK) + +def get_query(table_name, idx): + # Let's assume that even indexes will always be acquireable, to make + # deadlock detection more interesting. + if idx % 2 == 0: + return """SELECT * from %s WHERE a = %d LOCK IN SHARE MODE""" % (table_name, idx) + else: + r = random.randint(1, 3); + if r == 1: + return """SELECT * from %s WHERE a = %d FOR UPDATE""" % (table_name, idx) + elif r == 2: + return """INSERT INTO %s VALUES (%d, 1) + ON DUPLICATE KEY UPDATE b=b+1""" % (table_name, idx) + else: + return """DELETE from %s WHERE a = %d""" % (table_name, idx) + +class Worker(threading.Thread): + def __init__(self, con, table_name, num_iters): + threading.Thread.__init__(self) + self.con = con + self.table_name = table_name + self.num_iters = num_iters + self.exception = None + self.start() + def run(self): + try: + self.runme() + except Exception, e: + self.exception = traceback.format_exc() + def runme(self): + cur = self.con.cursor() + for x in xrange(self.num_iters): + try: + for i in random.sample(xrange(100), 10): + cur.execute(get_query(self.table_name, i)) + self.con.commit() + except MySQLdb.OperationalError, e: + self.con.rollback() + cur = self.con.cursor() + if not is_deadlock_error(e): + raise e + +if __name__ == '__main__': + if len(sys.argv) != 8: + print "Usage: rocksdb_deadlock_stress.py user host port db_name " \ + "table_name num_iters num_threads" + sys.exit(1) + + user = sys.argv[1] + host = sys.argv[2] + port = int(sys.argv[3]) + db = sys.argv[4] + table_name = sys.argv[5] + num_iters = int(sys.argv[6]) + num_workers = int(sys.argv[7]) + + worker_failed = False + workers = [] + for i in xrange(num_workers): + w = Worker( + MySQLdb.connect(user=user, host=host, port=port, db=db), table_name, + num_iters) + workers.append(w) + + for w in workers: + w.join() + if w.exception: + print "Worker hit an exception:\n%s\n" % w.exception + worker_failed = True + + if worker_failed: + sys.exit(1) diff --git a/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_deadlock_stress_rc-master.opt b/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_deadlock_stress_rc-master.opt new file mode 100644 index 00000000000..25b80282211 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_deadlock_stress_rc-master.opt @@ -0,0 +1 @@ +--transaction-isolation=read-committed diff --git a/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_deadlock_stress_rc.test b/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_deadlock_stress_rc.test new file mode 100644 index 00000000000..d80b4686d80 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_deadlock_stress_rc.test @@ -0,0 +1 @@ +--source suite/rocksdb/t/rocksdb_deadlock_stress.inc diff --git a/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_deadlock_stress_rr.test b/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_deadlock_stress_rr.test new file mode 100644 index 00000000000..d80b4686d80 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_deadlock_stress_rr.test @@ -0,0 +1 @@ +--source suite/rocksdb/t/rocksdb_deadlock_stress.inc diff --git a/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_locks-master.opt b/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_locks-master.opt new file mode 100644 index 00000000000..c9d9edb8565 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_locks-master.opt @@ -0,0 +1 @@ +--rocksdb_print_snapshot_conflict_queries=1 diff --git a/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_locks.test b/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_locks.test index 3b28df0d63b..9a25f39a8e3 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_locks.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_locks.test @@ -16,6 +16,7 @@ select * from t1 where pk=1 for update; --connect (con1,localhost,root,,) --connection con1 +call mtr.add_suppression("Got snapshot conflict errors"); --echo ### Connection con1 let $ID= `select connection_id()`; set @@rocksdb_lock_wait_timeout=500; diff --git a/storage/rocksdb/mysql-test/rocksdb/t/rollback_savepoint.test b/storage/rocksdb/mysql-test/rocksdb/t/rollback_savepoint.test new file mode 100644 index 00000000000..8543ce81de4 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/t/rollback_savepoint.test @@ -0,0 +1,31 @@ +--disable_warnings +DROP TABLE IF EXISTS t1, t2; +--enable_warnings + +CREATE TABLE t1 (a INT, b CHAR(8), pk INT AUTO_INCREMENT PRIMARY KEY) ENGINE=rocksdb; +INSERT INTO t1 (a,b) VALUES (1,'a'),(2,'b'),(3,'a'); +CREATE TABLE t2 LIKE t1; +INSERT INTO t2 SELECT * FROM t1; + +--connect (con1,localhost,root,,) +--connect (con2,localhost,root,,) + +--connection con1 +START TRANSACTION WITH CONSISTENT SNAPSHOT; +SAVEPOINT a; +SELECT * FROM t1 ORDER BY pk; +ROLLBACK TO SAVEPOINT a; +SAVEPOINT a; +SELECT * FROM t2 ORDER BY pk; +ROLLBACK TO SAVEPOINT a; + +# should not be blocked +--connection con2 +ALTER TABLE t1 RENAME TO t3; + +--connection default +DROP TABLE t2, t3; + +--disconnect con1 +--disconnect con2 + diff --git a/storage/rocksdb/mysql-test/rocksdb/t/select_lock_in_share_mode.test b/storage/rocksdb/mysql-test/rocksdb/t/select_lock_in_share_mode.test index d1d289bcb74..23ce6d45234 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/select_lock_in_share_mode.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/select_lock_in_share_mode.test @@ -31,10 +31,6 @@ SET lock_wait_timeout = 1; --sorted_result SELECT a,b FROM t1 WHERE b='a'; --sorted_result ---echo # ---echo # Currently, SELECT ... LOCK IN SHARE MODE works like ---echo # SELECT FOR UPDATE ---error ER_LOCK_WAIT_TIMEOUT SELECT a,b FROM t1 WHERE b='a' LOCK IN SHARE MODE; --error ER_LOCK_WAIT_TIMEOUT diff --git a/storage/rocksdb/mysql-test/rocksdb/t/show_engine.test b/storage/rocksdb/mysql-test/rocksdb/t/show_engine.test index 0cb32d95d8a..834887ddffb 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/show_engine.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/show_engine.test @@ -68,7 +68,8 @@ SHOW ENGINE rocksdb TRANSACTION STATUS; START TRANSACTION WITH CONSISTENT SNAPSHOT; ---replace_regex /[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}/TIMESTAMP/ /(ACTIVE) [0-9]+ /\1 NUM / /(thread id) [0-9]+/\1 TID/ /0x[0-9a-f]+/PTR/ +#select sleep(10); +--replace_regex /[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}/TIMESTAMP/ /(ACTIVE) [0-9]+ /\1 NUM / /(thread id) [0-9]+/\1 TID/ /0x[0-9a-f]+/PTR/ /(query id) [0-9]+/\1 QID/ /(root) [a-z ]+/\1 ACTION/ SHOW ENGINE rocksdb TRANSACTION STATUS; ROLLBACK; diff --git a/storage/rocksdb/mysql-test/rocksdb/t/tmpdir.test b/storage/rocksdb/mysql-test/rocksdb/t/tmpdir.test new file mode 100644 index 00000000000..28615264d35 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/t/tmpdir.test @@ -0,0 +1,35 @@ +--source include/have_rocksdb.inc + +--echo # If rocksdb_tmpdir is NULL or "", temporary file will be created in +--echo # server configuration variable location(--tmpdir) + +CREATE TABLE t1 (a INT PRIMARY KEY, b INT) ENGINE=RocksDB; +show session variables like 'rocksdb_tmpdir'; +--echo # Connection con1 +connect (con1,localhost,root); +show session variables like 'rocksdb_tmpdir'; + +ALTER TABLE t1 ADD INDEX kb(b), ALGORITHM=INPLACE; +connection default; +disconnect con1; +DROP TABLE t1; + +--echo # rocksdb_tmpdir with valid location. + +CREATE TABLE t1 (a INT PRIMARY KEY, b INT) ENGINE=RocksDB; +set @tmpdir = @@global.tmpdir; +set global rocksdb_tmpdir = @tmpdir; +show session variables like 'rocksdb_tmpdir'; + +--echo # Connection con3 +connect (con2,localhost,root); +--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR +show session variables like 'rocksdb_tmpdir'; + +ALTER TABLE t1 ADD INDEX kb(b), ALGORITHM=INPLACE; +disconnect con2; +connection default; +set global rocksdb_tmpdir=NULL; +DROP TABLE t1; + + diff --git a/storage/rocksdb/mysql-test/rocksdb/t/trx_info.test b/storage/rocksdb/mysql-test/rocksdb/t/trx_info.test new file mode 100644 index 00000000000..975bed6132c --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/t/trx_info.test @@ -0,0 +1,17 @@ +--source include/have_rocksdb.inc + +--disable_warnings +DROP TABLE IF EXISTS t1; +--enable_warnings + +create table t1 (a int) engine=rocksdb; +insert into t1 values (1); +insert into t1 values (2); + +set autocommit=0; +select * from t1 for update; + +--replace_column 1 _TRX_ID_ 3 _NAME_ 7 _KEY_ 14 _THREAD_ID_ +select * from information_schema.rocksdb_trx; + +DROP TABLE t1; diff --git a/storage/rocksdb/mysql-test/rocksdb/t/trx_info_rpl.cnf b/storage/rocksdb/mysql-test/rocksdb/t/trx_info_rpl.cnf new file mode 100644 index 00000000000..f5b725932e4 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/t/trx_info_rpl.cnf @@ -0,0 +1,8 @@ +!include suite/rpl/my.cnf + +[mysqld.1] +binlog_format=row +[mysqld.2] +binlog_format=row +slave_parallel_workers=1 +rocksdb_rpl_skip_tx_api=ON diff --git a/storage/rocksdb/mysql-test/rocksdb/t/trx_info_rpl.test b/storage/rocksdb/mysql-test/rocksdb/t/trx_info_rpl.test new file mode 100644 index 00000000000..19499765140 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/t/trx_info_rpl.test @@ -0,0 +1,42 @@ +--source include/master-slave.inc + +--disable_warnings +DROP TABLE IF EXISTS t1; +connection slave; +--source include/stop_slave.inc +--enable_warnings + +connection master; +create table t1 (a int) engine=rocksdb; +--disable_query_log +--let $aa= 0 +while ($aa < 1000) { + eval insert into t1 values ($aa); + --inc $aa +} +--enable_query_log + +connection slave; +show variables like 'rocksdb_rpl_skip_tx_api'; +--source include/start_slave.inc + +--let $it=0 +--let $stop=0 +while ($stop != 1) { +let $count= query_get_value(select count(*) as Value from information_schema.rocksdb_trx, Value, 1); + if ($count) { + --echo found + --let $stop=1 + } + + if ($it > 1000) { + --echo not found + --let $stop=1 + } + + --inc $it +} + +connection master; +DROP TABLE t1; +--source include/rpl_end.inc diff --git a/storage/rocksdb/mysql-test/rocksdb/t/type_char_indexes_collation.test b/storage/rocksdb/mysql-test/rocksdb/t/type_char_indexes_collation.test index d51b55f07e2..a0ae824f829 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/type_char_indexes_collation.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/type_char_indexes_collation.test @@ -1,21 +1,6 @@ --source include/have_rocksdb.inc --source include/have_debug.inc -# Test to see if index-only scan fails gracefully if unpack info is not -# available. -set session debug= '+d,MYROCKS_FORMAT_VERSION_INITIAL'; -create table t (id int not null auto_increment primary key, - c varchar(1) CHARACTER SET latin1 COLLATE latin1_swedish_ci, - key sk (c)); -set session debug= '-d,MYROCKS_FORMAT_VERSION_INITIAL'; -insert into t (c) values ('A'), ('b'), ('C'); ---replace_column 9 # -explain select c from t; -select c from t; -select c from t where c = 'a'; - -drop table t; - # Test if unknown collation works. set session debug= "+d,myrocks_enable_unknown_collation_index_only_scans"; create table t (id int not null auto_increment primary key, @@ -127,7 +112,7 @@ select c1 from t; drop table t; # Test varchar keyparts with key prefix -set session rocksdb_verify_checksums = on; +set session rocksdb_verify_row_debug_checksums = on; create table t (id int primary key, email varchar(100), KEY email_i (email(30))) engine=rocksdb default charset=latin1; insert into t values (1, ' a'); --replace_column 9 # diff --git a/storage/rocksdb/mysql-test/rocksdb/t/type_decimal-master.opt b/storage/rocksdb/mysql-test/rocksdb/t/type_decimal-master.opt new file mode 100644 index 00000000000..33e72265db2 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/t/type_decimal-master.opt @@ -0,0 +1 @@ +--rocksdb_debug_optimizer_n_rows=10 diff --git a/storage/rocksdb/mysql-test/rocksdb/t/type_decimal.test b/storage/rocksdb/mysql-test/rocksdb/t/type_decimal.test index d5ee75686df..ee325b34eff 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/type_decimal.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/type_decimal.test @@ -5,81 +5,6 @@ drop table if exists t1, t2; --enable_warnings ---echo # ---echo # Check that DECIMAL PK ---echo # -create table t0(a int); -insert into t0 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); - ---echo # First, make the server to create a dataset in the old format: -set session debug= '+d,MYROCKS_FORMAT_VERSION_INITIAL'; -create table t1 ( - pk1 decimal(32,16), - pk2 decimal(32,16), - pk3 decimal(32,16), - a smallint not null, - primary key(pk1, pk2, pk3) -); -insert into t1 -select - A.a, B.a, C.a, 1234 -from t0 A, t0 B, t0 C; - ---echo # ---echo # Looking at the table size, one can tell that the data is stored using ---echo # old format: ---echo # -set global rocksdb_force_flush_memtable_now=1; - ---let $data_length_old = query_get_value("select DATA_LENGTH from information_schema.tables where table_schema=database() and table_name='t1'", DATA_LENGTH, 1) - ---echo # Check the format version: -select table_name,index_name,kv_format_version -from information_schema.ROCKSDB_DDL -where TABLE_SCHEMA=database() AND table_name='t1'; - -flush tables; - -set session debug= '-d,MYROCKS_FORMAT_VERSION_INITIAL'; ---source include/restart_mysqld.inc - ---echo # Check that the new server reads the data in the old format: -select * from t1 order by pk1,pk2,pk3 limit 5; - ---echo # ---echo # Ok, now, enable the new data format: ---echo # -create table t2 ( - pk1 decimal(32,16), - pk2 decimal(32,16), - pk3 decimal(32,16), - a smallint not null, - primary key(pk1, pk2, pk3) -); -insert into t2 -select - A.a, B.a, C.a, 1234 -from t0 A, t0 B, t0 C; -set global rocksdb_force_flush_memtable_now=1; - ---let $data_length_new = query_get_value("select DATA_LENGTH from information_schema.tables where table_schema=database() and table_name='t2'", DATA_LENGTH, 1) ---disable_query_log ---eval select $data_length_old > $data_length_new as "larger" ---enable_query_log - ---echo # This should show the new PK data fromat -select table_name,index_name,kv_format_version from information_schema.ROCKSDB_DDL -where TABLE_SCHEMA=database() AND table_name='t2'; - ---echo # ---echo # Check that the server is able to read BOTH the old and the new formats: ---echo # -select * from t2 limit 3; -select * from t1 limit 3; - -drop table t1,t2; -drop table t0; - --echo # --echo # Check that DECIMAL datatype supports 'index-only' scans and is decoded correctly. --echo # (Decoding happens from the mem-comparable image in the index, regardless diff --git a/storage/rocksdb/mysql-test/rocksdb/t/type_varchar.test b/storage/rocksdb/mysql-test/rocksdb/t/type_varchar.test index 6416af7308f..e45b6836f67 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/type_varchar.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/type_varchar.test @@ -62,9 +62,9 @@ select 'email_i' as index_name, count(*) AS count from t force index(email_i); drop table t; set @save_rocksdb_checksums_pct = @@global.rocksdb_checksums_pct; -set @save_rocksdb_verify_checksums = @@session.rocksdb_verify_checksums; +set @save_rocksdb_verify_row_debug_checksums = @@session.rocksdb_verify_row_debug_checksums; set global rocksdb_checksums_pct = 100; -set session rocksdb_verify_checksums = on; +set session rocksdb_verify_row_debug_checksums = on; create table t (id int primary key, email varchar(100), KEY email_i (email(30))); insert into t values (1, 'a'); --replace_column 9 # @@ -72,4 +72,4 @@ explain select 'email_i' as index_name, count(*) AS count from t force index(ema select 'email_i' as index_name, count(*) AS count from t force index(email_i); drop table t; set global rocksdb_checksums_pct = @save_rocksdb_checksums_pct; -set session rocksdb_verify_checksums = @save_rocksdb_verify_checksums; +set session rocksdb_verify_row_debug_checksums = @save_rocksdb_verify_row_debug_checksums; diff --git a/storage/rocksdb/mysql-test/rocksdb/t/type_varchar_debug.test b/storage/rocksdb/mysql-test/rocksdb/t/type_varchar_debug.test deleted file mode 100644 index d61e85ed204..00000000000 --- a/storage/rocksdb/mysql-test/rocksdb/t/type_varchar_debug.test +++ /dev/null @@ -1,137 +0,0 @@ -# -# VARCHAR encoding tests that require debug support -# ---source include/have_rocksdb.inc ---source include/have_debug.inc - ---disable_warnings -drop table if exists t1,t2; ---enable_warnings - - -set session debug= "+d,myrocks_enable_unknown_collation_index_only_scans"; - ---let $character_set_collate=CHARACTER SET utf8 COLLATE utf8_general_ci ---source type_varchar_endspace.inc - -set session debug= "-d,myrocks_enable_unknown_collation_index_only_scans"; - ---echo # ---echo # Check backwards compatibility: ---echo # - -set session debug= '+d,MYROCKS_FORMAT_VERSION_INITIAL'; - ---echo # Create the tables in the old format - -create table t1 ( - pk varchar(64) collate latin1_bin, - col1 varchar(64), - primary key (pk) -); -insert into t1 values ('a','a'); ---echo # The following will not produce an error: -insert into t1 values ('a ', 'a-space'); -select pk, hex(pk), col1 from t1; - -create table t2 ( - pk int not null primary key, - col1 varchar(64) collate latin1_bin, - col2 varchar(64), - unique key (col1) -); - -insert into t2 values (0, 'ab', 'a-b'); ---echo # The following will not produce an error: -insert into t2 values (1, 'a ', 'a-space'); -insert into t2 values (2, 'a', 'a'); -select pk, col1, hex(col1), col2 from t2; - ---echo # Check the format version: -select table_name,index_name,kv_format_version -from information_schema.ROCKSDB_DDL -where TABLE_SCHEMA=database() AND table_name in ('t1','t2'); - -flush tables; -set session debug= '-d,MYROCKS_FORMAT_VERSION_INITIAL'; -select pk, hex(pk), col1 from t1; -select pk, col1, hex(col1), col2 from t2; - -## Check that we can still read the data when starting on the old datadir: ---source include/restart_mysqld.inc - -select pk, hex(pk), col1 from t1; -select pk, col1, hex(col1), col2 from t2; - -drop table t1,t2; - - ---echo # ---echo # General upgrade tests to see that they work. ---echo # -set session debug= '+d,MYROCKS_FORMAT_VERSION_INITIAL'; -create table t2 ( - id int primary key, - col1 varchar(64) collate latin1_swedish_ci, - unique key (col1) -) engine=rocksdb; -set session debug= '-d,MYROCKS_FORMAT_VERSION_INITIAL'; - -insert into t2 values (1, 'a'); -insert into t2 values (2, 'b'); -insert into t2 values (3, 'c'); -# Check if this is indeed the old format -insert into t2 values (4, 'c '); -select col1 from t2; -delete from t2 where id = 4; -alter table t2 engine=rocksdb; -select col1 from t2; -# Check if this is indeed the new format ---error ER_DUP_ENTRY -insert into t2 values (4, 'c '); -drop table t2; - -set session debug= '+d,MYROCKS_FORMAT_VERSION_INITIAL'; -create table t2 ( - id int primary key, - col1 varchar(64) collate latin1_bin, - unique key (col1) -) engine=rocksdb; -set session debug= '-d,MYROCKS_FORMAT_VERSION_INITIAL'; - -insert into t2 values (1, 'a'); -insert into t2 values (2, 'b'); -insert into t2 values (3, 'c'); -# Check if this is indeed the old format -insert into t2 values (4, 'c '); -select col1 from t2; -delete from t2 where id = 4; -alter table t2 engine=rocksdb; -select col1 from t2; -# Check if this is indeed the new format ---error ER_DUP_ENTRY -insert into t2 values (4, 'c '); -drop table t2; - ---echo # ---echo # Check what happens when one tries to 'upgrade' to the new data format ---echo # and causes a unique key violation: ---echo # -set session debug= '+d,MYROCKS_FORMAT_VERSION_INITIAL'; -create table t2 ( - pk int not null primary key, - col1 varchar(64) collate latin1_bin, - col2 varchar(64), - unique key (col1) -); - -insert into t2 values (1, 'a ', 'a-space'); -insert into t2 values (2, 'a', 'a'); - -select * from t2; -set session debug= '-d,MYROCKS_FORMAT_VERSION_INITIAL'; - ---error ER_DUP_ENTRY -alter table t2 engine=rocksdb; -drop table t2; - diff --git a/storage/rocksdb/mysql-test/rocksdb/t/use_direct_reads_writes.test b/storage/rocksdb/mysql-test/rocksdb/t/use_direct_reads_writes.test new file mode 100644 index 00000000000..349748e91a8 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb/t/use_direct_reads_writes.test @@ -0,0 +1,47 @@ +--source include/have_rocksdb.inc + +# Issue221 +# Turning on both --rocksdb-allow-mmap-reads and --rocksdb-use-direct-reads +# caused an assertion in RocksDB. Now it should not be allowed and the +# server will not start with that configuration + +# Write file to make mysql-test-run.pl expect the "crash", but don't restart +# the server until it is told to +--let $_server_id= `SELECT @@server_id` +--let $_expect_file_name= $MYSQLTEST_VARDIR/tmp/mysqld.$_server_id.expect +--exec echo "wait" >$_expect_file_name +shutdown_server 10; + +# Clear the log +--exec echo "" >$MYSQLTEST_VARDIR/log/mysqld.1.err + +# Attempt to restart the server with invalid options +--exec echo "restart:--rocksdb_use_direct_reads=1 --rocksdb_allow_mmap_reads=1" >$_expect_file_name +--sleep 0.1 # Wait 100ms - that is how long the sleep is in check_expected_crash_and_restart +--exec echo "restart:" >$_expect_file_name + +# Cleanup +--enable_reconnect +--source include/wait_until_connected_again.inc +--disable_reconnect + +# We should now have an error message +--exec grep "enable both use_direct_reads" $MYSQLTEST_VARDIR/log/mysqld.1.err | cut -d] -f2 + +# Repeat with --rocksdb-use-direct-writes +--let $_server_id= `SELECT @@server_id` +--let $_expect_file_name= $MYSQLTEST_VARDIR/tmp/mysqld.$_server_id.expect +--exec echo "wait" >$_expect_file_name +shutdown_server 10; + +--exec echo "" >$MYSQLTEST_VARDIR/log/mysqld.1.err + +--exec echo "restart:--rocksdb_use_direct_writes=1 --rocksdb_allow_mmap_writes=1" >$_expect_file_name +--sleep 0.1 +--exec echo "restart:" >$_expect_file_name + +--enable_reconnect +--source include/wait_until_connected_again.inc +--disable_reconnect + +--exec grep "enable both use_direct_writes" $MYSQLTEST_VARDIR/log/mysqld.1.err | cut -d] -f2 diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_allow_os_buffer_basic.result b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_allow_os_buffer_basic.result deleted file mode 100644 index 6099c3af344..00000000000 --- a/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_allow_os_buffer_basic.result +++ /dev/null @@ -1,7 +0,0 @@ -SET @start_global_value = @@global.ROCKSDB_ALLOW_OS_BUFFER; -SELECT @start_global_value; -@start_global_value -1 -"Trying to set variable @@global.ROCKSDB_ALLOW_OS_BUFFER to 444. It should fail because it is readonly." -SET @@global.ROCKSDB_ALLOW_OS_BUFFER = 444; -ERROR HY000: Variable 'rocksdb_allow_os_buffer' is a read only variable diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_block_cache_size_basic.result b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_block_cache_size_basic.result index fbd9d97e994..1cfe5385d5c 100644 --- a/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_block_cache_size_basic.result +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_block_cache_size_basic.result @@ -1,7 +1,7 @@ SET @start_global_value = @@global.ROCKSDB_BLOCK_CACHE_SIZE; SELECT @start_global_value; @start_global_value -8388608 +536870912 "Trying to set variable @@global.ROCKSDB_BLOCK_CACHE_SIZE to 444. It should fail because it is readonly." SET @@global.ROCKSDB_BLOCK_CACHE_SIZE = 444; ERROR HY000: Variable 'rocksdb_block_cache_size' is a read only variable diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_deadlock_detect_basic.result b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_deadlock_detect_basic.result new file mode 100644 index 00000000000..f200105b542 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_deadlock_detect_basic.result @@ -0,0 +1,121 @@ +CREATE TABLE valid_values (value varchar(255)); +INSERT INTO valid_values VALUES(1); +INSERT INTO valid_values VALUES(0); +INSERT INTO valid_values VALUES('on'); +INSERT INTO valid_values VALUES('off'); +CREATE TABLE invalid_values (value varchar(255)); +INSERT INTO invalid_values VALUES('\'aaa\''); +INSERT INTO invalid_values VALUES('\'bbb\''); +SET @start_global_value = @@global.ROCKSDB_DEADLOCK_DETECT; +SELECT @start_global_value; +@start_global_value +0 +SET @start_session_value = @@session.ROCKSDB_DEADLOCK_DETECT; +SELECT @start_session_value; +@start_session_value +0 +'# Setting to valid values in global scope#' +"Trying to set variable @@global.ROCKSDB_DEADLOCK_DETECT to 1" +SET @@global.ROCKSDB_DEADLOCK_DETECT = 1; +SELECT @@global.ROCKSDB_DEADLOCK_DETECT; +@@global.ROCKSDB_DEADLOCK_DETECT +1 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_DEADLOCK_DETECT = DEFAULT; +SELECT @@global.ROCKSDB_DEADLOCK_DETECT; +@@global.ROCKSDB_DEADLOCK_DETECT +0 +"Trying to set variable @@global.ROCKSDB_DEADLOCK_DETECT to 0" +SET @@global.ROCKSDB_DEADLOCK_DETECT = 0; +SELECT @@global.ROCKSDB_DEADLOCK_DETECT; +@@global.ROCKSDB_DEADLOCK_DETECT +0 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_DEADLOCK_DETECT = DEFAULT; +SELECT @@global.ROCKSDB_DEADLOCK_DETECT; +@@global.ROCKSDB_DEADLOCK_DETECT +0 +"Trying to set variable @@global.ROCKSDB_DEADLOCK_DETECT to on" +SET @@global.ROCKSDB_DEADLOCK_DETECT = on; +SELECT @@global.ROCKSDB_DEADLOCK_DETECT; +@@global.ROCKSDB_DEADLOCK_DETECT +1 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_DEADLOCK_DETECT = DEFAULT; +SELECT @@global.ROCKSDB_DEADLOCK_DETECT; +@@global.ROCKSDB_DEADLOCK_DETECT +0 +"Trying to set variable @@global.ROCKSDB_DEADLOCK_DETECT to off" +SET @@global.ROCKSDB_DEADLOCK_DETECT = off; +SELECT @@global.ROCKSDB_DEADLOCK_DETECT; +@@global.ROCKSDB_DEADLOCK_DETECT +0 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_DEADLOCK_DETECT = DEFAULT; +SELECT @@global.ROCKSDB_DEADLOCK_DETECT; +@@global.ROCKSDB_DEADLOCK_DETECT +0 +'# Setting to valid values in session scope#' +"Trying to set variable @@session.ROCKSDB_DEADLOCK_DETECT to 1" +SET @@session.ROCKSDB_DEADLOCK_DETECT = 1; +SELECT @@session.ROCKSDB_DEADLOCK_DETECT; +@@session.ROCKSDB_DEADLOCK_DETECT +1 +"Setting the session scope variable back to default" +SET @@session.ROCKSDB_DEADLOCK_DETECT = DEFAULT; +SELECT @@session.ROCKSDB_DEADLOCK_DETECT; +@@session.ROCKSDB_DEADLOCK_DETECT +0 +"Trying to set variable @@session.ROCKSDB_DEADLOCK_DETECT to 0" +SET @@session.ROCKSDB_DEADLOCK_DETECT = 0; +SELECT @@session.ROCKSDB_DEADLOCK_DETECT; +@@session.ROCKSDB_DEADLOCK_DETECT +0 +"Setting the session scope variable back to default" +SET @@session.ROCKSDB_DEADLOCK_DETECT = DEFAULT; +SELECT @@session.ROCKSDB_DEADLOCK_DETECT; +@@session.ROCKSDB_DEADLOCK_DETECT +0 +"Trying to set variable @@session.ROCKSDB_DEADLOCK_DETECT to on" +SET @@session.ROCKSDB_DEADLOCK_DETECT = on; +SELECT @@session.ROCKSDB_DEADLOCK_DETECT; +@@session.ROCKSDB_DEADLOCK_DETECT +1 +"Setting the session scope variable back to default" +SET @@session.ROCKSDB_DEADLOCK_DETECT = DEFAULT; +SELECT @@session.ROCKSDB_DEADLOCK_DETECT; +@@session.ROCKSDB_DEADLOCK_DETECT +0 +"Trying to set variable @@session.ROCKSDB_DEADLOCK_DETECT to off" +SET @@session.ROCKSDB_DEADLOCK_DETECT = off; +SELECT @@session.ROCKSDB_DEADLOCK_DETECT; +@@session.ROCKSDB_DEADLOCK_DETECT +0 +"Setting the session scope variable back to default" +SET @@session.ROCKSDB_DEADLOCK_DETECT = DEFAULT; +SELECT @@session.ROCKSDB_DEADLOCK_DETECT; +@@session.ROCKSDB_DEADLOCK_DETECT +0 +'# Testing with invalid values in global scope #' +"Trying to set variable @@global.ROCKSDB_DEADLOCK_DETECT to 'aaa'" +SET @@global.ROCKSDB_DEADLOCK_DETECT = 'aaa'; +Got one of the listed errors +SELECT @@global.ROCKSDB_DEADLOCK_DETECT; +@@global.ROCKSDB_DEADLOCK_DETECT +0 +"Trying to set variable @@global.ROCKSDB_DEADLOCK_DETECT to 'bbb'" +SET @@global.ROCKSDB_DEADLOCK_DETECT = 'bbb'; +Got one of the listed errors +SELECT @@global.ROCKSDB_DEADLOCK_DETECT; +@@global.ROCKSDB_DEADLOCK_DETECT +0 +SET @@global.ROCKSDB_DEADLOCK_DETECT = @start_global_value; +SELECT @@global.ROCKSDB_DEADLOCK_DETECT; +@@global.ROCKSDB_DEADLOCK_DETECT +0 +SET @@session.ROCKSDB_DEADLOCK_DETECT = @start_session_value; +SELECT @@session.ROCKSDB_DEADLOCK_DETECT; +@@session.ROCKSDB_DEADLOCK_DETECT +0 +DROP TABLE valid_values; +DROP TABLE invalid_values; diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_print_snapshot_conflict_queries_basic.result b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_print_snapshot_conflict_queries_basic.result new file mode 100644 index 00000000000..02a4b4040d7 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_print_snapshot_conflict_queries_basic.result @@ -0,0 +1,64 @@ +CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO valid_values VALUES(1); +INSERT INTO valid_values VALUES(0); +INSERT INTO valid_values VALUES('on'); +CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO invalid_values VALUES('\'aaa\''); +INSERT INTO invalid_values VALUES('\'bbb\''); +SET @start_global_value = @@global.ROCKSDB_PRINT_SNAPSHOT_CONFLICT_QUERIES; +SELECT @start_global_value; +@start_global_value +0 +'# Setting to valid values in global scope#' +"Trying to set variable @@global.ROCKSDB_PRINT_SNAPSHOT_CONFLICT_QUERIES to 1" +SET @@global.ROCKSDB_PRINT_SNAPSHOT_CONFLICT_QUERIES = 1; +SELECT @@global.ROCKSDB_PRINT_SNAPSHOT_CONFLICT_QUERIES; +@@global.ROCKSDB_PRINT_SNAPSHOT_CONFLICT_QUERIES +1 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_PRINT_SNAPSHOT_CONFLICT_QUERIES = DEFAULT; +SELECT @@global.ROCKSDB_PRINT_SNAPSHOT_CONFLICT_QUERIES; +@@global.ROCKSDB_PRINT_SNAPSHOT_CONFLICT_QUERIES +0 +"Trying to set variable @@global.ROCKSDB_PRINT_SNAPSHOT_CONFLICT_QUERIES to 0" +SET @@global.ROCKSDB_PRINT_SNAPSHOT_CONFLICT_QUERIES = 0; +SELECT @@global.ROCKSDB_PRINT_SNAPSHOT_CONFLICT_QUERIES; +@@global.ROCKSDB_PRINT_SNAPSHOT_CONFLICT_QUERIES +0 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_PRINT_SNAPSHOT_CONFLICT_QUERIES = DEFAULT; +SELECT @@global.ROCKSDB_PRINT_SNAPSHOT_CONFLICT_QUERIES; +@@global.ROCKSDB_PRINT_SNAPSHOT_CONFLICT_QUERIES +0 +"Trying to set variable @@global.ROCKSDB_PRINT_SNAPSHOT_CONFLICT_QUERIES to on" +SET @@global.ROCKSDB_PRINT_SNAPSHOT_CONFLICT_QUERIES = on; +SELECT @@global.ROCKSDB_PRINT_SNAPSHOT_CONFLICT_QUERIES; +@@global.ROCKSDB_PRINT_SNAPSHOT_CONFLICT_QUERIES +1 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_PRINT_SNAPSHOT_CONFLICT_QUERIES = DEFAULT; +SELECT @@global.ROCKSDB_PRINT_SNAPSHOT_CONFLICT_QUERIES; +@@global.ROCKSDB_PRINT_SNAPSHOT_CONFLICT_QUERIES +0 +"Trying to set variable @@session.ROCKSDB_PRINT_SNAPSHOT_CONFLICT_QUERIES to 444. It should fail because it is not session." +SET @@session.ROCKSDB_PRINT_SNAPSHOT_CONFLICT_QUERIES = 444; +ERROR HY000: Variable 'rocksdb_print_snapshot_conflict_queries' is a GLOBAL variable and should be set with SET GLOBAL +'# Testing with invalid values in global scope #' +"Trying to set variable @@global.ROCKSDB_PRINT_SNAPSHOT_CONFLICT_QUERIES to 'aaa'" +SET @@global.ROCKSDB_PRINT_SNAPSHOT_CONFLICT_QUERIES = 'aaa'; +Got one of the listed errors +SELECT @@global.ROCKSDB_PRINT_SNAPSHOT_CONFLICT_QUERIES; +@@global.ROCKSDB_PRINT_SNAPSHOT_CONFLICT_QUERIES +0 +"Trying to set variable @@global.ROCKSDB_PRINT_SNAPSHOT_CONFLICT_QUERIES to 'bbb'" +SET @@global.ROCKSDB_PRINT_SNAPSHOT_CONFLICT_QUERIES = 'bbb'; +Got one of the listed errors +SELECT @@global.ROCKSDB_PRINT_SNAPSHOT_CONFLICT_QUERIES; +@@global.ROCKSDB_PRINT_SNAPSHOT_CONFLICT_QUERIES +0 +SET @@global.ROCKSDB_PRINT_SNAPSHOT_CONFLICT_QUERIES = @start_global_value; +SELECT @@global.ROCKSDB_PRINT_SNAPSHOT_CONFLICT_QUERIES; +@@global.ROCKSDB_PRINT_SNAPSHOT_CONFLICT_QUERIES +0 +DROP TABLE valid_values; +DROP TABLE invalid_values; diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_rpl_skip_tx_api_basic.test b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_rpl_skip_tx_api_basic.result index 5f6522e4488..5f6522e4488 100644 --- a/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_rpl_skip_tx_api_basic.test +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_rpl_skip_tx_api_basic.result diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_store_checksums_basic.result b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_store_checksums_basic.result deleted file mode 100644 index 904a0bc536e..00000000000 --- a/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_store_checksums_basic.result +++ /dev/null @@ -1,100 +0,0 @@ -CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam; -INSERT INTO valid_values VALUES(1); -INSERT INTO valid_values VALUES(0); -INSERT INTO valid_values VALUES('on'); -CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; -INSERT INTO invalid_values VALUES('\'aaa\''); -INSERT INTO invalid_values VALUES('\'bbb\''); -SET @start_global_value = @@global.ROCKSDB_STORE_CHECKSUMS; -SELECT @start_global_value; -@start_global_value -0 -SET @start_session_value = @@session.ROCKSDB_STORE_CHECKSUMS; -SELECT @start_session_value; -@start_session_value -0 -'# Setting to valid values in global scope#' -"Trying to set variable @@global.ROCKSDB_STORE_CHECKSUMS to 1" -SET @@global.ROCKSDB_STORE_CHECKSUMS = 1; -SELECT @@global.ROCKSDB_STORE_CHECKSUMS; -@@global.ROCKSDB_STORE_CHECKSUMS -1 -"Setting the global scope variable back to default" -SET @@global.ROCKSDB_STORE_CHECKSUMS = DEFAULT; -SELECT @@global.ROCKSDB_STORE_CHECKSUMS; -@@global.ROCKSDB_STORE_CHECKSUMS -0 -"Trying to set variable @@global.ROCKSDB_STORE_CHECKSUMS to 0" -SET @@global.ROCKSDB_STORE_CHECKSUMS = 0; -SELECT @@global.ROCKSDB_STORE_CHECKSUMS; -@@global.ROCKSDB_STORE_CHECKSUMS -0 -"Setting the global scope variable back to default" -SET @@global.ROCKSDB_STORE_CHECKSUMS = DEFAULT; -SELECT @@global.ROCKSDB_STORE_CHECKSUMS; -@@global.ROCKSDB_STORE_CHECKSUMS -0 -"Trying to set variable @@global.ROCKSDB_STORE_CHECKSUMS to on" -SET @@global.ROCKSDB_STORE_CHECKSUMS = on; -SELECT @@global.ROCKSDB_STORE_CHECKSUMS; -@@global.ROCKSDB_STORE_CHECKSUMS -1 -"Setting the global scope variable back to default" -SET @@global.ROCKSDB_STORE_CHECKSUMS = DEFAULT; -SELECT @@global.ROCKSDB_STORE_CHECKSUMS; -@@global.ROCKSDB_STORE_CHECKSUMS -0 -'# Setting to valid values in session scope#' -"Trying to set variable @@session.ROCKSDB_STORE_CHECKSUMS to 1" -SET @@session.ROCKSDB_STORE_CHECKSUMS = 1; -SELECT @@session.ROCKSDB_STORE_CHECKSUMS; -@@session.ROCKSDB_STORE_CHECKSUMS -1 -"Setting the session scope variable back to default" -SET @@session.ROCKSDB_STORE_CHECKSUMS = DEFAULT; -SELECT @@session.ROCKSDB_STORE_CHECKSUMS; -@@session.ROCKSDB_STORE_CHECKSUMS -0 -"Trying to set variable @@session.ROCKSDB_STORE_CHECKSUMS to 0" -SET @@session.ROCKSDB_STORE_CHECKSUMS = 0; -SELECT @@session.ROCKSDB_STORE_CHECKSUMS; -@@session.ROCKSDB_STORE_CHECKSUMS -0 -"Setting the session scope variable back to default" -SET @@session.ROCKSDB_STORE_CHECKSUMS = DEFAULT; -SELECT @@session.ROCKSDB_STORE_CHECKSUMS; -@@session.ROCKSDB_STORE_CHECKSUMS -0 -"Trying to set variable @@session.ROCKSDB_STORE_CHECKSUMS to on" -SET @@session.ROCKSDB_STORE_CHECKSUMS = on; -SELECT @@session.ROCKSDB_STORE_CHECKSUMS; -@@session.ROCKSDB_STORE_CHECKSUMS -1 -"Setting the session scope variable back to default" -SET @@session.ROCKSDB_STORE_CHECKSUMS = DEFAULT; -SELECT @@session.ROCKSDB_STORE_CHECKSUMS; -@@session.ROCKSDB_STORE_CHECKSUMS -0 -'# Testing with invalid values in global scope #' -"Trying to set variable @@global.ROCKSDB_STORE_CHECKSUMS to 'aaa'" -SET @@global.ROCKSDB_STORE_CHECKSUMS = 'aaa'; -Got one of the listed errors -SELECT @@global.ROCKSDB_STORE_CHECKSUMS; -@@global.ROCKSDB_STORE_CHECKSUMS -0 -"Trying to set variable @@global.ROCKSDB_STORE_CHECKSUMS to 'bbb'" -SET @@global.ROCKSDB_STORE_CHECKSUMS = 'bbb'; -Got one of the listed errors -SELECT @@global.ROCKSDB_STORE_CHECKSUMS; -@@global.ROCKSDB_STORE_CHECKSUMS -0 -SET @@global.ROCKSDB_STORE_CHECKSUMS = @start_global_value; -SELECT @@global.ROCKSDB_STORE_CHECKSUMS; -@@global.ROCKSDB_STORE_CHECKSUMS -0 -SET @@session.ROCKSDB_STORE_CHECKSUMS = @start_session_value; -SELECT @@session.ROCKSDB_STORE_CHECKSUMS; -@@session.ROCKSDB_STORE_CHECKSUMS -0 -DROP TABLE valid_values; -DROP TABLE invalid_values; diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_store_row_debug_checksums_basic.result b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_store_row_debug_checksums_basic.result new file mode 100644 index 00000000000..a838d660a91 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_store_row_debug_checksums_basic.result @@ -0,0 +1,100 @@ +CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO valid_values VALUES(1); +INSERT INTO valid_values VALUES(0); +INSERT INTO valid_values VALUES('on'); +CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO invalid_values VALUES('\'aaa\''); +INSERT INTO invalid_values VALUES('\'bbb\''); +SET @start_global_value = @@global.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS; +SELECT @start_global_value; +@start_global_value +0 +SET @start_session_value = @@session.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS; +SELECT @start_session_value; +@start_session_value +0 +'# Setting to valid values in global scope#' +"Trying to set variable @@global.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS to 1" +SET @@global.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS = 1; +SELECT @@global.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS; +@@global.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS +1 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS = DEFAULT; +SELECT @@global.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS; +@@global.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS +0 +"Trying to set variable @@global.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS to 0" +SET @@global.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS = 0; +SELECT @@global.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS; +@@global.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS +0 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS = DEFAULT; +SELECT @@global.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS; +@@global.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS +0 +"Trying to set variable @@global.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS to on" +SET @@global.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS = on; +SELECT @@global.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS; +@@global.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS +1 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS = DEFAULT; +SELECT @@global.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS; +@@global.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS +0 +'# Setting to valid values in session scope#' +"Trying to set variable @@session.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS to 1" +SET @@session.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS = 1; +SELECT @@session.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS; +@@session.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS +1 +"Setting the session scope variable back to default" +SET @@session.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS = DEFAULT; +SELECT @@session.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS; +@@session.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS +0 +"Trying to set variable @@session.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS to 0" +SET @@session.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS = 0; +SELECT @@session.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS; +@@session.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS +0 +"Setting the session scope variable back to default" +SET @@session.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS = DEFAULT; +SELECT @@session.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS; +@@session.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS +0 +"Trying to set variable @@session.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS to on" +SET @@session.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS = on; +SELECT @@session.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS; +@@session.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS +1 +"Setting the session scope variable back to default" +SET @@session.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS = DEFAULT; +SELECT @@session.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS; +@@session.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS +0 +'# Testing with invalid values in global scope #' +"Trying to set variable @@global.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS to 'aaa'" +SET @@global.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS = 'aaa'; +Got one of the listed errors +SELECT @@global.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS; +@@global.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS +0 +"Trying to set variable @@global.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS to 'bbb'" +SET @@global.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS = 'bbb'; +Got one of the listed errors +SELECT @@global.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS; +@@global.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS +0 +SET @@global.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS = @start_global_value; +SELECT @@global.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS; +@@global.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS +0 +SET @@session.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS = @start_session_value; +SELECT @@session.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS; +@@session.ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS +0 +DROP TABLE valid_values; +DROP TABLE invalid_values; diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_tmpdir_basic.result b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_tmpdir_basic.result new file mode 100644 index 00000000000..25b19ee56a4 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_tmpdir_basic.result @@ -0,0 +1,29 @@ +SET @start_global_value = @@global.rocksdb_tmpdir; +SELECT @start_global_value; +@start_global_value + +select @@session.rocksdb_tmpdir; +@@session.rocksdb_tmpdir + +show global variables like 'rocksdb_tmpdir'; +Variable_name Value +rocksdb_tmpdir +show session variables like 'rocksdb_tmpdir'; +Variable_name Value +rocksdb_tmpdir +select * from information_schema.global_variables where variable_name='rocksdb_tmpdir'; +VARIABLE_NAME VARIABLE_VALUE +ROCKSDB_TMPDIR +select * from information_schema.session_variables where variable_name='rocksdb_tmpdir'; +VARIABLE_NAME VARIABLE_VALUE +ROCKSDB_TMPDIR +set global rocksdb_tmpdir='value'; +set session rocksdb_tmpdir='value'; +set global rocksdb_tmpdir=1.1; +ERROR 42000: Incorrect argument type to variable 'rocksdb_tmpdir' +set global rocksdb_tmpdir=1e1; +ERROR 42000: Incorrect argument type to variable 'rocksdb_tmpdir' +SET @@global.rocksdb_tmpdir = @start_global_value; +SELECT @@global.rocksdb_tmpdir; +@@global.rocksdb_tmpdir + diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_trace_sst_api_basic.result b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_trace_sst_api_basic.result new file mode 100644 index 00000000000..d4ffde80001 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_trace_sst_api_basic.result @@ -0,0 +1,100 @@ +CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO valid_values VALUES(1); +INSERT INTO valid_values VALUES(0); +INSERT INTO valid_values VALUES('on'); +CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO invalid_values VALUES('\'aaa\''); +INSERT INTO invalid_values VALUES('\'bbb\''); +SET @start_global_value = @@global.ROCKSDB_TRACE_SST_API; +SELECT @start_global_value; +@start_global_value +0 +SET @start_session_value = @@session.ROCKSDB_TRACE_SST_API; +SELECT @start_session_value; +@start_session_value +0 +'# Setting to valid values in global scope#' +"Trying to set variable @@global.ROCKSDB_TRACE_SST_API to 1" +SET @@global.ROCKSDB_TRACE_SST_API = 1; +SELECT @@global.ROCKSDB_TRACE_SST_API; +@@global.ROCKSDB_TRACE_SST_API +1 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_TRACE_SST_API = DEFAULT; +SELECT @@global.ROCKSDB_TRACE_SST_API; +@@global.ROCKSDB_TRACE_SST_API +0 +"Trying to set variable @@global.ROCKSDB_TRACE_SST_API to 0" +SET @@global.ROCKSDB_TRACE_SST_API = 0; +SELECT @@global.ROCKSDB_TRACE_SST_API; +@@global.ROCKSDB_TRACE_SST_API +0 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_TRACE_SST_API = DEFAULT; +SELECT @@global.ROCKSDB_TRACE_SST_API; +@@global.ROCKSDB_TRACE_SST_API +0 +"Trying to set variable @@global.ROCKSDB_TRACE_SST_API to on" +SET @@global.ROCKSDB_TRACE_SST_API = on; +SELECT @@global.ROCKSDB_TRACE_SST_API; +@@global.ROCKSDB_TRACE_SST_API +1 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_TRACE_SST_API = DEFAULT; +SELECT @@global.ROCKSDB_TRACE_SST_API; +@@global.ROCKSDB_TRACE_SST_API +0 +'# Setting to valid values in session scope#' +"Trying to set variable @@session.ROCKSDB_TRACE_SST_API to 1" +SET @@session.ROCKSDB_TRACE_SST_API = 1; +SELECT @@session.ROCKSDB_TRACE_SST_API; +@@session.ROCKSDB_TRACE_SST_API +1 +"Setting the session scope variable back to default" +SET @@session.ROCKSDB_TRACE_SST_API = DEFAULT; +SELECT @@session.ROCKSDB_TRACE_SST_API; +@@session.ROCKSDB_TRACE_SST_API +0 +"Trying to set variable @@session.ROCKSDB_TRACE_SST_API to 0" +SET @@session.ROCKSDB_TRACE_SST_API = 0; +SELECT @@session.ROCKSDB_TRACE_SST_API; +@@session.ROCKSDB_TRACE_SST_API +0 +"Setting the session scope variable back to default" +SET @@session.ROCKSDB_TRACE_SST_API = DEFAULT; +SELECT @@session.ROCKSDB_TRACE_SST_API; +@@session.ROCKSDB_TRACE_SST_API +0 +"Trying to set variable @@session.ROCKSDB_TRACE_SST_API to on" +SET @@session.ROCKSDB_TRACE_SST_API = on; +SELECT @@session.ROCKSDB_TRACE_SST_API; +@@session.ROCKSDB_TRACE_SST_API +1 +"Setting the session scope variable back to default" +SET @@session.ROCKSDB_TRACE_SST_API = DEFAULT; +SELECT @@session.ROCKSDB_TRACE_SST_API; +@@session.ROCKSDB_TRACE_SST_API +0 +'# Testing with invalid values in global scope #' +"Trying to set variable @@global.ROCKSDB_TRACE_SST_API to 'aaa'" +SET @@global.ROCKSDB_TRACE_SST_API = 'aaa'; +Got one of the listed errors +SELECT @@global.ROCKSDB_TRACE_SST_API; +@@global.ROCKSDB_TRACE_SST_API +0 +"Trying to set variable @@global.ROCKSDB_TRACE_SST_API to 'bbb'" +SET @@global.ROCKSDB_TRACE_SST_API = 'bbb'; +Got one of the listed errors +SELECT @@global.ROCKSDB_TRACE_SST_API; +@@global.ROCKSDB_TRACE_SST_API +0 +SET @@global.ROCKSDB_TRACE_SST_API = @start_global_value; +SELECT @@global.ROCKSDB_TRACE_SST_API; +@@global.ROCKSDB_TRACE_SST_API +0 +SET @@session.ROCKSDB_TRACE_SST_API = @start_session_value; +SELECT @@session.ROCKSDB_TRACE_SST_API; +@@session.ROCKSDB_TRACE_SST_API +0 +DROP TABLE valid_values; +DROP TABLE invalid_values; diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_use_direct_reads_basic.result b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_use_direct_reads_basic.result new file mode 100644 index 00000000000..ec36c309dca --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_use_direct_reads_basic.result @@ -0,0 +1,7 @@ +SET @start_global_value = @@global.ROCKSDB_USE_DIRECT_READS; +SELECT @start_global_value; +@start_global_value +0 +"Trying to set variable @@global.ROCKSDB_USE_DIRECT_READS to 444. It should fail because it is readonly." +SET @@global.ROCKSDB_USE_DIRECT_READS = 444; +ERROR HY000: Variable 'rocksdb_use_direct_reads' is a read only variable diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_use_direct_writes_basic.result b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_use_direct_writes_basic.result new file mode 100644 index 00000000000..4cc787e4586 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_use_direct_writes_basic.result @@ -0,0 +1,7 @@ +SET @start_global_value = @@global.ROCKSDB_USE_DIRECT_WRITES; +SELECT @start_global_value; +@start_global_value +0 +"Trying to set variable @@global.ROCKSDB_USE_DIRECT_WRITES to 444. It should fail because it is readonly." +SET @@global.ROCKSDB_USE_DIRECT_WRITES = 444; +ERROR HY000: Variable 'rocksdb_use_direct_writes' is a read only variable diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_verify_checksums_basic.result b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_verify_checksums_basic.result deleted file mode 100644 index da4cae7a151..00000000000 --- a/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_verify_checksums_basic.result +++ /dev/null @@ -1,100 +0,0 @@ -CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam; -INSERT INTO valid_values VALUES(1); -INSERT INTO valid_values VALUES(0); -INSERT INTO valid_values VALUES('on'); -CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; -INSERT INTO invalid_values VALUES('\'aaa\''); -INSERT INTO invalid_values VALUES('\'bbb\''); -SET @start_global_value = @@global.ROCKSDB_VERIFY_CHECKSUMS; -SELECT @start_global_value; -@start_global_value -0 -SET @start_session_value = @@session.ROCKSDB_VERIFY_CHECKSUMS; -SELECT @start_session_value; -@start_session_value -0 -'# Setting to valid values in global scope#' -"Trying to set variable @@global.ROCKSDB_VERIFY_CHECKSUMS to 1" -SET @@global.ROCKSDB_VERIFY_CHECKSUMS = 1; -SELECT @@global.ROCKSDB_VERIFY_CHECKSUMS; -@@global.ROCKSDB_VERIFY_CHECKSUMS -1 -"Setting the global scope variable back to default" -SET @@global.ROCKSDB_VERIFY_CHECKSUMS = DEFAULT; -SELECT @@global.ROCKSDB_VERIFY_CHECKSUMS; -@@global.ROCKSDB_VERIFY_CHECKSUMS -0 -"Trying to set variable @@global.ROCKSDB_VERIFY_CHECKSUMS to 0" -SET @@global.ROCKSDB_VERIFY_CHECKSUMS = 0; -SELECT @@global.ROCKSDB_VERIFY_CHECKSUMS; -@@global.ROCKSDB_VERIFY_CHECKSUMS -0 -"Setting the global scope variable back to default" -SET @@global.ROCKSDB_VERIFY_CHECKSUMS = DEFAULT; -SELECT @@global.ROCKSDB_VERIFY_CHECKSUMS; -@@global.ROCKSDB_VERIFY_CHECKSUMS -0 -"Trying to set variable @@global.ROCKSDB_VERIFY_CHECKSUMS to on" -SET @@global.ROCKSDB_VERIFY_CHECKSUMS = on; -SELECT @@global.ROCKSDB_VERIFY_CHECKSUMS; -@@global.ROCKSDB_VERIFY_CHECKSUMS -1 -"Setting the global scope variable back to default" -SET @@global.ROCKSDB_VERIFY_CHECKSUMS = DEFAULT; -SELECT @@global.ROCKSDB_VERIFY_CHECKSUMS; -@@global.ROCKSDB_VERIFY_CHECKSUMS -0 -'# Setting to valid values in session scope#' -"Trying to set variable @@session.ROCKSDB_VERIFY_CHECKSUMS to 1" -SET @@session.ROCKSDB_VERIFY_CHECKSUMS = 1; -SELECT @@session.ROCKSDB_VERIFY_CHECKSUMS; -@@session.ROCKSDB_VERIFY_CHECKSUMS -1 -"Setting the session scope variable back to default" -SET @@session.ROCKSDB_VERIFY_CHECKSUMS = DEFAULT; -SELECT @@session.ROCKSDB_VERIFY_CHECKSUMS; -@@session.ROCKSDB_VERIFY_CHECKSUMS -0 -"Trying to set variable @@session.ROCKSDB_VERIFY_CHECKSUMS to 0" -SET @@session.ROCKSDB_VERIFY_CHECKSUMS = 0; -SELECT @@session.ROCKSDB_VERIFY_CHECKSUMS; -@@session.ROCKSDB_VERIFY_CHECKSUMS -0 -"Setting the session scope variable back to default" -SET @@session.ROCKSDB_VERIFY_CHECKSUMS = DEFAULT; -SELECT @@session.ROCKSDB_VERIFY_CHECKSUMS; -@@session.ROCKSDB_VERIFY_CHECKSUMS -0 -"Trying to set variable @@session.ROCKSDB_VERIFY_CHECKSUMS to on" -SET @@session.ROCKSDB_VERIFY_CHECKSUMS = on; -SELECT @@session.ROCKSDB_VERIFY_CHECKSUMS; -@@session.ROCKSDB_VERIFY_CHECKSUMS -1 -"Setting the session scope variable back to default" -SET @@session.ROCKSDB_VERIFY_CHECKSUMS = DEFAULT; -SELECT @@session.ROCKSDB_VERIFY_CHECKSUMS; -@@session.ROCKSDB_VERIFY_CHECKSUMS -0 -'# Testing with invalid values in global scope #' -"Trying to set variable @@global.ROCKSDB_VERIFY_CHECKSUMS to 'aaa'" -SET @@global.ROCKSDB_VERIFY_CHECKSUMS = 'aaa'; -Got one of the listed errors -SELECT @@global.ROCKSDB_VERIFY_CHECKSUMS; -@@global.ROCKSDB_VERIFY_CHECKSUMS -0 -"Trying to set variable @@global.ROCKSDB_VERIFY_CHECKSUMS to 'bbb'" -SET @@global.ROCKSDB_VERIFY_CHECKSUMS = 'bbb'; -Got one of the listed errors -SELECT @@global.ROCKSDB_VERIFY_CHECKSUMS; -@@global.ROCKSDB_VERIFY_CHECKSUMS -0 -SET @@global.ROCKSDB_VERIFY_CHECKSUMS = @start_global_value; -SELECT @@global.ROCKSDB_VERIFY_CHECKSUMS; -@@global.ROCKSDB_VERIFY_CHECKSUMS -0 -SET @@session.ROCKSDB_VERIFY_CHECKSUMS = @start_session_value; -SELECT @@session.ROCKSDB_VERIFY_CHECKSUMS; -@@session.ROCKSDB_VERIFY_CHECKSUMS -0 -DROP TABLE valid_values; -DROP TABLE invalid_values; diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_verify_row_debug_checksums_basic.result b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_verify_row_debug_checksums_basic.result new file mode 100644 index 00000000000..ad71c8909a6 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_verify_row_debug_checksums_basic.result @@ -0,0 +1,100 @@ +CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO valid_values VALUES(1); +INSERT INTO valid_values VALUES(0); +INSERT INTO valid_values VALUES('on'); +CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO invalid_values VALUES('\'aaa\''); +INSERT INTO invalid_values VALUES('\'bbb\''); +SET @start_global_value = @@global.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS; +SELECT @start_global_value; +@start_global_value +0 +SET @start_session_value = @@session.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS; +SELECT @start_session_value; +@start_session_value +0 +'# Setting to valid values in global scope#' +"Trying to set variable @@global.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS to 1" +SET @@global.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS = 1; +SELECT @@global.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS; +@@global.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS +1 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS = DEFAULT; +SELECT @@global.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS; +@@global.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS +0 +"Trying to set variable @@global.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS to 0" +SET @@global.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS = 0; +SELECT @@global.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS; +@@global.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS +0 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS = DEFAULT; +SELECT @@global.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS; +@@global.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS +0 +"Trying to set variable @@global.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS to on" +SET @@global.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS = on; +SELECT @@global.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS; +@@global.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS +1 +"Setting the global scope variable back to default" +SET @@global.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS = DEFAULT; +SELECT @@global.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS; +@@global.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS +0 +'# Setting to valid values in session scope#' +"Trying to set variable @@session.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS to 1" +SET @@session.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS = 1; +SELECT @@session.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS; +@@session.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS +1 +"Setting the session scope variable back to default" +SET @@session.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS = DEFAULT; +SELECT @@session.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS; +@@session.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS +0 +"Trying to set variable @@session.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS to 0" +SET @@session.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS = 0; +SELECT @@session.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS; +@@session.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS +0 +"Setting the session scope variable back to default" +SET @@session.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS = DEFAULT; +SELECT @@session.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS; +@@session.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS +0 +"Trying to set variable @@session.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS to on" +SET @@session.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS = on; +SELECT @@session.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS; +@@session.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS +1 +"Setting the session scope variable back to default" +SET @@session.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS = DEFAULT; +SELECT @@session.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS; +@@session.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS +0 +'# Testing with invalid values in global scope #' +"Trying to set variable @@global.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS to 'aaa'" +SET @@global.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS = 'aaa'; +Got one of the listed errors +SELECT @@global.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS; +@@global.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS +0 +"Trying to set variable @@global.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS to 'bbb'" +SET @@global.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS = 'bbb'; +Got one of the listed errors +SELECT @@global.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS; +@@global.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS +0 +SET @@global.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS = @start_global_value; +SELECT @@global.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS; +@@global.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS +0 +SET @@session.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS = @start_session_value; +SELECT @@session.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS; +@@session.ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS +0 +DROP TABLE valid_values; +DROP TABLE invalid_values; diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_deadlock_detect_basic.test b/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_deadlock_detect_basic.test new file mode 100644 index 00000000000..aa532fdc1be --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_deadlock_detect_basic.test @@ -0,0 +1,20 @@ +--source include/have_rocksdb.inc + +CREATE TABLE valid_values (value varchar(255)); +INSERT INTO valid_values VALUES(1); +INSERT INTO valid_values VALUES(0); +INSERT INTO valid_values VALUES('on'); +INSERT INTO valid_values VALUES('off'); + +CREATE TABLE invalid_values (value varchar(255)); +INSERT INTO invalid_values VALUES('\'aaa\''); +INSERT INTO invalid_values VALUES('\'bbb\''); + +--let $sys_var=ROCKSDB_DEADLOCK_DETECT +--let $read_only=0 +--let $session=1 +--let $sticky=1 +--source suite/sys_vars/inc/rocksdb_sys_var.inc + +DROP TABLE valid_values; +DROP TABLE invalid_values; diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_print_snapshot_conflict_queries_basic.test b/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_print_snapshot_conflict_queries_basic.test new file mode 100644 index 00000000000..92a419a8636 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_print_snapshot_conflict_queries_basic.test @@ -0,0 +1,18 @@ +--source include/have_rocksdb.inc + +CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO valid_values VALUES(1); +INSERT INTO valid_values VALUES(0); +INSERT INTO valid_values VALUES('on'); + +CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO invalid_values VALUES('\'aaa\''); +INSERT INTO invalid_values VALUES('\'bbb\''); + +--let $sys_var=ROCKSDB_PRINT_SNAPSHOT_CONFLICT_QUERIES +--let $read_only=0 +--let $session=0 +--source suite/sys_vars/inc/rocksdb_sys_var.inc + +DROP TABLE valid_values; +DROP TABLE invalid_values; diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_verify_checksums_basic.test b/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_store_row_debug_checksums_basic.test index d8c9c559703..e9c04bcc45e 100644 --- a/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_verify_checksums_basic.test +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_store_row_debug_checksums_basic.test @@ -9,7 +9,7 @@ CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; INSERT INTO invalid_values VALUES('\'aaa\''); INSERT INTO invalid_values VALUES('\'bbb\''); ---let $sys_var=ROCKSDB_VERIFY_CHECKSUMS +--let $sys_var=ROCKSDB_STORE_ROW_DEBUG_CHECKSUMS --let $read_only=0 --let $session=1 --source suite/sys_vars/inc/rocksdb_sys_var.inc diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_tmpdir_basic.test b/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_tmpdir_basic.test new file mode 100644 index 00000000000..8865914dd18 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_tmpdir_basic.test @@ -0,0 +1,38 @@ +--source include/have_rocksdb.inc + +SET @start_global_value = @@global.rocksdb_tmpdir; +SELECT @start_global_value; + +# +# exists as global and session +# +select @@session.rocksdb_tmpdir; + +show global variables like 'rocksdb_tmpdir'; +show session variables like 'rocksdb_tmpdir'; + +select * from information_schema.global_variables where variable_name='rocksdb_tmpdir'; +select * from information_schema.session_variables where variable_name='rocksdb_tmpdir'; + +# +# Show that it is writable +# + +set global rocksdb_tmpdir='value'; +set session rocksdb_tmpdir='value'; + +# +# incorrect types +# +--error ER_WRONG_TYPE_FOR_VAR +set global rocksdb_tmpdir=1.1; +--error ER_WRONG_TYPE_FOR_VAR +set global rocksdb_tmpdir=1e1; + +# +# Cleanup +# + +SET @@global.rocksdb_tmpdir = @start_global_value; +SELECT @@global.rocksdb_tmpdir; + diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_store_checksums_basic.test b/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_trace_sst_api_basic.test index 023b6420b96..83a0faaffe6 100644 --- a/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_store_checksums_basic.test +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_trace_sst_api_basic.test @@ -9,7 +9,7 @@ CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; INSERT INTO invalid_values VALUES('\'aaa\''); INSERT INTO invalid_values VALUES('\'bbb\''); ---let $sys_var=ROCKSDB_STORE_CHECKSUMS +--let $sys_var=ROCKSDB_TRACE_SST_API --let $read_only=0 --let $session=1 --source suite/sys_vars/inc/rocksdb_sys_var.inc diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_allow_os_buffer_basic.test b/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_use_direct_reads_basic.test index c38d0c7b210..b730eca6f3b 100644 --- a/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_allow_os_buffer_basic.test +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_use_direct_reads_basic.test @@ -1,6 +1,6 @@ --source include/have_rocksdb.inc ---let $sys_var=ROCKSDB_ALLOW_OS_BUFFER +--let $sys_var=ROCKSDB_USE_DIRECT_READS --let $read_only=1 --let $session=0 --source suite/sys_vars/inc/rocksdb_sys_var.inc diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_use_direct_writes_basic.test b/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_use_direct_writes_basic.test new file mode 100644 index 00000000000..2abb2478d82 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_use_direct_writes_basic.test @@ -0,0 +1,6 @@ +--source include/have_rocksdb.inc + +--let $sys_var=ROCKSDB_USE_DIRECT_WRITES +--let $read_only=1 +--let $session=0 +--source suite/sys_vars/inc/rocksdb_sys_var.inc diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_verify_row_debug_checksums_basic.test b/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_verify_row_debug_checksums_basic.test new file mode 100644 index 00000000000..cc35fdb0345 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_verify_row_debug_checksums_basic.test @@ -0,0 +1,18 @@ +--source include/have_rocksdb.inc + +CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO valid_values VALUES(1); +INSERT INTO valid_values VALUES(0); +INSERT INTO valid_values VALUES('on'); + +CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam; +INSERT INTO invalid_values VALUES('\'aaa\''); +INSERT INTO invalid_values VALUES('\'bbb\''); + +--let $sys_var=ROCKSDB_VERIFY_ROW_DEBUG_CHECKSUMS +--let $read_only=0 +--let $session=1 +--source suite/sys_vars/inc/rocksdb_sys_var.inc + +DROP TABLE valid_values; +DROP TABLE invalid_values; diff --git a/storage/rocksdb/properties_collector.cc b/storage/rocksdb/properties_collector.cc index fc8cd97fc72..e5b5b25792d 100644 --- a/storage/rocksdb/properties_collector.cc +++ b/storage/rocksdb/properties_collector.cc @@ -42,10 +42,10 @@ std::atomic<uint64_t> rocksdb_num_sst_entry_other(0); my_bool rocksdb_compaction_sequential_deletes_count_sd = false; Rdb_tbl_prop_coll::Rdb_tbl_prop_coll( - Rdb_ddl_manager* ddl_manager, - Rdb_compact_params params, - uint32_t cf_id, - const uint8_t table_stats_sampling_pct + Rdb_ddl_manager* const ddl_manager, + const Rdb_compact_params ¶ms, + const uint32_t &cf_id, + const uint8_t &table_stats_sampling_pct ) : m_cf_id(cf_id), m_ddl_manager(ddl_manager), @@ -56,6 +56,8 @@ Rdb_tbl_prop_coll::Rdb_tbl_prop_coll( m_seed(time(nullptr)), m_card_adj_extra(1.) { + DBUG_ASSERT(ddl_manager != nullptr); + // We need to adjust the index cardinality numbers based on the sampling // rate so that the output of "SHOW INDEX" command will reflect reality // more closely. It will still be an approximation, just a better one. @@ -97,9 +99,9 @@ void Rdb_tbl_prop_coll::AdjustDeletedRows(rocksdb::EntryType type) // m_rows % m_deleted_rows_window.size() // m_deleted_rows is the current number of 1's in the vector // --update the counter for the element which will be overridden - bool is_delete= (type == rocksdb::kEntryDelete || - (type == rocksdb::kEntrySingleDelete && - rocksdb_compaction_sequential_deletes_count_sd)); + const bool is_delete= (type == rocksdb::kEntryDelete || + (type == rocksdb::kEntrySingleDelete && + rocksdb_compaction_sequential_deletes_count_sd)); // Only make changes if the value at the current position needs to change if (is_delete != m_deleted_rows_window[m_window_pos]) @@ -167,9 +169,9 @@ Rdb_index_stats* Rdb_tbl_prop_coll::AccessStats( void Rdb_tbl_prop_coll::CollectStatsForRow( const rocksdb::Slice& key, const rocksdb::Slice& value, - rocksdb::EntryType type, uint64_t file_size) + const rocksdb::EntryType &type, const uint64_t &file_size) { - auto stats = AccessStats(key); + const auto stats = AccessStats(key); stats->m_data_size += key.size()+value.size(); @@ -241,7 +243,7 @@ const char* Rdb_tbl_prop_coll::INDEXSTATS_KEY = "__indexstats__"; */ rocksdb::Status Rdb_tbl_prop_coll::Finish( - rocksdb::UserCollectedProperties* properties + rocksdb::UserCollectedProperties* const properties ) { uint64_t num_sst_entry_put = 0; uint64_t num_sst_entry_delete = 0; @@ -249,6 +251,8 @@ Rdb_tbl_prop_coll::Finish( uint64_t num_sst_entry_merge = 0; uint64_t num_sst_entry_other = 0; + DBUG_ASSERT(properties != nullptr); + for (auto it = m_stats.begin(); it != m_stats.end(); it++) { num_sst_entry_put += it->m_rows; @@ -303,7 +307,7 @@ bool Rdb_tbl_prop_coll::ShouldCollectStats() { return true; } - int val = rand_r(&m_seed) % + const int val = rand_r(&m_seed) % (RDB_TBL_STATS_SAMPLE_PCT_MAX - RDB_TBL_STATS_SAMPLE_PCT_MIN + 1) + RDB_TBL_STATS_SAMPLE_PCT_MIN; @@ -377,11 +381,11 @@ Rdb_tbl_prop_coll::GetReadableStats( void Rdb_tbl_prop_coll::read_stats_from_tbl_props( const std::shared_ptr<const rocksdb::TableProperties>& table_props, - std::vector<Rdb_index_stats>* out_stats_vector) + std::vector<Rdb_index_stats>* const out_stats_vector) { DBUG_ASSERT(out_stats_vector != nullptr); const auto& user_properties = table_props->user_collected_properties; - auto it2 = user_properties.find(std::string(INDEXSTATS_KEY)); + const auto it2 = user_properties.find(std::string(INDEXSTATS_KEY)); if (it2 != user_properties.end()) { auto result __attribute__((__unused__)) = @@ -400,7 +404,7 @@ std::string Rdb_index_stats::materialize( { String ret; rdb_netstr_append_uint16(&ret, INDEX_STATS_VERSION_ENTRY_TYPES); - for (auto i : stats) { + for (const auto &i : stats) { rdb_netstr_append_uint32(&ret, i.m_gl_index_id.cf_id); rdb_netstr_append_uint32(&ret, i.m_gl_index_id.index_id); DBUG_ASSERT(sizeof i.m_data_size <= 8); @@ -412,8 +416,8 @@ std::string Rdb_index_stats::materialize( rdb_netstr_append_uint64(&ret, i.m_entry_single_deletes); rdb_netstr_append_uint64(&ret, i.m_entry_merges); rdb_netstr_append_uint64(&ret, i.m_entry_others); - for (auto num_keys : i.m_distinct_keys_per_prefix) { - float upd_num_keys = num_keys * card_adj_extra; + for (const auto &num_keys : i.m_distinct_keys_per_prefix) { + const float upd_num_keys = num_keys * card_adj_extra; rdb_netstr_append_uint64(&ret, static_cast<int64_t>(upd_num_keys)); } } @@ -428,10 +432,10 @@ std::string Rdb_index_stats::materialize( @return 0 if completes successfully */ int Rdb_index_stats::unmaterialize( - const std::string& s, std::vector<Rdb_index_stats>* ret) + const std::string& s, std::vector<Rdb_index_stats>* const ret) { const uchar* p= rdb_std_str_to_uchar_ptr(s); - const uchar* p2= p + s.size(); + const uchar* const p2= p + s.size(); DBUG_ASSERT(ret != nullptr); @@ -440,7 +444,7 @@ int Rdb_index_stats::unmaterialize( return 1; } - int version= rdb_netbuf_read_uint16(&p); + const int version= rdb_netbuf_read_uint16(&p); Rdb_index_stats stats; // Make sure version is within supported range. if (version < INDEX_STATS_VERSION_INITIAL || @@ -503,10 +507,13 @@ int Rdb_index_stats::unmaterialize( for the index based on stats for each sst */ void Rdb_index_stats::merge( - const Rdb_index_stats& s, bool increment, int64_t estimated_data_len) + const Rdb_index_stats& s, const bool &increment, + const int64_t &estimated_data_len) { std::size_t i; + DBUG_ASSERT(estimated_data_len >= 0); + m_gl_index_id = s.m_gl_index_id; if (m_distinct_keys_per_prefix.size() < s.m_distinct_keys_per_prefix.size()) { diff --git a/storage/rocksdb/properties_collector.h b/storage/rocksdb/properties_collector.h index 1781d8a26c6..b0f8e5deb59 100644 --- a/storage/rocksdb/properties_collector.h +++ b/storage/rocksdb/properties_collector.h @@ -63,7 +63,7 @@ struct Rdb_index_stats static std::string materialize(const std::vector<Rdb_index_stats>& stats, const float card_adj_extra); static int unmaterialize(const std::string& s, - std::vector<Rdb_index_stats>* ret); + std::vector<Rdb_index_stats>* const ret); Rdb_index_stats() : Rdb_index_stats({0, 0}) {} explicit Rdb_index_stats(GL_INDEX_ID gl_index_id) : @@ -76,8 +76,8 @@ struct Rdb_index_stats m_entry_merges(0), m_entry_others(0) {} - void merge(const Rdb_index_stats& s, bool increment = true, - int64_t estimated_data_len = 0); + void merge(const Rdb_index_stats& s, const bool &increment = true, + const int64_t &estimated_data_len = 0); }; @@ -85,10 +85,10 @@ class Rdb_tbl_prop_coll : public rocksdb::TablePropertiesCollector { public: Rdb_tbl_prop_coll( - Rdb_ddl_manager* ddl_manager, - Rdb_compact_params params, - uint32_t cf_id, - const uint8_t table_stats_sampling_pct + Rdb_ddl_manager* const ddl_manager, + const Rdb_compact_params ¶ms, + const uint32_t &cf_id, + const uint8_t &table_stats_sampling_pct ); /* @@ -124,13 +124,14 @@ class Rdb_tbl_prop_coll : public rocksdb::TablePropertiesCollector bool ShouldCollectStats(); void CollectStatsForRow(const rocksdb::Slice& key, - const rocksdb::Slice& value, rocksdb::EntryType type, uint64_t file_size); + const rocksdb::Slice& value, const rocksdb::EntryType &type, + const uint64_t &file_size); Rdb_index_stats* AccessStats(const rocksdb::Slice& key); void AdjustDeletedRows(rocksdb::EntryType type); private: uint32_t m_cf_id; - std::shared_ptr<Rdb_key_def> m_keydef; + std::shared_ptr<const Rdb_key_def> m_keydef; Rdb_ddl_manager* m_ddl_manager; std::vector<Rdb_index_stats> m_stats; Rdb_index_stats* m_last_stats; @@ -153,6 +154,9 @@ class Rdb_tbl_prop_coll : public rocksdb::TablePropertiesCollector class Rdb_tbl_prop_coll_factory : public rocksdb::TablePropertiesCollectorFactory { public: + Rdb_tbl_prop_coll_factory(const Rdb_tbl_prop_coll_factory&) = delete; + Rdb_tbl_prop_coll_factory& operator=(const Rdb_tbl_prop_coll_factory&) = delete; + explicit Rdb_tbl_prop_coll_factory(Rdb_ddl_manager* ddl_manager) : m_ddl_manager(ddl_manager) { } @@ -177,12 +181,12 @@ class Rdb_tbl_prop_coll_factory m_params = params; } - void SetTableStatsSamplingPct(const uint8_t table_stats_sampling_pct) { + void SetTableStatsSamplingPct(const uint8_t &table_stats_sampling_pct) { m_table_stats_sampling_pct = table_stats_sampling_pct; } private: - Rdb_ddl_manager* m_ddl_manager; + Rdb_ddl_manager* const m_ddl_manager; Rdb_compact_params m_params; uint8_t m_table_stats_sampling_pct; }; diff --git a/storage/rocksdb/rdb_buff.h b/storage/rocksdb/rdb_buff.h index d29f365ba31..deb718e88c9 100644 --- a/storage/rocksdb/rdb_buff.h +++ b/storage/rocksdb/rdb_buff.h @@ -27,7 +27,8 @@ namespace myrocks { ("netstr") which stores data in Network Byte Order (Big Endian). */ -inline void rdb_netstr_append_uint64(my_core::String *out_netstr, uint64 val) +inline void rdb_netstr_append_uint64(my_core::String* const out_netstr, + const uint64 &val) { DBUG_ASSERT(out_netstr != nullptr); @@ -37,7 +38,8 @@ inline void rdb_netstr_append_uint64(my_core::String *out_netstr, uint64 val) out_netstr->append(reinterpret_cast<char*>(&net_val), sizeof(net_val)); } -inline void rdb_netstr_append_uint32(my_core::String *out_netstr, uint32 val) +inline void rdb_netstr_append_uint32(my_core::String* const out_netstr, + const uint32 &val) { DBUG_ASSERT(out_netstr != nullptr); @@ -47,7 +49,8 @@ inline void rdb_netstr_append_uint32(my_core::String *out_netstr, uint32 val) out_netstr->append(reinterpret_cast<char*>(&net_val), sizeof(net_val)); } -inline void rdb_netstr_append_uint16(my_core::String *out_netstr, uint16 val) +inline void rdb_netstr_append_uint16(my_core::String* const out_netstr, + const uint16 &val) { DBUG_ASSERT(out_netstr != nullptr); @@ -62,15 +65,17 @@ inline void rdb_netstr_append_uint16(my_core::String *out_netstr, uint16 val) Basic network buffer ("netbuf") write helper functions. */ -inline void rdb_netbuf_store_uint64(uchar *dst_netbuf, uint64 n) +inline void rdb_netbuf_store_uint64(uchar* const dst_netbuf, const uint64 &n) { + DBUG_ASSERT(dst_netbuf != nullptr); + // Convert from host byte order (usually Little Endian) to network byte order // (Big Endian). uint64 net_val= htobe64(n); memcpy(dst_netbuf, &net_val, sizeof(net_val)); } -inline void rdb_netbuf_store_uint32(uchar *dst_netbuf, uint32 n) +inline void rdb_netbuf_store_uint32(uchar* const dst_netbuf, const uint32 &n) { DBUG_ASSERT(dst_netbuf != nullptr); @@ -80,7 +85,7 @@ inline void rdb_netbuf_store_uint32(uchar *dst_netbuf, uint32 n) memcpy(dst_netbuf, &net_val, sizeof(net_val)); } -inline void rdb_netbuf_store_uint16(uchar *dst_netbuf, uint16 n) +inline void rdb_netbuf_store_uint16(uchar* const dst_netbuf, const uint16 &n) { DBUG_ASSERT(dst_netbuf != nullptr); @@ -90,14 +95,15 @@ inline void rdb_netbuf_store_uint16(uchar *dst_netbuf, uint16 n) memcpy(dst_netbuf, &net_val, sizeof(net_val)); } -inline void rdb_netbuf_store_byte(uchar *dst_netbuf, uchar c) +inline void rdb_netbuf_store_byte(uchar* const dst_netbuf, const uchar &c) { DBUG_ASSERT(dst_netbuf != nullptr); *dst_netbuf= c; } -inline void rdb_netbuf_store_index(uchar *dst_netbuf, uint32 number) +inline void rdb_netbuf_store_index(uchar* const dst_netbuf, + const uint32 &number) { DBUG_ASSERT(dst_netbuf != nullptr); @@ -110,7 +116,7 @@ inline void rdb_netbuf_store_index(uchar *dst_netbuf, uint32 number) machine byte order (usually Little Endian). */ -inline uint64 rdb_netbuf_to_uint64(const uchar *netbuf) +inline uint64 rdb_netbuf_to_uint64(const uchar* const netbuf) { DBUG_ASSERT(netbuf != nullptr); @@ -122,7 +128,7 @@ inline uint64 rdb_netbuf_to_uint64(const uchar *netbuf) return be64toh(net_val); } -inline uint32 rdb_netbuf_to_uint32(const uchar *netbuf) +inline uint32 rdb_netbuf_to_uint32(const uchar* const netbuf) { DBUG_ASSERT(netbuf != nullptr); @@ -134,7 +140,7 @@ inline uint32 rdb_netbuf_to_uint32(const uchar *netbuf) return be32toh(net_val); } -inline uint16 rdb_netbuf_to_uint16(const uchar *netbuf) +inline uint16 rdb_netbuf_to_uint16(const uchar* const netbuf) { DBUG_ASSERT(netbuf != nullptr); @@ -146,7 +152,7 @@ inline uint16 rdb_netbuf_to_uint16(const uchar *netbuf) return be16toh(net_val); } -inline uchar rdb_netbuf_to_byte(const uchar* netbuf) +inline uchar rdb_netbuf_to_byte(const uchar* const netbuf) { DBUG_ASSERT(netbuf != nullptr); @@ -167,7 +173,7 @@ inline uint64 rdb_netbuf_read_uint64(const uchar **netbuf_ptr) // Convert from network byte order (Big Endian) to host machine byte order // (usually Little Endian). - uint64 host_val= rdb_netbuf_to_uint64(*netbuf_ptr); + const uint64 host_val= rdb_netbuf_to_uint64(*netbuf_ptr); // Advance pointer. *netbuf_ptr += sizeof(host_val); @@ -181,7 +187,7 @@ inline uint32 rdb_netbuf_read_uint32(const uchar **netbuf_ptr) // Convert from network byte order (Big Endian) to host machine byte order // (usually Little Endian). - uint32 host_val= rdb_netbuf_to_uint32(*netbuf_ptr); + const uint32 host_val= rdb_netbuf_to_uint32(*netbuf_ptr); // Advance pointer. *netbuf_ptr += sizeof(host_val); @@ -195,7 +201,7 @@ inline uint16 rdb_netbuf_read_uint16(const uchar **netbuf_ptr) // Convert from network byte order (Big Endian) to host machine byte order // (usually Little Endian). - uint16 host_val= rdb_netbuf_to_uint16(*netbuf_ptr); + const uint16 host_val= rdb_netbuf_to_uint16(*netbuf_ptr); // Advance pointer. *netbuf_ptr += sizeof(host_val); @@ -204,7 +210,7 @@ inline uint16 rdb_netbuf_read_uint16(const uchar **netbuf_ptr) } inline void rdb_netbuf_read_gl_index(const uchar **netbuf_ptr, - GL_INDEX_ID *gl_index_id) + GL_INDEX_ID* const gl_index_id) { DBUG_ASSERT(gl_index_id != nullptr); DBUG_ASSERT(netbuf_ptr != nullptr); @@ -223,7 +229,20 @@ class Rdb_string_reader { const char* m_ptr; uint m_len; + private: + Rdb_string_reader& operator=(const Rdb_string_reader&) = default; public: + Rdb_string_reader(const Rdb_string_reader&) = default; + /* named constructor */ + static Rdb_string_reader read_or_empty(const rocksdb::Slice* const slice) + { + if (!slice) { + return Rdb_string_reader(""); + } else { + return Rdb_string_reader(slice); + } + } + explicit Rdb_string_reader(const std::string &str) { m_len= str.length(); @@ -243,7 +262,7 @@ class Rdb_string_reader } } - explicit Rdb_string_reader(const rocksdb::Slice *slice) + explicit Rdb_string_reader(const rocksdb::Slice* const slice) { m_ptr= slice->data(); m_len= slice->size(); @@ -253,7 +272,7 @@ class Rdb_string_reader Read the next @param size bytes. Returns pointer to the bytes read, or nullptr if the remaining string doesn't have that many bytes. */ - const char *read(uint size) + const char *read(const uint &size) { const char *res; if (m_len < size) @@ -269,7 +288,7 @@ class Rdb_string_reader return res; } - bool read_uint8(uint* res) + bool read_uint8(uint* const res) { const uchar *p; if (!(p= reinterpret_cast<const uchar*>(read(1)))) @@ -281,7 +300,7 @@ class Rdb_string_reader } } - bool read_uint16(uint* res) + bool read_uint16(uint* const res) { const uchar *p; if (!(p= reinterpret_cast<const uchar*>(read(2)))) @@ -323,42 +342,47 @@ class Rdb_string_writer { std::vector<uchar> m_data; public: + Rdb_string_writer(const Rdb_string_writer&) = delete; + Rdb_string_writer& operator=(const Rdb_string_writer&) = delete; + Rdb_string_writer() = default; + void clear() { m_data.clear(); } - void write_uint8(uint val) + void write_uint8(const uint &val) { m_data.push_back(static_cast<uchar>(val)); } - void write_uint16(uint val) + void write_uint16(const uint &val) { - auto size= m_data.size(); + const auto size= m_data.size(); m_data.resize(size + 2); rdb_netbuf_store_uint16(m_data.data() + size, val); } - void write_uint32(uint val) + void write_uint32(const uint &val) { - auto size= m_data.size(); + const auto size= m_data.size(); m_data.resize(size + 4); rdb_netbuf_store_uint32(m_data.data() + size, val); } - void write(uchar *new_data, size_t len) + void write(const uchar* const new_data, const size_t &len) { + DBUG_ASSERT(new_data != nullptr); m_data.insert(m_data.end(), new_data, new_data + len); } uchar* ptr() { return m_data.data(); } size_t get_current_pos() const { return m_data.size(); } - void write_uint8_at(size_t pos, uint new_val) + void write_uint8_at(const size_t &pos, const uint &new_val) { // This function will only overwrite what was written DBUG_ASSERT(pos < get_current_pos()); m_data.data()[pos]= new_val; } - void write_uint16_at(size_t pos, uint new_val) + void write_uint16_at(const size_t &pos, const uint &new_val) { // This function will only overwrite what was written DBUG_ASSERT(pos < get_current_pos() && (pos + 1) < get_current_pos()); @@ -378,13 +402,16 @@ class Rdb_bit_writer Rdb_string_writer *m_writer; uchar m_offset; public: + Rdb_bit_writer(const Rdb_bit_writer&) = delete; + Rdb_bit_writer& operator=(const Rdb_bit_writer&) = delete; + explicit Rdb_bit_writer(Rdb_string_writer* writer_arg) : m_writer(writer_arg), m_offset(0) { } - void write(uint size, uint value) + void write(uint size, const uint &value) { DBUG_ASSERT((value & ((1 << size) - 1)) == value); @@ -395,8 +422,8 @@ class Rdb_bit_writer m_writer->write_uint8(0); } // number of bits to put in this byte - uint bits = std::min(size, (uint)(8 - m_offset)); - uchar *last_byte= m_writer->ptr() + m_writer->get_current_pos() - 1; + const uint bits = std::min(size, (uint)(8 - m_offset)); + uchar* const last_byte= m_writer->ptr() + m_writer->get_current_pos() - 1; *last_byte |= (uchar) ((value >> (size - bits)) & ((1 << bits) - 1)) << m_offset; size -= bits; @@ -410,9 +437,12 @@ class Rdb_bit_reader const uchar *m_cur; uchar m_offset; uint m_ret; - Rdb_string_reader *m_reader; + Rdb_string_reader* const m_reader; public: - explicit Rdb_bit_reader(Rdb_string_reader *reader) + Rdb_bit_reader(const Rdb_bit_reader&) = delete; + Rdb_bit_reader& operator=(const Rdb_bit_reader&) = delete; + + explicit Rdb_bit_reader(Rdb_string_reader* const reader) : m_cur(nullptr), m_offset(0), m_reader(reader) @@ -438,7 +468,7 @@ class Rdb_bit_reader } } // how many bits from the current byte? - uint bits = std::min((uint)(8 - m_offset), size); + const uint bits = std::min((uint)(8 - m_offset), size); m_ret <<= bits; m_ret |= (*m_cur >> m_offset) & ((1 << bits) - 1); size -= bits; diff --git a/storage/rocksdb/rdb_cf_manager.cc b/storage/rocksdb/rdb_cf_manager.cc index 3ee28b49cb6..7ae7d362159 100644 --- a/storage/rocksdb/rdb_cf_manager.cc +++ b/storage/rocksdb/rdb_cf_manager.cc @@ -28,7 +28,7 @@ namespace myrocks { /* Check if ColumnFamily name says it's a reverse-ordered CF */ -bool Rdb_cf_manager::is_cf_name_reverse(const char *name) +bool Rdb_cf_manager::is_cf_name_reverse(const char* const name) { /* nullptr means the default CF is used.. (TODO: can the default CF be * reverse?) */ @@ -43,8 +43,8 @@ static PSI_mutex_key ex_key_cfm; #endif void Rdb_cf_manager::init( - Rdb_cf_options* cf_options, - std::vector<rocksdb::ColumnFamilyHandle*> *handles) + Rdb_cf_options* const cf_options, + std::vector<rocksdb::ColumnFamilyHandle*>* const handles) { mysql_mutex_init(ex_key_cfm, &m_mutex, MY_MUTEX_INIT_FAST); @@ -78,8 +78,8 @@ void Rdb_cf_manager::cleanup() */ void Rdb_cf_manager::get_per_index_cf_name(const std::string& db_table_name, - const char *index_name, - std::string *res) + const char* const index_name, + std::string* const res) { DBUG_ASSERT(index_name != nullptr); DBUG_ASSERT(res != nullptr); @@ -96,11 +96,11 @@ void Rdb_cf_manager::get_per_index_cf_name(const std::string& db_table_name, See Rdb_cf_manager::get_cf */ rocksdb::ColumnFamilyHandle* -Rdb_cf_manager::get_or_create_cf(rocksdb::DB *rdb, +Rdb_cf_manager::get_or_create_cf(rocksdb::DB* const rdb, const char *cf_name, const std::string& db_table_name, - const char *index_name, - bool *is_automatic) + const char* const index_name, + bool* const is_automatic) { DBUG_ASSERT(rdb != nullptr); DBUG_ASSERT(is_automatic != nullptr); @@ -120,13 +120,13 @@ Rdb_cf_manager::get_or_create_cf(rocksdb::DB *rdb, *is_automatic= true; } - auto it = m_cf_name_map.find(cf_name); + const auto it = m_cf_name_map.find(cf_name); if (it != m_cf_name_map.end()) cf_handle= it->second; else { /* Create a Column Family. */ - std::string cf_name_str(cf_name); + const std::string cf_name_str(cf_name); rocksdb::ColumnFamilyOptions opts; m_cf_options->get_cf_options(cf_name_str, &opts); @@ -135,7 +135,8 @@ Rdb_cf_manager::get_or_create_cf(rocksdb::DB *rdb, sql_print_information(" target_file_size_base=%" PRIu64, opts.target_file_size_base); - rocksdb::Status s= rdb->CreateColumnFamily(opts, cf_name_str, &cf_handle); + const rocksdb::Status s= + rdb->CreateColumnFamily(opts, cf_name_str, &cf_handle); if (s.ok()) { m_cf_name_map[cf_handle->GetName()] = cf_handle; m_cf_id_map[cf_handle->GetID()] = cf_handle; @@ -164,10 +165,9 @@ Rdb_cf_manager::get_or_create_cf(rocksdb::DB *rdb, rocksdb::ColumnFamilyHandle* Rdb_cf_manager::get_cf(const char *cf_name, const std::string& db_table_name, - const char *index_name, - bool *is_automatic) const + const char* const index_name, + bool* const is_automatic) const { - DBUG_ASSERT(cf_name != nullptr); DBUG_ASSERT(is_automatic != nullptr); rocksdb::ColumnFamilyHandle* cf_handle; @@ -185,7 +185,7 @@ Rdb_cf_manager::get_cf(const char *cf_name, *is_automatic= true; } - auto it = m_cf_name_map.find(cf_name); + const auto it = m_cf_name_map.find(cf_name); cf_handle = (it != m_cf_name_map.end()) ? it->second : nullptr; mysql_mutex_unlock(&m_mutex); @@ -193,12 +193,12 @@ Rdb_cf_manager::get_cf(const char *cf_name, return cf_handle; } -rocksdb::ColumnFamilyHandle* Rdb_cf_manager::get_cf(const uint32_t id) const +rocksdb::ColumnFamilyHandle* Rdb_cf_manager::get_cf(const uint32_t &id) const { rocksdb::ColumnFamilyHandle* cf_handle = nullptr; mysql_mutex_lock(&m_mutex); - auto it = m_cf_id_map.find(id); + const auto it = m_cf_id_map.find(id); if (it != m_cf_id_map.end()) cf_handle = it->second; mysql_mutex_unlock(&m_mutex); diff --git a/storage/rocksdb/rdb_cf_manager.h b/storage/rocksdb/rdb_cf_manager.h index 5a43b533c6d..4fb5f7437e8 100644 --- a/storage/rocksdb/rdb_cf_manager.h +++ b/storage/rocksdb/rdb_cf_manager.h @@ -55,19 +55,24 @@ class Rdb_cf_manager static void get_per_index_cf_name(const std::string& db_table_name, - const char *index_name, std::string *res); + const char* const index_name, + std::string* const res); Rdb_cf_options* m_cf_options= nullptr; public: - static bool is_cf_name_reverse(const char *name); + Rdb_cf_manager(const Rdb_cf_manager&) = delete; + Rdb_cf_manager& operator=(const Rdb_cf_manager&) = delete; + Rdb_cf_manager() = default; + + static bool is_cf_name_reverse(const char* const name); /* This is called right after the DB::Open() call. The parameters describe column families that are present in the database. The first CF is the default CF. */ void init(Rdb_cf_options* cf_options, - std::vector<rocksdb::ColumnFamilyHandle*> *handles); + std::vector<rocksdb::ColumnFamilyHandle*>* const handles); void cleanup(); /* @@ -76,17 +81,18 @@ public: - cf_name=_auto_ means use 'dbname.tablename.indexname' */ rocksdb::ColumnFamilyHandle* get_or_create_cf( - rocksdb::DB *rdb, const char *cf_name, const std::string& db_table_name, - const char *index_name, bool *is_automatic); + rocksdb::DB* const rdb, const char *cf_name, + const std::string& db_table_name, const char* const index_name, + bool* const is_automatic); /* Used by table open */ rocksdb::ColumnFamilyHandle* get_cf(const char *cf_name, const std::string& db_table_name, - const char *index_name, - bool *is_automatic) const; + const char* const index_name, + bool* const is_automatic) const; /* Look up cf by id; used by datadic */ - rocksdb::ColumnFamilyHandle* get_cf(const uint32_t id) const; + rocksdb::ColumnFamilyHandle* get_cf(const uint32_t &id) const; /* Used to iterate over column families for show status */ std::vector<std::string> get_cf_names(void) const; @@ -98,7 +104,7 @@ public: void get_cf_options( const std::string &cf_name, - rocksdb::ColumnFamilyOptions *opts) __attribute__((__nonnull__)) { + rocksdb::ColumnFamilyOptions* const opts) __attribute__((__nonnull__)) { m_cf_options->get_cf_options(cf_name, opts); } }; diff --git a/storage/rocksdb/rdb_cf_options.cc b/storage/rocksdb/rdb_cf_options.cc index ccdb46a654d..bd4d78d0796 100644 --- a/storage/rocksdb/rdb_cf_options.cc +++ b/storage/rocksdb/rdb_cf_options.cc @@ -41,16 +41,17 @@ Rdb_pk_comparator Rdb_cf_options::s_pk_comparator; Rdb_rev_comparator Rdb_cf_options::s_rev_pk_comparator; bool Rdb_cf_options::init( - size_t default_write_buffer_size, const rocksdb::BlockBasedTableOptions& table_options, std::shared_ptr<rocksdb::TablePropertiesCollectorFactory> prop_coll_factory, - const char * default_cf_options, - const char * override_cf_options) + const char* const default_cf_options, + const char* const override_cf_options) { + DBUG_ASSERT(default_cf_options != nullptr); + DBUG_ASSERT(override_cf_options != nullptr); + m_default_cf_opts.comparator = &s_pk_comparator; m_default_cf_opts.compaction_filter_factory.reset( new Rdb_compact_filter_factory); - m_default_cf_opts.write_buffer_size = default_write_buffer_size; m_default_cf_opts.table_factory.reset( rocksdb::NewBlockBasedTableFactory(table_options)); @@ -69,7 +70,7 @@ bool Rdb_cf_options::init( } void Rdb_cf_options::get(const std::string &cf_name, - rocksdb::ColumnFamilyOptions *opts) + rocksdb::ColumnFamilyOptions* const opts) { DBUG_ASSERT(opts != nullptr); @@ -106,7 +107,7 @@ bool Rdb_cf_options::set_default(const std::string &default_config) } // Skip over any spaces in the input string. -void Rdb_cf_options::skip_spaces(const std::string& input, size_t* pos) +void Rdb_cf_options::skip_spaces(const std::string& input, size_t* const pos) { DBUG_ASSERT(pos != nullptr); @@ -117,13 +118,14 @@ void Rdb_cf_options::skip_spaces(const std::string& input, size_t* pos) // Find a valid column family name. Note that all characters except a // semicolon are valid (should this change?) and all spaces are trimmed from // the beginning and end but are not removed between other characters. -bool Rdb_cf_options::find_column_family(const std::string& input, size_t* pos, - std::string* key) +bool Rdb_cf_options::find_column_family(const std::string& input, + size_t* const pos, + std::string* const key) { DBUG_ASSERT(pos != nullptr); DBUG_ASSERT(key != nullptr); - size_t beg_pos = *pos; + const size_t beg_pos = *pos; size_t end_pos = *pos - 1; // Loop through the characters in the string until we see a '='. @@ -148,8 +150,8 @@ bool Rdb_cf_options::find_column_family(const std::string& input, size_t* pos, // Find a valid options portion. Everything is deemed valid within the options // portion until we hit as many close curly braces as we have seen open curly // braces. -bool Rdb_cf_options::find_options(const std::string& input, size_t* pos, - std::string* options) +bool Rdb_cf_options::find_options(const std::string& input, size_t* const pos, + std::string* const options) { DBUG_ASSERT(pos != nullptr); DBUG_ASSERT(options != nullptr); @@ -169,7 +171,7 @@ bool Rdb_cf_options::find_options(const std::string& input, size_t* pos, // Set up our brace_count, the begin position and current end position. size_t brace_count = 1; - size_t beg_pos = *pos; + const size_t beg_pos = *pos; // Loop through the characters in the string until we find the appropriate // number of closing curly braces. @@ -211,9 +213,9 @@ bool Rdb_cf_options::find_options(const std::string& input, size_t* pos, } bool Rdb_cf_options::find_cf_options_pair(const std::string& input, - size_t* pos, - std::string* cf, - std::string* opt_str) + size_t* const pos, + std::string* const cf, + std::string* const opt_str) { DBUG_ASSERT(pos != nullptr); DBUG_ASSERT(cf != nullptr); @@ -326,7 +328,7 @@ const rocksdb::Comparator* Rdb_cf_options::get_cf_comparator( } void Rdb_cf_options::get_cf_options(const std::string &cf_name, - rocksdb::ColumnFamilyOptions *opts) + rocksdb::ColumnFamilyOptions* const opts) { DBUG_ASSERT(opts != nullptr); diff --git a/storage/rocksdb/rdb_cf_options.h b/storage/rocksdb/rdb_cf_options.h index e709e42e8b5..8151d907eb7 100644 --- a/storage/rocksdb/rdb_cf_options.h +++ b/storage/rocksdb/rdb_cf_options.h @@ -41,14 +41,17 @@ namespace myrocks { class Rdb_cf_options { public: - void get(const std::string &cf_name, rocksdb::ColumnFamilyOptions *opts); + Rdb_cf_options(const Rdb_cf_options&) = delete; + Rdb_cf_options& operator=(const Rdb_cf_options&) = delete; + Rdb_cf_options() = default; + + void get(const std::string &cf_name, rocksdb::ColumnFamilyOptions* const opts); bool init( - size_t default_write_buffer_size, const rocksdb::BlockBasedTableOptions& table_options, std::shared_ptr<rocksdb::TablePropertiesCollectorFactory> prop_coll_factory, - const char * default_cf_options, - const char * override_cf_options); + const char* const default_cf_options, + const char* const override_cf_options); const rocksdb::ColumnFamilyOptions& get_defaults() const { return m_default_cf_opts; @@ -59,20 +62,21 @@ class Rdb_cf_options void get_cf_options( const std::string &cf_name, - rocksdb::ColumnFamilyOptions *opts) __attribute__((__nonnull__)); + rocksdb::ColumnFamilyOptions* const opts) __attribute__((__nonnull__)); private: bool set_default(const std::string &default_config); bool set_override(const std::string &overide_config); /* Helper string manipulation functions */ - static void skip_spaces(const std::string& input, size_t* pos); - static bool find_column_family(const std::string& input, size_t* pos, - std::string* key); - static bool find_options(const std::string& input, size_t* pos, - std::string* options); - static bool find_cf_options_pair(const std::string& input, size_t* pos, - std::string* cf, std::string* opt_str); + static void skip_spaces(const std::string& input, size_t* const pos); + static bool find_column_family(const std::string& input, size_t* const pos, + std::string* const key); + static bool find_options(const std::string& input, size_t* const pos, + std::string* const options); + static bool find_cf_options_pair(const std::string& input, size_t* const pos, + std::string* const cf, + std::string* const opt_str); private: static Rdb_pk_comparator s_pk_comparator; diff --git a/storage/rocksdb/rdb_compact_filter.h b/storage/rocksdb/rdb_compact_filter.h index db2011721b7..ca634f74d43 100644 --- a/storage/rocksdb/rdb_compact_filter.h +++ b/storage/rocksdb/rdb_compact_filter.h @@ -35,6 +35,9 @@ namespace myrocks { class Rdb_compact_filter : public rocksdb::CompactionFilter { public: + Rdb_compact_filter(const Rdb_compact_filter&) = delete; + Rdb_compact_filter& operator=(const Rdb_compact_filter&) = delete; + explicit Rdb_compact_filter(uint32_t _cf_id) : m_cf_id(_cf_id) {} ~Rdb_compact_filter() {} @@ -95,6 +98,8 @@ class Rdb_compact_filter : public rocksdb::CompactionFilter class Rdb_compact_filter_factory : public rocksdb::CompactionFilterFactory { public: + Rdb_compact_filter_factory(const Rdb_compact_filter_factory&) = delete; + Rdb_compact_filter_factory& operator=(const Rdb_compact_filter_factory&) = delete; Rdb_compact_filter_factory() {} ~Rdb_compact_filter_factory() {} diff --git a/storage/rocksdb/rdb_comparator.h b/storage/rocksdb/rdb_comparator.h index 7e56c5ab00a..0e47556a778 100644 --- a/storage/rocksdb/rdb_comparator.h +++ b/storage/rocksdb/rdb_comparator.h @@ -35,11 +35,15 @@ namespace myrocks { class Rdb_pk_comparator : public rocksdb::Comparator { public: + Rdb_pk_comparator(const Rdb_pk_comparator&) = delete; + Rdb_pk_comparator& operator=(const Rdb_pk_comparator&) = delete; + Rdb_pk_comparator() = default; + static int bytewise_compare(const rocksdb::Slice& a, const rocksdb::Slice& b) { - size_t a_size= a.size(); - size_t b_size= b.size(); - size_t len= (a_size < b_size) ? a_size : b_size; + const size_t a_size= a.size(); + const size_t b_size= b.size(); + const size_t len= (a_size < b_size) ? a_size : b_size; int res; if ((res= memcmp(a.data(), b.data(), len))) @@ -75,6 +79,10 @@ class Rdb_pk_comparator : public rocksdb::Comparator class Rdb_rev_comparator : public rocksdb::Comparator { public: + Rdb_rev_comparator(const Rdb_rev_comparator&) = delete; + Rdb_rev_comparator& operator=(const Rdb_rev_comparator&) = delete; + Rdb_rev_comparator() = default; + static int bytewise_compare(const rocksdb::Slice& a, const rocksdb::Slice& b) { return -Rdb_pk_comparator::bytewise_compare(a, b); diff --git a/storage/rocksdb/rdb_datadic.cc b/storage/rocksdb/rdb_datadic.cc index d80c81a3d9a..3fa5f07844b 100644 --- a/storage/rocksdb/rdb_datadic.cc +++ b/storage/rocksdb/rdb_datadic.cc @@ -99,14 +99,14 @@ Rdb_key_def::Rdb_key_def(const Rdb_key_def& k) : rdb_netbuf_store_index(m_index_number_storage_form, m_index_number); if (k.m_pack_info) { - size_t size= sizeof(Rdb_field_packing) * k.m_key_parts; + const size_t size= sizeof(Rdb_field_packing) * k.m_key_parts; m_pack_info= reinterpret_cast<Rdb_field_packing*>(my_malloc(size, MYF(0))); memcpy(m_pack_info, k.m_pack_info, size); } if (k.m_pk_part_no) { - size_t size = sizeof(uint)*m_key_parts; + const size_t size = sizeof(uint)*m_key_parts; m_pk_part_no= reinterpret_cast<uint*>(my_malloc(size, MYF(0))); memcpy(m_pk_part_no, k.m_pk_part_no, size); } @@ -123,7 +123,8 @@ Rdb_key_def::~Rdb_key_def() m_pack_info = nullptr; } -void Rdb_key_def::setup(const TABLE *tbl, const Rdb_tbl_def *tbl_def) +void Rdb_key_def::setup(const TABLE* const tbl, + const Rdb_tbl_def* const tbl_def) { DBUG_ASSERT(tbl != nullptr); DBUG_ASSERT(tbl_def != nullptr); @@ -194,7 +195,7 @@ void Rdb_key_def::setup(const TABLE *tbl, const Rdb_tbl_def *tbl_def) else m_pk_part_no= nullptr; - size_t size= sizeof(Rdb_field_packing) * m_key_parts; + const size_t size= sizeof(Rdb_field_packing) * m_key_parts; m_pack_info= reinterpret_cast<Rdb_field_packing*>(my_malloc(size, MYF(0))); size_t max_len= INDEX_NUMBER_SIZE; @@ -222,7 +223,7 @@ void Rdb_key_def::setup(const TABLE *tbl, const Rdb_tbl_def *tbl_def) /* this loop also loops over the 'extended key' tail */ for (uint src_i= 0; src_i < m_key_parts; src_i++, keypart_to_set++) { - Field *field= key_part ? key_part->field : nullptr; + Field* const field= key_part ? key_part->field : nullptr; if (simulating_extkey && !hidden_pk_exists) { @@ -346,13 +347,12 @@ void Rdb_key_def::setup(const TABLE *tbl, const Rdb_tbl_def *tbl_def) set of queries for which we would check the checksum twice. */ -uint Rdb_key_def::get_primary_key_tuple(TABLE *table, - const std::shared_ptr<const Rdb_key_def>& pk_descr, - const rocksdb::Slice *key, - uchar *pk_buffer) const +uint Rdb_key_def::get_primary_key_tuple(const TABLE* const table, + const Rdb_key_def& pk_descr, + const rocksdb::Slice* const key, + uchar* const pk_buffer) const { DBUG_ASSERT(table != nullptr); - DBUG_ASSERT(pk_descr != nullptr); DBUG_ASSERT(key != nullptr); DBUG_ASSERT(pk_buffer); @@ -361,7 +361,7 @@ uint Rdb_key_def::get_primary_key_tuple(TABLE *table, DBUG_ASSERT(m_pk_key_parts); /* Put the PK number */ - rdb_netbuf_store_index(buf, pk_descr->m_index_number); + rdb_netbuf_store_index(buf, pk_descr.m_index_number); buf += INDEX_NUMBER_SIZE; size += INDEX_NUMBER_SIZE; @@ -404,10 +404,10 @@ uint Rdb_key_def::get_primary_key_tuple(TABLE *table, if (have_value) { - Rdb_field_packing *fpi= &m_pack_info[i]; + Rdb_field_packing* const fpi= &m_pack_info[i]; DBUG_ASSERT(table->s != nullptr); - bool is_hidden_pk_part= (i + 1 == m_key_parts) && + const bool is_hidden_pk_part= (i + 1 == m_key_parts) && (table->s->primary_key == MAX_INDEXES); Field *field= nullptr; if (!is_hidden_pk_part) @@ -424,7 +424,7 @@ uint Rdb_key_def::get_primary_key_tuple(TABLE *table, for (i= 0; i < m_pk_key_parts; i++) { - uint part_size= end_offs[i] - start_offs[i]; + const uint part_size= end_offs[i] - start_offs[i]; memcpy(buf, start_offs[i], end_offs[i] - start_offs[i]); buf += part_size; size += part_size; @@ -445,9 +445,10 @@ uint Rdb_key_def::get_primary_key_tuple(TABLE *table, size is at least max_storage_fmt_length() bytes. */ -uint Rdb_key_def::pack_index_tuple(TABLE *tbl, uchar *pack_buffer, - uchar *packed_tuple, const uchar *key_tuple, - key_part_map keypart_map) const +uint Rdb_key_def::pack_index_tuple(TABLE* const tbl, uchar* const pack_buffer, + uchar* const packed_tuple, + const uchar* const key_tuple, + const key_part_map &keypart_map) const { DBUG_ASSERT(tbl != nullptr); DBUG_ASSERT(pack_buffer != nullptr); @@ -455,7 +456,7 @@ uint Rdb_key_def::pack_index_tuple(TABLE *tbl, uchar *pack_buffer, DBUG_ASSERT(key_tuple != nullptr); /* We were given a record in KeyTupleFormat. First, save it to record */ - uint key_len= calculate_key_len(tbl, m_keyno, key_tuple, keypart_map); + const uint key_len= calculate_key_len(tbl, m_keyno, key_tuple, keypart_map); key_restore(tbl->record[0], key_tuple, &tbl->key_info[m_keyno], key_len); uint n_used_parts= my_count_bits(keypart_map); @@ -486,7 +487,7 @@ bool Rdb_key_def::unpack_info_has_checksum(const rocksdb::Slice &unpack_info) if (size >= RDB_UNPACK_HEADER_SIZE && ptr[0] == RDB_UNPACK_DATA_TAG) { - uint16 skip_len= rdb_netbuf_to_uint16(ptr + 1); + const uint16 skip_len= rdb_netbuf_to_uint16(ptr + 1); SHIP_ASSERT(size >= skip_len); size -= skip_len; @@ -499,7 +500,7 @@ bool Rdb_key_def::unpack_info_has_checksum(const rocksdb::Slice &unpack_info) /* @return Number of bytes that were changed */ -int Rdb_key_def::successor(uchar *packed_tuple, uint len) +int Rdb_key_def::successor(uchar* const packed_tuple, const uint &len) { DBUG_ASSERT(packed_tuple != nullptr); @@ -541,12 +542,14 @@ int Rdb_key_def::successor(uchar *packed_tuple, uint len) Length of the packed tuple */ -uint Rdb_key_def::pack_record(const TABLE *tbl, uchar *pack_buffer, - const uchar *record, uchar *packed_tuple, - Rdb_string_writer *unpack_info, - bool should_store_checksums, - longlong hidden_pk_id, uint n_key_parts, - uint *n_null_fields) const +uint Rdb_key_def::pack_record(const TABLE* const tbl, uchar* const pack_buffer, + const uchar* const record, + uchar* const packed_tuple, + Rdb_string_writer* const unpack_info, + const bool &should_store_row_debug_checksums, + const longlong &hidden_pk_id, + uint n_key_parts, + uint* const n_null_fields) const { DBUG_ASSERT(tbl != nullptr); DBUG_ASSERT(pack_buffer != nullptr); @@ -554,7 +557,7 @@ uint Rdb_key_def::pack_record(const TABLE *tbl, uchar *pack_buffer, DBUG_ASSERT(packed_tuple != nullptr); // Checksums for PKs are made when record is packed. // We should never attempt to make checksum just from PK values - DBUG_ASSERT_IMP(should_store_checksums, + DBUG_ASSERT_IMP(should_store_row_debug_checksums, (m_index_type == INDEX_TYPE_SECONDARY)); uchar *tuple= packed_tuple; @@ -568,7 +571,7 @@ uint Rdb_key_def::pack_record(const TABLE *tbl, uchar *pack_buffer, // The following includes the 'extended key' tail. // The 'extended key' includes primary key. This is done to 'uniqify' // non-unique indexes - bool use_all_columns = n_key_parts == 0 || n_key_parts == MAX_REF_PARTS; + const bool use_all_columns = n_key_parts == 0 || n_key_parts == MAX_REF_PARTS; // If hidden pk exists, but hidden pk wasnt passed in, we can't pack the // hidden key part. So we skip it (its always 1 part). @@ -599,13 +602,13 @@ uint Rdb_key_def::pack_record(const TABLE *tbl, uchar *pack_buffer, break; } - Field *field= m_pack_info[i].get_field_in_table(tbl); + Field* const field= m_pack_info[i].get_field_in_table(tbl); DBUG_ASSERT(field != nullptr); // Old Field methods expected the record pointer to be at tbl->record[0]. // The quick and easy way to fix this was to pass along the offset // for the pointer. - my_ptrdiff_t ptr_diff= record - tbl->record[0]; + const my_ptrdiff_t ptr_diff= record - tbl->record[0]; if (field->real_maybe_null()) { @@ -626,11 +629,9 @@ uint Rdb_key_def::pack_record(const TABLE *tbl, uchar *pack_buffer, } } - bool create_unpack_info= + const bool create_unpack_info= (unpack_info && // we were requested to generate unpack_info - m_pack_info[i].uses_unpack_info() && // and this keypart uses it - index_format_min_check(PRIMARY_FORMAT_VERSION_UPDATE1, - SECONDARY_FORMAT_VERSION_UPDATE1)); + m_pack_info[i].uses_unpack_info()); // and this keypart uses it Rdb_pack_field_context pack_ctx(unpack_info); // Set the offset for methods which do not take an offset as an argument @@ -652,7 +653,7 @@ uint Rdb_key_def::pack_record(const TABLE *tbl, uchar *pack_buffer, if (unpack_info) { - size_t len= unpack_info->get_current_pos(); + const size_t len= unpack_info->get_current_pos(); DBUG_ASSERT(len <= std::numeric_limits<uint16_t>::max()); // Don't store the unpack_info if it has only the header (that is, there's @@ -676,10 +677,10 @@ uint Rdb_key_def::pack_record(const TABLE *tbl, uchar *pack_buffer, // so the checksums are computed and stored by // ha_rocksdb::convert_record_to_storage_format // - if (should_store_checksums) + if (should_store_row_debug_checksums) { - uint32_t key_crc32= crc32(0, packed_tuple, tuple - packed_tuple); - uint32_t val_crc32= crc32(0, unpack_info->ptr(), + const uint32_t key_crc32= crc32(0, packed_tuple, tuple - packed_tuple); + const uint32_t val_crc32= crc32(0, unpack_info->ptr(), unpack_info->get_current_pos()); unpack_info->write_uint8(RDB_CHECKSUM_DATA_TAG); @@ -705,8 +706,8 @@ uint Rdb_key_def::pack_record(const TABLE *tbl, uchar *pack_buffer, Length of the packed tuple */ -uint Rdb_key_def::pack_hidden_pk(longlong hidden_pk_id, - uchar *packed_tuple) const +uint Rdb_key_def::pack_hidden_pk(const longlong &hidden_pk_id, + uchar* const packed_tuple) const { DBUG_ASSERT(packed_tuple != nullptr); @@ -728,10 +729,11 @@ uint Rdb_key_def::pack_hidden_pk(longlong hidden_pk_id, Function of type rdb_index_field_pack_t */ -void rdb_pack_with_make_sort_key(Rdb_field_packing *fpi, Field *field, - uchar *buf __attribute__((__unused__)), +void rdb_pack_with_make_sort_key(Rdb_field_packing* const fpi, + Field* const field, + uchar* const buf __attribute__((__unused__)), uchar **dst, - Rdb_pack_field_context *pack_ctx + Rdb_pack_field_context* const pack_ctx __attribute__((__unused__))) { DBUG_ASSERT(fpi != nullptr); @@ -756,7 +758,7 @@ void rdb_pack_with_make_sort_key(Rdb_field_packing *fpi, Field *field, int Rdb_key_def::compare_keys( const rocksdb::Slice *key1, const rocksdb::Slice *key2, - std::size_t* column_index + std::size_t* const column_index ) const { DBUG_ASSERT(key1 != nullptr); @@ -779,11 +781,11 @@ int Rdb_key_def::compare_keys( for (uint i= 0; i < m_key_parts ; i++) { - Rdb_field_packing *fpi= &m_pack_info[i]; + const Rdb_field_packing* const fpi= &m_pack_info[i]; if (fpi->m_maybe_null) { - auto nullp1= reader1.read(1); - auto nullp2= reader2.read(1); + const auto nullp1= reader1.read(1); + const auto nullp2= reader2.read(1); if (nullp1 == nullptr || nullp2 == nullptr) return 1; //error @@ -800,15 +802,15 @@ int Rdb_key_def::compare_keys( } } - auto before_skip1 = reader1.get_current_ptr(); - auto before_skip2 = reader2.get_current_ptr(); + const auto before_skip1 = reader1.get_current_ptr(); + const auto before_skip2 = reader2.get_current_ptr(); DBUG_ASSERT(fpi->m_skip_func); if (fpi->m_skip_func(fpi, nullptr, &reader1)) return 1; if (fpi->m_skip_func(fpi, nullptr, &reader2)) return 1; - auto size1 = reader1.get_current_ptr() - before_skip1; - auto size2 = reader2.get_current_ptr() - before_skip2; + const auto size1 = reader1.get_current_ptr() - before_skip1; + const auto size2 = reader2.get_current_ptr() - before_skip2; if (size1 != size2) { *column_index = i; @@ -835,7 +837,8 @@ int Rdb_key_def::compare_keys( Fixed-size skip functions just read. */ -size_t Rdb_key_def::key_length(TABLE *table, const rocksdb::Slice &key) const +size_t Rdb_key_def::key_length(const TABLE* const table, + const rocksdb::Slice &key) const { DBUG_ASSERT(table != nullptr); @@ -846,8 +849,8 @@ size_t Rdb_key_def::key_length(TABLE *table, const rocksdb::Slice &key) const for (uint i= 0; i < m_key_parts ; i++) { - Rdb_field_packing *fpi= &m_pack_info[i]; - Field *field= nullptr; + const Rdb_field_packing *fpi= &m_pack_info[i]; + const Field *field= nullptr; if (m_index_type != INDEX_TYPE_HIDDEN_PRIMARY) field= fpi->get_field_in_table(table); if (fpi->m_skip_func(fpi, field, &reader)) @@ -870,30 +873,26 @@ size_t Rdb_key_def::key_length(TABLE *table, const rocksdb::Slice &key) const unpacking. */ -int Rdb_key_def::unpack_record(TABLE *table, uchar *buf, - const rocksdb::Slice *packed_key, - const rocksdb::Slice *unpack_info, - bool verify_checksums) const +int Rdb_key_def::unpack_record(TABLE* const table, uchar* const buf, + const rocksdb::Slice* const packed_key, + const rocksdb::Slice* const unpack_info, + const bool &verify_row_debug_checksums) const { Rdb_string_reader reader(packed_key); - Rdb_string_reader unp_reader(""); + Rdb_string_reader unp_reader= Rdb_string_reader::read_or_empty(unpack_info); + const bool is_hidden_pk= (m_index_type == INDEX_TYPE_HIDDEN_PRIMARY); const bool hidden_pk_exists= table_has_hidden_pk(table); const bool secondary_key= (m_index_type == INDEX_TYPE_SECONDARY); // There is no checksuming data after unpack_info for primary keys, because // the layout there is different. The checksum is verified in // ha_rocksdb::convert_record_from_storage_format instead. - DBUG_ASSERT_IMP(!secondary_key, !verify_checksums); - - if (unpack_info) - { - unp_reader= Rdb_string_reader(unpack_info); - } + DBUG_ASSERT_IMP(!secondary_key, !verify_row_debug_checksums); // Old Field methods expected the record pointer to be at tbl->record[0]. // The quick and easy way to fix this was to pass along the offset // for the pointer. - my_ptrdiff_t ptr_diff= buf - table->record[0]; + const my_ptrdiff_t ptr_diff= buf - table->record[0]; // Skip the index number if ((!reader.read(INDEX_NUMBER_SIZE))) @@ -904,7 +903,7 @@ int Rdb_key_def::unpack_record(TABLE *table, uchar *buf, // For secondary keys, we expect the value field to contain unpack data and // checksum data in that order. One or both can be missing, but they cannot // be reordered. - bool has_unpack_info= unp_reader.remaining_bytes() && + const bool has_unpack_info= unp_reader.remaining_bytes() && *unp_reader.get_current_ptr() == RDB_UNPACK_DATA_TAG; if (has_unpack_info && !unp_reader.read(RDB_UNPACK_HEADER_SIZE)) { @@ -913,7 +912,7 @@ int Rdb_key_def::unpack_record(TABLE *table, uchar *buf, for (uint i= 0; i < m_key_parts ; i++) { - Rdb_field_packing *fpi= &m_pack_info[i]; + Rdb_field_packing* const fpi= &m_pack_info[i]; /* Hidden pk field is packed at the end of the secondary keys, but the SQL @@ -930,12 +929,9 @@ int Rdb_key_def::unpack_record(TABLE *table, uchar *buf, continue; } - Field *field= fpi->get_field_in_table(table); + Field* const field= fpi->get_field_in_table(table); - bool do_unpack= secondary_key || - !fpi->uses_unpack_info() || - (m_kv_format_version >= Rdb_key_def::PRIMARY_FORMAT_VERSION_UPDATE1); - if (fpi->m_unpack_func && do_unpack) + if (fpi->m_unpack_func) { /* It is possible to unpack this column. Do it. */ @@ -964,9 +960,9 @@ int Rdb_key_def::unpack_record(TABLE *table, uchar *buf, // If we need unpack info, but there is none, tell the unpack function // this by passing unp_reader as nullptr. If we never read unpack_info // during unpacking anyway, then there won't an error. - int res; - bool maybe_missing_unpack= !has_unpack_info && fpi->uses_unpack_info(); - res= fpi->m_unpack_func(fpi, field, field->ptr + ptr_diff, + const bool maybe_missing_unpack= + !has_unpack_info && fpi->uses_unpack_info(); + const int res= fpi->m_unpack_func(fpi, field, field->ptr + ptr_diff, &reader, maybe_missing_unpack ? nullptr : &unp_reader); @@ -1001,16 +997,16 @@ int Rdb_key_def::unpack_record(TABLE *table, uchar *buf, const char* ptr; if ((ptr= unp_reader.read(1)) && *ptr == RDB_CHECKSUM_DATA_TAG) { - if (verify_checksums) + if (verify_row_debug_checksums) { uint32_t stored_key_chksum= rdb_netbuf_to_uint32( (const uchar*)unp_reader.read(RDB_CHECKSUM_SIZE)); - uint32_t stored_val_chksum= rdb_netbuf_to_uint32( + const uint32_t stored_val_chksum= rdb_netbuf_to_uint32( (const uchar*)unp_reader.read(RDB_CHECKSUM_SIZE)); - uint32_t computed_key_chksum= + const uint32_t computed_key_chksum= crc32(0, (const uchar*)packed_key->data(), packed_key->size()); - uint32_t computed_val_chksum= + const uint32_t computed_val_chksum= crc32(0, (const uchar*) unpack_info->data(), unpack_info->size() - RDB_CHECKSUM_CHUNK_SIZE); @@ -1044,20 +1040,20 @@ int Rdb_key_def::unpack_record(TABLE *table, uchar *buf, return 0; } -bool Rdb_key_def::table_has_hidden_pk(const TABLE* table) +bool Rdb_key_def::table_has_hidden_pk(const TABLE* const table) { return table->s->primary_key == MAX_INDEXES; } -void Rdb_key_def::report_checksum_mismatch(bool is_key, const char *data, - size_t data_size) const +void Rdb_key_def::report_checksum_mismatch(const bool &is_key, + const char* const data, + const size_t data_size) const { - std::string buf; // NO_LINT_DEBUG sql_print_error("Checksum mismatch in %s of key-value pair for index 0x%x", is_key? "key" : "value", get_index_number()); - buf = rdb_hexdump(data, data_size, 1000); + const std::string buf = rdb_hexdump(data, data_size, RDB_MAX_HEXDUMP_LEN); // NO_LINT_DEBUG sql_print_error("Data with incorrect checksum (%" PRIu64 " bytes): %s", (uint64_t)data_size, buf.c_str()); @@ -1065,7 +1061,8 @@ void Rdb_key_def::report_checksum_mismatch(bool is_key, const char *data, my_error(ER_INTERNAL_ERROR, MYF(0), "Record checksum mismatch"); } -bool Rdb_key_def::index_format_min_check(int pk_min, int sk_min) const +bool Rdb_key_def::index_format_min_check(const int &pk_min, + const int &sk_min) const { switch (m_index_type) { @@ -1088,9 +1085,9 @@ bool Rdb_key_def::index_format_min_check(int pk_min, int sk_min) const Function of type rdb_index_field_skip_t */ -int rdb_skip_max_length(const Rdb_field_packing *fpi, - const Field *field __attribute__((__unused__)), - Rdb_string_reader *reader) +int rdb_skip_max_length(const Rdb_field_packing* const fpi, + const Field* const field __attribute__((__unused__)), + Rdb_string_reader* const reader) { if (!reader->read(fpi->m_max_image_len)) return 1; @@ -1112,8 +1109,8 @@ static_assert((RDB_ESCAPE_LENGTH - 1) % 2 == 0, */ static int rdb_skip_variable_length( - const Rdb_field_packing *fpi __attribute__((__unused__)), - const Field *field, Rdb_string_reader *reader) + const Rdb_field_packing* const fpi __attribute__((__unused__)), + const Field* const field, Rdb_string_reader* const reader) { const uchar *ptr; bool finished= false; @@ -1121,7 +1118,7 @@ static int rdb_skip_variable_length( size_t dst_len; /* How much data can be there */ if (field) { - const Field_varstring* field_var= + const Field_varstring* const field_var= static_cast<const Field_varstring*>(field); dst_len= field_var->pack_length() - field_var->length_bytes; } @@ -1134,8 +1131,8 @@ static int rdb_skip_variable_length( while ((ptr= (const uchar*)reader->read(RDB_ESCAPE_LENGTH))) { /* See rdb_pack_with_varchar_encoding. */ - uchar pad= 255 - ptr[RDB_ESCAPE_LENGTH - 1]; // number of padding bytes - uchar used_bytes= RDB_ESCAPE_LENGTH - 1 - pad; + const uchar pad= 255 - ptr[RDB_ESCAPE_LENGTH - 1]; // number of padding bytes + const uchar used_bytes= RDB_ESCAPE_LENGTH - 1 - pad; if (used_bytes > RDB_ESCAPE_LENGTH - 1 || used_bytes > dst_len) { @@ -1167,8 +1164,8 @@ const int VARCHAR_CMP_GREATER_THAN_SPACES = 3; */ static int rdb_skip_variable_space_pad( - const Rdb_field_packing *fpi, - const Field *field, Rdb_string_reader *reader) + const Rdb_field_packing* const fpi, + const Field* const field, Rdb_string_reader* const reader) { const uchar *ptr; bool finished= false; @@ -1177,7 +1174,7 @@ static int rdb_skip_variable_space_pad( if (field) { - const Field_varstring* field_var= + const Field_varstring* const field_var= static_cast<const Field_varstring*>(field); dst_len= field_var->pack_length() - field_var->length_bytes; } @@ -1186,7 +1183,7 @@ static int rdb_skip_variable_space_pad( while ((ptr= (const uchar*)reader->read(fpi->m_segment_size))) { // See rdb_pack_with_varchar_space_pad - uchar c= ptr[fpi->m_segment_size-1]; + const uchar c= ptr[fpi->m_segment_size-1]; if (c == VARCHAR_CMP_EQUAL_TO_SPACES) { // This is the last segment @@ -1221,9 +1218,9 @@ static int rdb_skip_variable_space_pad( */ int rdb_unpack_integer( - Rdb_field_packing *fpi, Field *field, uchar *to, - Rdb_string_reader *reader, - Rdb_string_reader *unp_reader __attribute__((__unused__))) + Rdb_field_packing* const fpi, Field* const field, uchar* const to, + Rdb_string_reader* const reader, + Rdb_string_reader* const unp_reader __attribute__((__unused__))) { const int length= fpi->m_max_image_len; @@ -1254,7 +1251,7 @@ int rdb_unpack_integer( } #if !defined(WORDS_BIGENDIAN) -static void rdb_swap_double_bytes(uchar *dst, const uchar *src) +static void rdb_swap_double_bytes(uchar* const dst, const uchar* const src) { #if defined(__FLOAT_WORD_ORDER) && (__FLOAT_WORD_ORDER == __BIG_ENDIAN) // A few systems store the most-significant _word_ first on little-endian @@ -1266,7 +1263,7 @@ static void rdb_swap_double_bytes(uchar *dst, const uchar *src) #endif } -static void rdb_swap_float_bytes(uchar *dst, const uchar *src) +static void rdb_swap_float_bytes(uchar* const dst, const uchar* const src) { dst[0] = src[3]; dst[1] = src[2]; dst[2] = src[1]; dst[3] = src[0]; } @@ -1276,15 +1273,13 @@ static void rdb_swap_float_bytes(uchar *dst, const uchar *src) #endif static int rdb_unpack_floating_point( - uchar *dst, Rdb_string_reader *reader, - size_t size, int exp_digit, - const uchar *zero_pattern, - const uchar *zero_val, + uchar* const dst, Rdb_string_reader* const reader, const size_t &size, + const int &exp_digit, + const uchar* const zero_pattern, + const uchar* const zero_val, void (*swap_func)(uchar *, const uchar *)) { - const uchar* from; - - from= (const uchar*) reader->read(size); + const uchar* const from = (const uchar*) reader->read(size); if (from == nullptr) return UNPACK_FAILURE; /* Mem-comparable image doesn't have enough bytes */ @@ -1297,7 +1292,7 @@ static int rdb_unpack_floating_point( #if defined(WORDS_BIGENDIAN) // On big-endian, output can go directly into result - uchar *tmp = dst; + uchar* const tmp = dst; #else // Otherwise use a temporary buffer to make byte-swapping easier later uchar tmp[8]; @@ -1347,11 +1342,11 @@ static int rdb_unpack_floating_point( allowed in the database. */ static int rdb_unpack_double( - Rdb_field_packing *fpi __attribute__((__unused__)), - Field *field __attribute__((__unused__)), - uchar *field_ptr, - Rdb_string_reader *reader, - Rdb_string_reader *unp_reader __attribute__((__unused__))) + Rdb_field_packing* const fpi __attribute__((__unused__)), + Field* const field __attribute__((__unused__)), + uchar* const field_ptr, + Rdb_string_reader* const reader, + Rdb_string_reader* const unp_reader __attribute__((__unused__))) { static double zero_val = 0.0; static const uchar zero_pattern[8] = { 128, 0, 0, 0, 0, 0, 0, 0 }; @@ -1375,10 +1370,10 @@ static int rdb_unpack_double( allowed in the database. */ static int rdb_unpack_float( - Rdb_field_packing *, Field *field __attribute__((__unused__)), - uchar *field_ptr, - Rdb_string_reader *reader, - Rdb_string_reader *unp_reader __attribute__((__unused__))) + Rdb_field_packing* const, Field* const field __attribute__((__unused__)), + uchar* const field_ptr, + Rdb_string_reader* const reader, + Rdb_string_reader* const unp_reader __attribute__((__unused__))) { static float zero_val = 0.0; static const uchar zero_pattern[4] = { 128, 0, 0, 0 }; @@ -1395,10 +1390,10 @@ static int rdb_unpack_float( */ int rdb_unpack_newdate( - Rdb_field_packing *fpi, Field *field, - uchar *field_ptr, - Rdb_string_reader *reader, - Rdb_string_reader *unp_reader __attribute__((__unused__))) + Rdb_field_packing* const fpi, Field* constfield, + uchar* const field_ptr, + Rdb_string_reader* const reader, + Rdb_string_reader* const unp_reader __attribute__((__unused__))) { const char* from; DBUG_ASSERT(fpi->m_max_image_len == 3); @@ -1420,10 +1415,9 @@ int rdb_unpack_newdate( */ static int rdb_unpack_binary_str( - Rdb_field_packing *fpi, Field *field, - uchar *to, - Rdb_string_reader *reader, - Rdb_string_reader *unp_reader __attribute__((__unused__))) + Rdb_field_packing* const fpi, Field* const field, uchar* const to, + Rdb_string_reader* const reader, + Rdb_string_reader* const unp_reader __attribute__((__unused__))) { const char* from; if (!(from= reader->read(fpi->m_max_image_len))) @@ -1441,18 +1435,18 @@ static int rdb_unpack_binary_str( */ static int rdb_unpack_utf8_str( - Rdb_field_packing *fpi, Field *field, + Rdb_field_packing* const fpi, Field* const field, uchar *dst, - Rdb_string_reader *reader, - Rdb_string_reader *unp_reader __attribute__((__unused__))) + Rdb_string_reader* const reader, + Rdb_string_reader* const unp_reader __attribute__((__unused__))) { - my_core::CHARSET_INFO *cset= (my_core::CHARSET_INFO*)field->charset(); + my_core::CHARSET_INFO* const cset= (my_core::CHARSET_INFO*)field->charset(); const uchar *src; if (!(src= (const uchar*)reader->read(fpi->m_max_image_len))) return UNPACK_FAILURE; /* Mem-comparable image doesn't have enough bytes */ - const uchar *src_end= src + fpi->m_max_image_len; - uchar *dst_end= dst + field->pack_length(); + const uchar* const src_end= src + fpi->m_max_image_len; + uchar* const dst_end= dst + field->pack_length(); while (src < src_end) { @@ -1476,8 +1470,8 @@ static int rdb_unpack_utf8_str( */ static void rdb_pack_with_varchar_encoding( - Rdb_field_packing *fpi, Field *field, uchar *buf, uchar **dst, - Rdb_pack_field_context *pack_ctx __attribute__((__unused__))) + Rdb_field_packing* const fpi, Field* const field, uchar *buf, uchar **dst, + Rdb_pack_field_context* const pack_ctx __attribute__((__unused__))) { /* Use a flag byte every Nth byte. Set it to (255 - #pad) where #pad is 0 @@ -1489,14 +1483,14 @@ static void rdb_pack_with_varchar_encoding( * 4 bytes (1, 2, 3, 0) this is encoded as: 1, 2, 3, 0, 0, 0, 0, 252 And the 4 byte string compares as greater than the 3 byte string */ - const CHARSET_INFO *charset= field->charset(); - Field_varstring *field_var= (Field_varstring*)field; - - size_t value_length= (field_var->length_bytes == 1) ? - (uint) *field->ptr : - uint2korr(field->ptr); - size_t xfrm_len; - xfrm_len= charset->coll->strnxfrm(charset, + const CHARSET_INFO* const charset= field->charset(); + Field_varstring* const field_var= (Field_varstring*)field; + + const size_t value_length= (field_var->length_bytes == 1) ? + (uint) *field->ptr : + uint2korr(field->ptr); + size_t xfrm_len= charset->coll->strnxfrm( + charset, buf, fpi->m_max_image_len, field_var->char_length(), field_var->ptr + field_var->length_bytes, @@ -1506,11 +1500,11 @@ static void rdb_pack_with_varchar_encoding( /* Got a mem-comparable image in 'buf'. Now, produce varlength encoding */ size_t encoded_size= 0; - uchar *ptr= *dst; + uchar* ptr= *dst; while (1) { - size_t copy_len= std::min((size_t)RDB_ESCAPE_LENGTH-1, xfrm_len); - size_t padding_bytes= RDB_ESCAPE_LENGTH - 1 - copy_len; + const size_t copy_len= std::min((size_t)RDB_ESCAPE_LENGTH-1, xfrm_len); + const size_t padding_bytes= RDB_ESCAPE_LENGTH - 1 - copy_len; memcpy(ptr, buf, copy_len); ptr += copy_len; buf += copy_len; @@ -1534,8 +1528,8 @@ static void rdb_pack_with_varchar_encoding( */ static -int rdb_compare_string_with_spaces(const uchar *buf, const uchar *buf_end, - const std::vector<uchar> *space_xfrm) +int rdb_compare_string_with_spaces(const uchar *buf, const uchar* const buf_end, + const std::vector<uchar>* const space_xfrm) { int cmp= 0; while (buf < buf_end) @@ -1621,24 +1615,24 @@ static const int RDB_TRIMMED_CHARS_OFFSET= 8; */ static void rdb_pack_with_varchar_space_pad( - Rdb_field_packing *fpi, Field *field, uchar *buf, uchar **dst, - Rdb_pack_field_context *pack_ctx) + Rdb_field_packing* const fpi, Field* const field, uchar* buf, + uchar **dst, Rdb_pack_field_context* const pack_ctx) { - Rdb_string_writer *unpack_info= pack_ctx->writer; - const CHARSET_INFO *charset= field->charset(); - auto field_var= static_cast<Field_varstring *>(field); + Rdb_string_writer* const unpack_info= pack_ctx->writer; + const CHARSET_INFO* const charset= field->charset(); + const auto field_var= static_cast<Field_varstring *>(field); - size_t value_length= (field_var->length_bytes == 1) ? - (uint) *field->ptr : - uint2korr(field->ptr); + const size_t value_length= (field_var->length_bytes == 1) ? + (uint) *field->ptr : + uint2korr(field->ptr); - size_t trimmed_len= + const size_t trimmed_len= charset->cset->lengthsp(charset, (const char*)field_var->ptr + field_var->length_bytes, value_length); - size_t xfrm_len; - xfrm_len= charset->coll->strnxfrm(charset, + const size_t xfrm_len = charset->coll->strnxfrm( + charset, buf, fpi->m_max_image_len, field_var->char_length(), field_var->ptr + field_var->length_bytes, @@ -1646,14 +1640,15 @@ static void rdb_pack_with_varchar_space_pad( 0); /* Got a mem-comparable image in 'buf'. Now, produce varlength encoding */ - uchar *buf_end= buf + xfrm_len; + uchar* const buf_end= buf + xfrm_len; size_t encoded_size= 0; uchar *ptr= *dst; size_t padding_bytes; while (true) { - size_t copy_len= std::min<size_t>(fpi->m_segment_size-1, buf_end - buf); + const size_t copy_len= + std::min<size_t>(fpi->m_segment_size-1, buf_end - buf); padding_bytes= fpi->m_segment_size - 1 - copy_len; memcpy(ptr, buf, copy_len); ptr += copy_len; @@ -1670,7 +1665,8 @@ static void rdb_pack_with_varchar_space_pad( // Compare the string suffix with a hypothetical infinite string of // spaces. It could be that the first difference is beyond the end of // current chunk. - int cmp= rdb_compare_string_with_spaces(buf, buf_end, fpi->space_xfrm); + const int cmp= + rdb_compare_string_with_spaces(buf, buf_end, fpi->space_xfrm); if (cmp < 0) *ptr= VARCHAR_CMP_LESS_THAN_SPACES; @@ -1698,7 +1694,8 @@ static void rdb_pack_with_varchar_space_pad( // then, we add 8, because we don't store negative values. DBUG_ASSERT(padding_bytes % fpi->space_xfrm_len == 0); DBUG_ASSERT((value_length - trimmed_len)% fpi->space_mb_len == 0); - size_t removed_chars= RDB_TRIMMED_CHARS_OFFSET + + const size_t removed_chars= + RDB_TRIMMED_CHARS_OFFSET + (value_length - trimmed_len) / fpi->space_mb_len - padding_bytes/fpi->space_xfrm_len; @@ -1721,20 +1718,20 @@ static void rdb_pack_with_varchar_space_pad( */ static int rdb_unpack_binary_or_utf8_varchar( - Rdb_field_packing *fpi, Field *field, - uchar *dst, - Rdb_string_reader *reader, - Rdb_string_reader *unp_reader __attribute__((__unused__))) + Rdb_field_packing* const fpi, Field* const field, + uchar* dst, + Rdb_string_reader* const reader, + Rdb_string_reader* const unp_reader __attribute__((__unused__))) { const uchar *ptr; size_t len= 0; bool finished= false; uchar *d0= dst; - Field_varstring* field_var= (Field_varstring*)field; + Field_varstring* const field_var= (Field_varstring*)field; dst += field_var->length_bytes; // How much we can unpack size_t dst_len= field_var->pack_length() - field_var->length_bytes; - uchar *dst_end= dst + dst_len; + uchar* const dst_end= dst + dst_len; /* Decode the length-emitted encoding here */ while ((ptr= (const uchar*)reader->read(RDB_ESCAPE_LENGTH))) @@ -1823,15 +1820,15 @@ static int rdb_unpack_binary_or_utf8_varchar( rdb_skip_variable_space_pad - skip function */ static int rdb_unpack_binary_or_utf8_varchar_space_pad( - Rdb_field_packing *fpi, Field *field, - uchar *dst, - Rdb_string_reader *reader, - Rdb_string_reader *unp_reader) + Rdb_field_packing* const fpi, Field* const field, + uchar* dst, + Rdb_string_reader* const reader, + Rdb_string_reader* const unp_reader) { const uchar *ptr; size_t len= 0; bool finished= false; - Field_varstring* field_var= static_cast<Field_varstring *>(field); + Field_varstring* const field_var= static_cast<Field_varstring *>(field); uchar *d0= dst; uchar *dst_end= dst + field_var->pack_length(); dst += field_var->length_bytes; @@ -1859,7 +1856,7 @@ static int rdb_unpack_binary_or_utf8_varchar_space_pad( /* Decode the length-emitted encoding here */ while ((ptr= (const uchar*)reader->read(fpi->m_segment_size))) { - char last_byte= ptr[fpi->m_segment_size - 1]; + const char last_byte= ptr[fpi->m_segment_size - 1]; size_t used_bytes; if (last_byte == VARCHAR_CMP_EQUAL_TO_SPACES) // this is the last segment { @@ -1891,7 +1888,7 @@ static int rdb_unpack_binary_or_utf8_varchar_space_pad( } const uchar *src= ptr; - const uchar *src_end= ptr + used_bytes; + const uchar* const src_end= ptr + used_bytes; while (src < src_end) { my_wc_t wc= (src[0] <<8) | src[1]; @@ -1953,7 +1950,7 @@ static int rdb_unpack_binary_or_utf8_varchar_space_pad( static void rdb_make_unpack_unknown( const Rdb_collation_codec *codec __attribute__((__unused__)), - const Field *field, Rdb_pack_field_context *pack_ctx) + const Field* const field, Rdb_pack_field_context* const pack_ctx) { pack_ctx->writer->write(field->ptr, field->pack_length()); } @@ -1978,13 +1975,13 @@ static void rdb_dummy_make_unpack_info( Function of type rdb_index_field_unpack_t */ -static int rdb_unpack_unknown(Rdb_field_packing *fpi, Field *field, - uchar *dst, - Rdb_string_reader *reader, - Rdb_string_reader *unp_reader) +static int rdb_unpack_unknown(Rdb_field_packing* const fpi, Field* const field, + uchar* const dst, + Rdb_string_reader* const reader, + Rdb_string_reader* const unp_reader) { const uchar *ptr; - uint len = fpi->m_unpack_data_len; + const uint len = fpi->m_unpack_data_len; // We don't use anything from the key, so skip over it. if (rdb_skip_max_length(fpi, field, reader)) { @@ -2008,10 +2005,10 @@ static int rdb_unpack_unknown(Rdb_field_packing *fpi, Field *field, */ static void rdb_make_unpack_unknown_varchar( - const Rdb_collation_codec *codec __attribute__((__unused__)), - const Field *field, Rdb_pack_field_context *pack_ctx) + const Rdb_collation_codec* const codec __attribute__((__unused__)), + const Field* const field, Rdb_pack_field_context* const pack_ctx) { - auto f= static_cast<const Field_varstring *>(field); + const auto f= static_cast<const Field_varstring *>(field); uint len= f->length_bytes == 1 ? (uint) *f->ptr : uint2korr(f->ptr); len+= f->length_bytes; pack_ctx->writer->write(field->ptr, len); @@ -2032,16 +2029,17 @@ static void rdb_make_unpack_unknown_varchar( rdb_make_unpack_unknown, rdb_unpack_unknown */ -static int rdb_unpack_unknown_varchar(Rdb_field_packing *fpi, Field *field, +static int rdb_unpack_unknown_varchar(Rdb_field_packing* const fpi, + Field* const field, uchar *dst, - Rdb_string_reader *reader, - Rdb_string_reader *unp_reader) + Rdb_string_reader* const reader, + Rdb_string_reader* const unp_reader) { const uchar *ptr; - uchar *d0= dst; - auto f= static_cast<Field_varstring *>(field); + uchar* const d0= dst; + const auto f= static_cast<Field_varstring *>(field); dst += f->length_bytes; - uint len_bytes= f->length_bytes; + const uint len_bytes= f->length_bytes; // We don't use anything from the key, so skip over it. if (fpi->m_skip_func(fpi, field, reader)) { @@ -2056,7 +2054,7 @@ static int rdb_unpack_unknown_varchar(Rdb_field_packing *fpi, Field *field, if ((ptr= (const uchar*)unp_reader->read(len_bytes))) { memcpy(d0, ptr, len_bytes); - uint len= len_bytes == 1 ? (uint) *ptr : uint2korr(ptr); + const uint len= len_bytes == 1 ? (uint) *ptr : uint2korr(ptr); if ((ptr= (const uchar*)unp_reader->read(len))) { memcpy(dst, ptr, len); @@ -2070,9 +2068,10 @@ static int rdb_unpack_unknown_varchar(Rdb_field_packing *fpi, Field *field, /* Write unpack_data for a "simple" collation */ -static void rdb_write_unpack_simple(Rdb_bit_writer *writer, - const Rdb_collation_codec *codec, - const uchar *src, size_t src_len) +static void rdb_write_unpack_simple(Rdb_bit_writer* const writer, + const Rdb_collation_codec* const codec, + const uchar* const src, + const size_t src_len) { for (uint i= 0; i < src_len; i++) { @@ -2081,10 +2080,10 @@ static void rdb_write_unpack_simple(Rdb_bit_writer *writer, } -static uint rdb_read_unpack_simple(Rdb_bit_reader *reader, - const Rdb_collation_codec *codec, - const uchar *src, size_t src_len, - uchar *dst) +static uint rdb_read_unpack_simple(Rdb_bit_reader* const reader, + const Rdb_collation_codec* const codec, + const uchar* const src, + const size_t &src_len, uchar* const dst) { for (uint i= 0; i < src_len; i++) { @@ -2120,13 +2119,14 @@ static uint rdb_read_unpack_simple(Rdb_bit_reader *reader, */ static void -rdb_make_unpack_simple_varchar(const Rdb_collation_codec* codec, - const Field *field, - Rdb_pack_field_context *pack_ctx) +rdb_make_unpack_simple_varchar(const Rdb_collation_codec* const codec, + const Field* const field, + Rdb_pack_field_context* const pack_ctx) { - auto f= static_cast<const Field_varstring *>(field); - uchar *src= f->ptr + f->length_bytes; - size_t src_len= f->length_bytes == 1 ? (uint) *f->ptr : uint2korr(f->ptr); + const auto f= static_cast<const Field_varstring *>(field); + uchar* const src= f->ptr + f->length_bytes; + const size_t src_len= + f->length_bytes == 1 ? (uint) *f->ptr : uint2korr(f->ptr); Rdb_bit_writer bit_writer(pack_ctx->writer); // The std::min compares characters with bytes, but for simple collations, // mbmaxlen = 1. @@ -2143,16 +2143,17 @@ rdb_make_unpack_simple_varchar(const Rdb_collation_codec* codec, */ int -rdb_unpack_simple_varchar_space_pad(Rdb_field_packing *fpi, Field *field, - uchar *dst, - Rdb_string_reader *reader, - Rdb_string_reader *unp_reader) +rdb_unpack_simple_varchar_space_pad(Rdb_field_packing* const fpi, + Field* const field, + uchar* dst, + Rdb_string_reader* const reader, + Rdb_string_reader * const unp_reader) { const uchar *ptr; size_t len= 0; bool finished= false; uchar *d0= dst; - Field_varstring* field_var= static_cast<Field_varstring*>(field); + const Field_varstring* const field_var= static_cast<Field_varstring*>(field); // For simple collations, char_length is also number of bytes. DBUG_ASSERT((size_t)fpi->m_max_image_len >= field_var->char_length()); uchar *dst_end= dst + field_var->pack_length(); @@ -2186,7 +2187,7 @@ rdb_unpack_simple_varchar_space_pad(Rdb_field_packing *fpi, Field *field, /* Decode the length-emitted encoding here */ while ((ptr= (const uchar*)reader->read(fpi->m_segment_size))) { - char last_byte= ptr[fpi->m_segment_size - 1]; // number of padding bytes + const char last_byte= ptr[fpi->m_segment_size - 1]; // number of padding bytes size_t used_bytes; if (last_byte == VARCHAR_CMP_EQUAL_TO_SPACES) { @@ -2266,11 +2267,11 @@ rdb_unpack_simple_varchar_space_pad(Rdb_field_packing *fpi, Field *field, The VARCHAR variant is in rdb_make_unpack_simple_varchar */ -static void rdb_make_unpack_simple(const Rdb_collation_codec *codec, - const Field *field, - Rdb_pack_field_context *pack_ctx) +static void rdb_make_unpack_simple(const Rdb_collation_codec* const codec, + const Field* const field, + Rdb_pack_field_context* const pack_ctx) { - uchar *src= field->ptr; + const uchar* const src= field->ptr; Rdb_bit_writer bit_writer(pack_ctx->writer); rdb_write_unpack_simple(&bit_writer, codec, src, field->pack_length()); } @@ -2279,14 +2280,14 @@ static void rdb_make_unpack_simple(const Rdb_collation_codec *codec, Function of type rdb_index_field_unpack_t */ -static int rdb_unpack_simple(Rdb_field_packing *fpi, - Field *field __attribute__((__unused__)), - uchar *dst, - Rdb_string_reader *reader, - Rdb_string_reader *unp_reader) +static int rdb_unpack_simple(Rdb_field_packing* const fpi, + Field* const field __attribute__((__unused__)), + uchar* const dst, + Rdb_string_reader* const reader, + Rdb_string_reader* const unp_reader) { const uchar *ptr; - uint len = fpi->m_max_image_len; + const uint len = fpi->m_max_image_len; Rdb_bit_reader bit_reader(unp_reader); if (!(ptr= (const uchar*)reader->read(len))) @@ -2307,6 +2308,10 @@ const int RDB_SPACE_XFRM_SIZE= 32; class Rdb_charset_space_info { public: + Rdb_charset_space_info(const Rdb_charset_space_info&) = delete; + Rdb_charset_space_info& operator=(const Rdb_charset_space_info&) = delete; + Rdb_charset_space_info() = default; + // A few strxfrm'ed space characters, at least RDB_SPACE_XFRM_SIZE bytes std::vector<uchar> spaces_xfrm; @@ -2343,10 +2348,10 @@ rdb_mem_comparable_space; */ static -void rdb_get_mem_comparable_space(const CHARSET_INFO *cs, +void rdb_get_mem_comparable_space(const CHARSET_INFO* const cs, const std::vector<uchar> **xfrm, - size_t *xfrm_len, - size_t *mb_len) + size_t* const xfrm_len, + size_t* const mb_len) { DBUG_ASSERT(cs->number < MY_ALL_CHARSETS_SIZE); if (!rdb_mem_comparable_space[cs->number].get()) @@ -2362,19 +2367,15 @@ void rdb_get_mem_comparable_space(const CHARSET_INFO *cs, // multi-byte form of the ' ' (space) character uchar space_mb[MAX_MULTI_BYTE_CHAR_SIZE]; - size_t space_mb_len= cs->cset->wc_mb(cs, (my_wc_t) cs->pad_char, - space_mb, - space_mb + sizeof(space_mb)); + const size_t space_mb_len= cs->cset->wc_mb(cs, (my_wc_t) cs->pad_char, + space_mb, + space_mb + sizeof(space_mb)); uchar space[20]; // mem-comparable image of the space character - size_t space_len= cs->coll->strnxfrm(cs, - space, sizeof(space), - 1, - space_mb, - space_mb_len, - 0); - Rdb_charset_space_info *info= new Rdb_charset_space_info; + const size_t space_len= cs->coll->strnxfrm(cs, space, sizeof(space), 1, + space_mb, space_mb_len, 0); + Rdb_charset_space_info* const info= new Rdb_charset_space_info; info->space_xfrm_len= space_len; info->space_mb_len= space_mb_len; while (info->spaces_xfrm.size() < RDB_SPACE_XFRM_SIZE) @@ -2398,16 +2399,16 @@ std::array<const Rdb_collation_codec*, MY_ALL_CHARSETS_SIZE> rdb_collation_data; mysql_mutex_t rdb_collation_data_mutex; -static bool rdb_is_collation_supported(const my_core::CHARSET_INFO * cs) +static bool rdb_is_collation_supported(const my_core::CHARSET_INFO* const cs) { return (cs->coll == &my_collation_8bit_simple_ci_handler); } static const Rdb_collation_codec *rdb_init_collation_mapping( - const my_core::CHARSET_INFO *cs) + const my_core::CHARSET_INFO* const cs) { DBUG_ASSERT(cs && cs->state & MY_CS_AVAILABLE); - const Rdb_collation_codec *codec= rdb_collation_data[cs->number]; + const Rdb_collation_codec* codec= rdb_collation_data[cs->number]; if (codec == nullptr && rdb_is_collation_supported(cs)) { @@ -2469,7 +2470,7 @@ static const Rdb_collation_codec *rdb_init_collation_mapping( } -static int get_segment_size_from_collation(const CHARSET_INFO* cs) +static int get_segment_size_from_collation(const CHARSET_INFO* const cs) { int ret; if (cs == &my_charset_utf8mb4_bin || @@ -2526,9 +2527,10 @@ static int get_segment_size_from_collation(const CHARSET_INFO* cs) FALSE - Otherwise */ -bool Rdb_field_packing::setup(const Rdb_key_def *key_descr, const Field *field, - uint keynr_arg, uint key_part_arg, - uint16 key_length) +bool Rdb_field_packing::setup(const Rdb_key_def* const key_descr, + const Field* const field, + const uint &keynr_arg, const uint &key_part_arg, + const uint16 &key_length) { int res= false; enum_field_types type= field ? field->real_type() : MYSQL_TYPE_LONGLONG; @@ -2595,10 +2597,7 @@ bool Rdb_field_packing::setup(const Rdb_key_def *key_descr, const Field *field, case MYSQL_TYPE_LONG_BLOB: case MYSQL_TYPE_BLOB: { - if (key_descr && - key_descr->index_format_min_check( - Rdb_key_def::PRIMARY_FORMAT_VERSION_UPDATE1, - Rdb_key_def::SECONDARY_FORMAT_VERSION_UPDATE1)) + if (key_descr) { // The my_charset_bin collation is special in that it will consider // shorter strings sorting as less than longer strings. @@ -2634,7 +2633,7 @@ bool Rdb_field_packing::setup(const Rdb_key_def *key_descr, const Field *field, const bool is_varchar= (type == MYSQL_TYPE_VARCHAR); const CHARSET_INFO *cs= field->charset(); // max_image_len before chunking is taken into account - int max_image_len_before_chunks= m_max_image_len; + const int max_image_len_before_chunks= m_max_image_len; if (is_varchar) { @@ -2646,7 +2645,7 @@ bool Rdb_field_packing::setup(const Rdb_key_def *key_descr, const Field *field, m_max_image_len= (m_max_image_len/(RDB_ESCAPE_LENGTH-1) + 1) * RDB_ESCAPE_LENGTH; - auto field_var= static_cast<const Field_varstring*>(field); + const auto field_var= static_cast<const Field_varstring*>(field); m_unpack_info_uses_two_bytes= (field_var->field_length + 8 >= 0x100); } @@ -2675,33 +2674,19 @@ bool Rdb_field_packing::setup(const Rdb_key_def *key_descr, const Field *field, if (is_varchar) { - if (!key_descr || - key_descr->index_format_min_check( - Rdb_key_def::PRIMARY_FORMAT_VERSION_UPDATE1, - Rdb_key_def::SECONDARY_FORMAT_VERSION_UPDATE1)) - { - // VARCHARs - // - are compared as if they were space-padded - // - but are not actually space-padded (reading the value back - // produces the original value, without the padding) - m_unpack_func= rdb_unpack_binary_or_utf8_varchar_space_pad; - m_skip_func= rdb_skip_variable_space_pad; - m_pack_func= rdb_pack_with_varchar_space_pad; - m_make_unpack_info_func= rdb_dummy_make_unpack_info; - m_segment_size= get_segment_size_from_collation(cs); - m_max_image_len= - (max_image_len_before_chunks/(m_segment_size-1) + 1) * - m_segment_size; - rdb_get_mem_comparable_space(cs, &space_xfrm, &space_xfrm_len, - &space_mb_len); - } - else - { - // Older variant where VARCHARs were not compared as space-padded: - m_unpack_func= rdb_unpack_binary_or_utf8_varchar; - m_skip_func= rdb_skip_variable_length; - m_pack_func= rdb_pack_with_varchar_encoding; - } + // VARCHARs - are compared as if they were space-padded - but are + // not actually space-padded (reading the value back produces the + // original value, without the padding) + m_unpack_func= rdb_unpack_binary_or_utf8_varchar_space_pad; + m_skip_func= rdb_skip_variable_space_pad; + m_pack_func= rdb_pack_with_varchar_space_pad; + m_make_unpack_info_func= rdb_dummy_make_unpack_info; + m_segment_size= get_segment_size_from_collation(cs); + m_max_image_len= + (max_image_len_before_chunks/(m_segment_size-1) + 1) * + m_segment_size; + rdb_get_mem_comparable_space(cs, &space_xfrm, &space_xfrm_len, + &space_mb_len); } else { @@ -2718,20 +2703,11 @@ bool Rdb_field_packing::setup(const Rdb_key_def *key_descr, const Field *field, res= true; // index-only scans are possible m_unpack_data_len= is_varchar ? 0 : field->field_length; - uint idx= is_varchar ? 0 : 1; + const uint idx= is_varchar ? 0 : 1; const Rdb_collation_codec *codec= nullptr; if (is_varchar) { - if (cs->levels_for_order != 1) - { - // NO_LINT_DEBUG - sql_print_warning("RocksDB: you're trying to create an index " - "with a multi-level collation %s", cs->name); - // NO_LINT_DEBUG - sql_print_warning("MyRocks will handle this collation internally " - " as if it had a NO_PAD attribute."); - } // VARCHAR requires space-padding for doing comparisons // // The check for cs->levels_for_order is to catch @@ -2741,11 +2717,7 @@ bool Rdb_field_packing::setup(const Rdb_key_def *key_descr, const Field *field, // either. // Currently we handle these collations as NO_PAD, even if they have // PAD_SPACE attribute. - if ((!key_descr || - key_descr->index_format_min_check( - Rdb_key_def::PRIMARY_FORMAT_VERSION_UPDATE1, - Rdb_key_def::SECONDARY_FORMAT_VERSION_UPDATE1)) && - cs->levels_for_order == 1) + if (cs->levels_for_order == 1) { m_pack_func= rdb_pack_with_varchar_space_pad; m_skip_func= rdb_skip_variable_space_pad; @@ -2758,6 +2730,12 @@ bool Rdb_field_packing::setup(const Rdb_key_def *key_descr, const Field *field, } else { + // NO_LINT_DEBUG + sql_print_warning("RocksDB: you're trying to create an index " + "with a multi-level collation %s", cs->name); + // NO_LINT_DEBUG + sql_print_warning("MyRocks will handle this collation internally " + " as if it had a NO_PAD attribute."); m_pack_func= rdb_pack_with_varchar_encoding; m_skip_func= rdb_skip_variable_length; } @@ -2824,14 +2802,14 @@ bool Rdb_field_packing::setup(const Rdb_key_def *key_descr, const Field *field, } -Field *Rdb_field_packing::get_field_in_table(const TABLE *tbl) const +Field *Rdb_field_packing::get_field_in_table(const TABLE* const tbl) const { return tbl->key_info[m_keynr].key_part[m_key_part].field; } void Rdb_field_packing::fill_hidden_pk_val(uchar **dst, - longlong hidden_pk_id) const + const longlong &hidden_pk_id) const { DBUG_ASSERT(m_max_image_len == 8); @@ -2877,8 +2855,9 @@ Rdb_tbl_def::~Rdb_tbl_def() ( cf_id, index_nr ) */ -bool Rdb_tbl_def::put_dict(Rdb_dict_manager* dict, rocksdb::WriteBatch *batch, - uchar *key, size_t keylen) +bool Rdb_tbl_def::put_dict(Rdb_dict_manager* const dict, + rocksdb::WriteBatch* const batch, + uchar* const key, const size_t &keylen) { StringBuffer<8 * Rdb_key_def::PACKED_SIZE> indexes; indexes.alloc(Rdb_key_def::VERSION_SIZE + @@ -2887,13 +2866,13 @@ bool Rdb_tbl_def::put_dict(Rdb_dict_manager* dict, rocksdb::WriteBatch *batch, for (uint i = 0; i < m_key_count; i++) { - const std::shared_ptr<const Rdb_key_def>& kd= m_key_descr_arr[i]; + const Rdb_key_def& kd= *m_key_descr_arr[i]; - uchar flags = - (kd->m_is_reverse_cf ? Rdb_key_def::REVERSE_CF_FLAG : 0) | - (kd->m_is_auto_cf ? Rdb_key_def::AUTO_CF_FLAG : 0); + const uchar flags = + (kd.m_is_reverse_cf ? Rdb_key_def::REVERSE_CF_FLAG : 0) | + (kd.m_is_auto_cf ? Rdb_key_def::AUTO_CF_FLAG : 0); - uint cf_id= kd->get_cf()->GetID(); + const uint cf_id= kd.get_cf()->GetID(); /* If cf_id already exists, cf_flags must be the same. To prevent race condition, reading/modifying/committing CF flags @@ -2919,14 +2898,14 @@ bool Rdb_tbl_def::put_dict(Rdb_dict_manager* dict, rocksdb::WriteBatch *batch, } rdb_netstr_append_uint32(&indexes, cf_id); - rdb_netstr_append_uint32(&indexes, kd->m_index_number); - dict->add_or_update_index_cf_mapping(batch, kd->m_index_type, - kd->m_kv_format_version, - kd->m_index_number, cf_id); + rdb_netstr_append_uint32(&indexes, kd.m_index_number); + dict->add_or_update_index_cf_mapping(batch, kd.m_index_type, + kd.m_kv_format_version, + kd.m_index_number, cf_id); } - rocksdb::Slice skey((char*)key, keylen); - rocksdb::Slice svalue(indexes.c_ptr(), indexes.length()); + const rocksdb::Slice skey((char*)key, keylen); + const rocksdb::Slice svalue(indexes.c_ptr(), indexes.length()); dict->put_key(batch, skey, svalue); return false; @@ -2968,7 +2947,7 @@ void Rdb_tbl_def::set_name(const std::string& name) (Rdb_tbl_def in our case). */ const uchar* Rdb_ddl_manager::get_hash_key( - Rdb_tbl_def *rec, size_t *length, + Rdb_tbl_def* const rec, size_t* const length, my_bool not_used __attribute__((__unused__))) { const std::string& dbname_tablename= rec->full_tablename(); @@ -2982,13 +2961,13 @@ const uchar* Rdb_ddl_manager::get_hash_key( invoked by the m_ddl_hash object of type my_core::HASH. It deletes a record (Rdb_tbl_def in our case). */ -void Rdb_ddl_manager::free_hash_elem(void* data) +void Rdb_ddl_manager::free_hash_elem(void* const data) { Rdb_tbl_def* elem= reinterpret_cast<Rdb_tbl_def*>(data); delete elem; } -void Rdb_ddl_manager::erase_index_num(GL_INDEX_ID gl_index_id) +void Rdb_ddl_manager::erase_index_num(const GL_INDEX_ID &gl_index_id) { m_index_num_to_keydef.erase(gl_index_id); } @@ -3209,7 +3188,7 @@ bool Rdb_validate_tbls::compare_to_actual_tables( bool Rdb_ddl_manager::validate_schemas(void) { bool has_errors= false; - std::string datadir= std::string(mysql_real_data_home); + const std::string datadir= std::string(mysql_real_data_home); Rdb_validate_tbls table_list; /* Get the list of tables from the database dictionary */ @@ -3243,15 +3222,16 @@ bool Rdb_ddl_manager::validate_schemas(void) return !has_errors; } -bool Rdb_ddl_manager::init(Rdb_dict_manager *dict_arg, - Rdb_cf_manager *cf_manager, - uint32_t validate_tables) +bool Rdb_ddl_manager::init(Rdb_dict_manager* const dict_arg, + Rdb_cf_manager* const cf_manager, + const uint32_t &validate_tables) { + const ulong TABLE_HASH_SIZE= 32; m_dict= dict_arg; mysql_rwlock_init(0, &m_rwlock); (void) my_hash_init(&m_ddl_hash, /*system_charset_info*/ &my_charset_bin, - 32, 0, 0, + TABLE_HASH_SIZE, 0, 0, (my_hash_get_key) Rdb_ddl_manager::get_hash_key, Rdb_ddl_manager::free_hash_elem, 0); @@ -3259,7 +3239,7 @@ bool Rdb_ddl_manager::init(Rdb_dict_manager *dict_arg, /* Read the data dictionary and populate the hash */ uchar ddl_entry[Rdb_key_def::INDEX_NUMBER_SIZE]; rdb_netbuf_store_index(ddl_entry, Rdb_key_def::DDL_ENTRY_INDEX_START_NUMBER); - rocksdb::Slice ddl_entry_slice((char*)ddl_entry, + const rocksdb::Slice ddl_entry_slice((char*)ddl_entry, Rdb_key_def::INDEX_NUMBER_SIZE); /* Reading data dictionary should always skip bloom filter */ @@ -3273,8 +3253,8 @@ bool Rdb_ddl_manager::init(Rdb_dict_manager *dict_arg, { const uchar *ptr; const uchar *ptr_end; - rocksdb::Slice key= it->key(); - rocksdb::Slice val= it->value(); + const rocksdb::Slice key= it->key(); + const rocksdb::Slice val= it->value(); if (key.size() >= Rdb_key_def::INDEX_NUMBER_SIZE && memcmp(key.data(), ddl_entry, Rdb_key_def::INDEX_NUMBER_SIZE)) @@ -3287,10 +3267,11 @@ bool Rdb_ddl_manager::init(Rdb_dict_manager *dict_arg, return true; } - Rdb_tbl_def *tdef= new Rdb_tbl_def(key, Rdb_key_def::INDEX_NUMBER_SIZE); + Rdb_tbl_def* const tdef= + new Rdb_tbl_def(key, Rdb_key_def::INDEX_NUMBER_SIZE); // Now, read the DDLs. - int real_val_size= val.size() - Rdb_key_def::VERSION_SIZE; + const int real_val_size= val.size() - Rdb_key_def::VERSION_SIZE; if (real_val_size % Rdb_key_def::PACKED_SIZE*2) { sql_print_error("RocksDB: Table_store: invalid keylist for table %s", @@ -3301,7 +3282,7 @@ bool Rdb_ddl_manager::init(Rdb_dict_manager *dict_arg, tdef->m_key_descr_arr= new std::shared_ptr<Rdb_key_def>[tdef->m_key_count]; ptr= reinterpret_cast<const uchar*>(val.data()); - int version= rdb_netbuf_read_uint16(&ptr); + const int version= rdb_netbuf_read_uint16(&ptr); if (version != Rdb_key_def::DDL_ENTRY_INDEX_VERSION) { sql_print_error("RocksDB: DDL ENTRY Version was not expected." @@ -3344,7 +3325,8 @@ bool Rdb_ddl_manager::init(Rdb_dict_manager *dict_arg, return true; } - rocksdb::ColumnFamilyHandle* cfh = cf_manager->get_cf(gl_index_id.cf_id); + rocksdb::ColumnFamilyHandle* const cfh = + cf_manager->get_cf(gl_index_id.cf_id); DBUG_ASSERT(cfh != nullptr); /* @@ -3387,7 +3369,7 @@ bool Rdb_ddl_manager::init(Rdb_dict_manager *dict_arg, if (!it->status().ok()) { - std::string s= it->status().ToString(); + const std::string s= it->status().ToString(); sql_print_error("RocksDB: Table_store: load error: %s", s.c_str()); return true; } @@ -3397,14 +3379,15 @@ bool Rdb_ddl_manager::init(Rdb_dict_manager *dict_arg, } -Rdb_tbl_def* Rdb_ddl_manager::find(const std::string& table_name, bool lock) +Rdb_tbl_def* Rdb_ddl_manager::find(const std::string& table_name, + const bool &lock) { if (lock) { mysql_rwlock_rdlock(&m_rwlock); } - Rdb_tbl_def* rec= reinterpret_cast<Rdb_tbl_def*>( + Rdb_tbl_def* const rec= reinterpret_cast<Rdb_tbl_def*>( my_hash_search(&m_ddl_hash, reinterpret_cast<const uchar*>(table_name.c_str()), table_name.size())); @@ -3421,19 +3404,20 @@ Rdb_tbl_def* Rdb_ddl_manager::find(const std::string& table_name, bool lock) // lock on m_rwlock to make sure the Rdb_key_def is not discarded while we // are finding it. Copying it into 'ret' increments the count making sure // that the object will not be discarded until we are finished with it. -std::shared_ptr<Rdb_key_def> Rdb_ddl_manager::safe_find(GL_INDEX_ID gl_index_id) +std::shared_ptr<const Rdb_key_def> Rdb_ddl_manager::safe_find( + GL_INDEX_ID gl_index_id) { - std::shared_ptr<Rdb_key_def> ret(nullptr); + std::shared_ptr<const Rdb_key_def> ret(nullptr); mysql_rwlock_rdlock(&m_rwlock); auto it= m_index_num_to_keydef.find(gl_index_id); if (it != m_index_num_to_keydef.end()) { - auto table_def = find(it->second.first, false); + const auto table_def = find(it->second.first, false); if (table_def && it->second.second < table_def->m_key_count) { - auto& kd= table_def->m_key_descr_arr[it->second.second]; + const auto &kd= table_def->m_key_descr_arr[it->second.second]; if (kd->max_storage_fmt_length() != 0) { ret = kd; @@ -3470,9 +3454,10 @@ void Rdb_ddl_manager::set_stats( { mysql_rwlock_wrlock(&m_rwlock); for (auto src : stats) { - auto keydef = find(src.second.m_gl_index_id); + const auto& keydef = find(src.second.m_gl_index_id); if (keydef) { keydef->m_stats = src.second; + m_stats2store[keydef->m_stats.m_gl_index_id] = keydef->m_stats; } } mysql_rwlock_unlock(&m_rwlock); @@ -3488,7 +3473,7 @@ void Rdb_ddl_manager::adjust_stats( { for (const auto& src : data) { - auto keydef= find(src.m_gl_index_id); + const auto& keydef= find(src.m_gl_index_id); if (keydef) { keydef->m_stats.merge(src, i == 0, keydef->max_storage_fmt_length()); @@ -3497,7 +3482,7 @@ void Rdb_ddl_manager::adjust_stats( } i++; } - bool should_save_stats= !m_stats2store.empty(); + const bool should_save_stats= !m_stats2store.empty(); mysql_rwlock_unlock(&m_rwlock); if (should_save_stats) { @@ -3506,15 +3491,15 @@ void Rdb_ddl_manager::adjust_stats( } } -void Rdb_ddl_manager::persist_stats(bool sync) +void Rdb_ddl_manager::persist_stats(const bool &sync) { mysql_rwlock_wrlock(&m_rwlock); - auto local_stats2store = std::move(m_stats2store); + const auto local_stats2store = std::move(m_stats2store); m_stats2store.clear(); mysql_rwlock_unlock(&m_rwlock); // Persist stats - std::unique_ptr<rocksdb::WriteBatch> wb = m_dict->begin(); + const std::unique_ptr<rocksdb::WriteBatch> wb = m_dict->begin(); std::vector<Rdb_index_stats> stats; std::transform( local_stats2store.begin(), local_stats2store.end(), @@ -3531,8 +3516,8 @@ void Rdb_ddl_manager::persist_stats(bool sync) on-disk data dictionary. */ -int Rdb_ddl_manager::put_and_write(Rdb_tbl_def *tbl, - rocksdb::WriteBatch *batch) +int Rdb_ddl_manager::put_and_write(Rdb_tbl_def* const tbl, + rocksdb::WriteBatch* const batch) { uchar buf[FN_LEN * 2 + Rdb_key_def::INDEX_NUMBER_SIZE]; uint pos= 0; @@ -3564,7 +3549,7 @@ int Rdb_ddl_manager::put_and_write(Rdb_tbl_def *tbl, See the discussion here: https://reviews.facebook.net/D35925#inline-259167 Tracked by https://github.com/facebook/mysql-5.6/issues/33 */ -int Rdb_ddl_manager::put(Rdb_tbl_def *tbl, bool lock) +int Rdb_ddl_manager::put(Rdb_tbl_def* const tbl, const bool &lock) { Rdb_tbl_def *rec; my_bool result; @@ -3575,7 +3560,7 @@ int Rdb_ddl_manager::put(Rdb_tbl_def *tbl, bool lock) // We have to do this find because 'tbl' is not yet in the list. We need // to find the one we are replacing ('rec') - rec= reinterpret_cast<Rdb_tbl_def*>(find(dbname_tablename, false)); + rec= find(dbname_tablename, false); if (rec) { // this will free the old record. @@ -3594,8 +3579,9 @@ int Rdb_ddl_manager::put(Rdb_tbl_def *tbl, bool lock) } -void Rdb_ddl_manager::remove(Rdb_tbl_def *tbl, - rocksdb::WriteBatch *batch, bool lock) +void Rdb_ddl_manager::remove(Rdb_tbl_def* const tbl, + rocksdb::WriteBatch * const batch, + const bool &lock) { if (lock) mysql_rwlock_wrlock(&m_rwlock); @@ -3610,7 +3596,7 @@ void Rdb_ddl_manager::remove(Rdb_tbl_def *tbl, memcpy(buf + pos, dbname_tablename.c_str(), dbname_tablename.size()); pos += dbname_tablename.size(); - rocksdb::Slice tkey((char*)buf, pos); + const rocksdb::Slice tkey((char*)buf, pos); m_dict->delete_key(batch, tkey); /* The following will also delete the object: */ @@ -3622,7 +3608,7 @@ void Rdb_ddl_manager::remove(Rdb_tbl_def *tbl, bool Rdb_ddl_manager::rename(const std::string& from, const std::string& to, - rocksdb::WriteBatch *batch) + rocksdb::WriteBatch* const batch) { Rdb_tbl_def *rec; Rdb_tbl_def *new_rec; @@ -3675,7 +3661,7 @@ void Rdb_ddl_manager::cleanup() } -int Rdb_ddl_manager::scan_for_tables(Rdb_tables_scanner* tables_scanner) +int Rdb_ddl_manager::scan_for_tables(Rdb_tables_scanner* const tables_scanner) { int i, ret; Rdb_tbl_def *rec; @@ -3705,7 +3691,7 @@ int Rdb_ddl_manager::scan_for_tables(Rdb_tables_scanner* tables_scanner) Rdb_binlog_manager class implementation */ -bool Rdb_binlog_manager::init(Rdb_dict_manager *dict_arg) +bool Rdb_binlog_manager::init(Rdb_dict_manager* const dict_arg) { DBUG_ASSERT(dict_arg != nullptr); m_dict= dict_arg; @@ -3728,21 +3714,22 @@ void Rdb_binlog_manager::cleanup() write succeeded or not is not possible here. @param binlog_name Binlog name @param binlog_pos Binlog pos - @param binlog_gtid Binlog GTID + @param binlog_gtid Binlog max GTID @param batch WriteBatch */ -void Rdb_binlog_manager::update(const char* binlog_name, +void Rdb_binlog_manager::update(const char* const binlog_name, const my_off_t binlog_pos, - const char* binlog_gtid, - rocksdb::WriteBatchBase* batch) + const char* const binlog_max_gtid, + rocksdb::WriteBatchBase* const batch) { if (binlog_name && binlog_pos) { // max binlog length (512) + binlog pos (4) + binlog gtid (57) < 1024 - uchar value_buf[1024]; + const size_t RDB_MAX_BINLOG_INFO_LEN= 1024; + uchar value_buf[RDB_MAX_BINLOG_INFO_LEN]; m_dict->put_key(batch, m_key_slice, pack_value(value_buf, binlog_name, - binlog_pos, binlog_gtid)); + binlog_pos, binlog_max_gtid)); } } @@ -3755,8 +3742,9 @@ void Rdb_binlog_manager::update(const char* binlog_name, true is binlog info was found (valid behavior) false otherwise */ -bool Rdb_binlog_manager::read(char *binlog_name, my_off_t *binlog_pos, - char *binlog_gtid) +bool Rdb_binlog_manager::read(char* const binlog_name, + my_off_t* const binlog_pos, + char* const binlog_gtid) const { bool ret= false; if (binlog_name) @@ -3782,10 +3770,11 @@ bool Rdb_binlog_manager::read(char *binlog_name, my_off_t *binlog_pos, @param binlog_gtid Binlog GTID @return rocksdb::Slice converted from buf and its length */ -rocksdb::Slice Rdb_binlog_manager::pack_value(uchar *buf, - const char* binlog_name, - const my_off_t binlog_pos, - const char* binlog_gtid) +rocksdb::Slice Rdb_binlog_manager::pack_value(uchar* const buf, + const char* const binlog_name, + const my_off_t &binlog_pos, + const char* const binlog_gtid + ) const { uint pack_len= 0; @@ -3794,10 +3783,10 @@ rocksdb::Slice Rdb_binlog_manager::pack_value(uchar *buf, pack_len += Rdb_key_def::VERSION_SIZE; // store binlog file name length - DBUG_ASSERT(strlen(binlog_name) <= 65535); - uint16_t binlog_name_len = strlen(binlog_name); + DBUG_ASSERT(strlen(binlog_name) <= FN_REFLEN); + const uint16_t binlog_name_len = strlen(binlog_name); rdb_netbuf_store_uint16(buf+pack_len, binlog_name_len); - pack_len += 2; + pack_len += sizeof(uint16); // store binlog file name memcpy(buf+pack_len, binlog_name, binlog_name_len); @@ -3805,13 +3794,13 @@ rocksdb::Slice Rdb_binlog_manager::pack_value(uchar *buf, // store binlog pos rdb_netbuf_store_uint32(buf+pack_len, binlog_pos); - pack_len += 4; + pack_len += sizeof(uint32); // store binlog gtid length. // If gtid was not set, store 0 instead - uint16_t binlog_gtid_len = binlog_gtid? strlen(binlog_gtid) : 0; + const uint16_t binlog_gtid_len = binlog_gtid? strlen(binlog_gtid) : 0; rdb_netbuf_store_uint16(buf+pack_len, binlog_gtid_len); - pack_len += 2; + pack_len += sizeof(uint16); if (binlog_gtid_len > 0) { @@ -3831,23 +3820,24 @@ rocksdb::Slice Rdb_binlog_manager::pack_value(uchar *buf, @param[OUT] binlog_gtid Binlog GTID @return true on error */ -bool Rdb_binlog_manager::unpack_value(const uchar *value, char *binlog_name, - my_off_t *binlog_pos, - char *binlog_gtid) +bool Rdb_binlog_manager::unpack_value(const uchar* const value, + char* const binlog_name, + my_off_t* const binlog_pos, + char* const binlog_gtid) const { uint pack_len= 0; DBUG_ASSERT(binlog_pos != nullptr); // read version - uint16_t version= rdb_netbuf_to_uint16(value); + const uint16_t version= rdb_netbuf_to_uint16(value); pack_len += Rdb_key_def::VERSION_SIZE; if (version != Rdb_key_def::BINLOG_INFO_INDEX_NUMBER_VERSION) return true; // read binlog file name length - uint16_t binlog_name_len= rdb_netbuf_to_uint16(value+pack_len); - pack_len += 2; + const uint16_t binlog_name_len= rdb_netbuf_to_uint16(value+pack_len); + pack_len += sizeof(uint16); if (binlog_name_len) { // read and set binlog name @@ -3857,11 +3847,11 @@ bool Rdb_binlog_manager::unpack_value(const uchar *value, char *binlog_name, // read and set binlog pos *binlog_pos= rdb_netbuf_to_uint32(value+pack_len); - pack_len += 4; + pack_len += sizeof(uint32); // read gtid length - uint16_t binlog_gtid_len= rdb_netbuf_to_uint16(value+pack_len); - pack_len += 2; + const uint16_t binlog_gtid_len= rdb_netbuf_to_uint16(value+pack_len); + pack_len += sizeof(uint16); if (binlog_gtid && binlog_gtid_len > 0) { // read and set gtid @@ -3883,8 +3873,8 @@ bool Rdb_binlog_manager::unpack_value(const uchar *value, char *binlog_name, @param[IN] write_batch Handle to storage engine writer. */ void Rdb_binlog_manager::update_slave_gtid_info( - uint id, const char* db, const char* gtid, - rocksdb::WriteBatchBase* write_batch) + const uint &id, const char* const db, const char* const gtid, + rocksdb::WriteBatchBase* const write_batch) { if (id && db && gtid) { // Make sure that if the slave_gtid_info table exists we have a @@ -3910,14 +3900,14 @@ void Rdb_binlog_manager::update_slave_gtid_info( buf += Rdb_key_def::INDEX_NUMBER_SIZE; rdb_netbuf_store_uint32(buf, id); buf += 4; - rocksdb::Slice key_slice = + const rocksdb::Slice key_slice = rocksdb::Slice((const char*)key_buf, buf-key_buf); // Build value uchar value_buf[128]= {0}; DBUG_ASSERT(gtid); - uint db_len= strlen(db); - uint gtid_len= strlen(gtid); + const uint db_len= strlen(db); + const uint gtid_len= strlen(gtid); buf= value_buf; // 1 byte used for flags. Empty here. buf++; @@ -3935,14 +3925,15 @@ void Rdb_binlog_manager::update_slave_gtid_info( buf++; memcpy(buf, gtid, gtid_len); buf += gtid_len; - rocksdb::Slice value_slice = + const rocksdb::Slice value_slice = rocksdb::Slice((const char*)value_buf, buf-value_buf); write_batch->Put(kd->get_cf(), key_slice, value_slice); } } -bool Rdb_dict_manager::init(rocksdb::DB *rdb_dict, Rdb_cf_manager *cf_manager) +bool Rdb_dict_manager::init(rocksdb::DB* const rdb_dict, + Rdb_cf_manager* const cf_manager) { mysql_mutex_init(0, &m_mutex, MY_MUTEX_INIT_FAST); m_db= rdb_dict; @@ -3960,20 +3951,20 @@ bool Rdb_dict_manager::init(rocksdb::DB *rdb_dict, Rdb_cf_manager *cf_manager) return (m_system_cfh == nullptr); } -std::unique_ptr<rocksdb::WriteBatch> Rdb_dict_manager::begin() +std::unique_ptr<rocksdb::WriteBatch> Rdb_dict_manager::begin() const { return std::unique_ptr<rocksdb::WriteBatch>(new rocksdb::WriteBatch); } -void Rdb_dict_manager::put_key(rocksdb::WriteBatchBase *batch, +void Rdb_dict_manager::put_key(rocksdb::WriteBatchBase* const batch, const rocksdb::Slice &key, - const rocksdb::Slice &value) + const rocksdb::Slice &value) const { batch->Put(m_system_cfh, key, value); } rocksdb::Status Rdb_dict_manager::get_value(const rocksdb::Slice &key, - std::string *value) const + std::string* const value) const { rocksdb::ReadOptions options; options.total_order_seek= true; @@ -3986,7 +3977,7 @@ void Rdb_dict_manager::delete_key(rocksdb::WriteBatchBase *batch, batch->Delete(m_system_cfh, key); } -rocksdb::Iterator* Rdb_dict_manager::new_iterator() +rocksdb::Iterator* Rdb_dict_manager::new_iterator() const { /* Reading data dictionary should always skip bloom filter */ rocksdb::ReadOptions read_options; @@ -3994,7 +3985,8 @@ rocksdb::Iterator* Rdb_dict_manager::new_iterator() return m_db->NewIterator(read_options, m_system_cfh); } -int Rdb_dict_manager::commit(rocksdb::WriteBatch *batch, bool sync) +int Rdb_dict_manager::commit(rocksdb::WriteBatch* const batch, const bool &sync) +const { if (!batch) return 1; @@ -4011,7 +4003,7 @@ int Rdb_dict_manager::commit(rocksdb::WriteBatch *batch, bool sync) return res; } -void Rdb_dict_manager::dump_index_id(uchar *netbuf, +void Rdb_dict_manager::dump_index_id(uchar* const netbuf, Rdb_key_def::DATA_DICT_TYPE dict_type, const GL_INDEX_ID &gl_index_id) { @@ -4022,7 +4014,7 @@ void Rdb_dict_manager::dump_index_id(uchar *netbuf, gl_index_id.index_id); } -void Rdb_dict_manager::delete_with_prefix(rocksdb::WriteBatch* batch, +void Rdb_dict_manager::delete_with_prefix(rocksdb::WriteBatch* const batch, Rdb_key_def::DATA_DICT_TYPE dict_type, const GL_INDEX_ID &gl_index_id) const { @@ -4038,13 +4030,13 @@ void Rdb_dict_manager::add_or_update_index_cf_mapping( const uchar m_index_type, const uint16_t kv_version, const uint32_t index_id, - const uint32_t cf_id) + const uint32_t cf_id) const { uchar key_buf[Rdb_key_def::INDEX_NUMBER_SIZE*3]= {0}; uchar value_buf[256]= {0}; GL_INDEX_ID gl_index_id= {cf_id, index_id}; dump_index_id(key_buf, Rdb_key_def::INDEX_INFO, gl_index_id); - rocksdb::Slice key= rocksdb::Slice((char*)key_buf, sizeof(key_buf)); + const rocksdb::Slice key= rocksdb::Slice((char*)key_buf, sizeof(key_buf)); uchar* ptr= value_buf; rdb_netbuf_store_uint16(ptr, Rdb_key_def::INDEX_INFO_VERSION_LATEST); @@ -4054,24 +4046,25 @@ void Rdb_dict_manager::add_or_update_index_cf_mapping( rdb_netbuf_store_uint16(ptr, kv_version); ptr+= 2; - rocksdb::Slice value= rocksdb::Slice((char*)value_buf, ptr-value_buf); + const rocksdb::Slice value= rocksdb::Slice((char*)value_buf, ptr-value_buf); batch->Put(m_system_cfh, key, value); } -void Rdb_dict_manager::add_cf_flags(rocksdb::WriteBatch* batch, - const uint32_t cf_id, - const uint32_t cf_flags) +void Rdb_dict_manager::add_cf_flags(rocksdb::WriteBatch* const batch, + const uint32_t &cf_id, + const uint32_t &cf_flags) const { uchar key_buf[Rdb_key_def::INDEX_NUMBER_SIZE*2]= {0}; uchar value_buf[Rdb_key_def::VERSION_SIZE+ Rdb_key_def::INDEX_NUMBER_SIZE]= {0}; rdb_netbuf_store_uint32(key_buf, Rdb_key_def::CF_DEFINITION); rdb_netbuf_store_uint32(key_buf + Rdb_key_def::INDEX_NUMBER_SIZE, cf_id); - rocksdb::Slice key= rocksdb::Slice((char*)key_buf, sizeof(key_buf)); + const rocksdb::Slice key= rocksdb::Slice((char*)key_buf, sizeof(key_buf)); rdb_netbuf_store_uint16(value_buf, Rdb_key_def::CF_DEFINITION_VERSION); rdb_netbuf_store_uint32(value_buf + Rdb_key_def::VERSION_SIZE, cf_flags); - rocksdb::Slice value= rocksdb::Slice((char*)value_buf, sizeof(value_buf)); + const rocksdb::Slice value= + rocksdb::Slice((char*)value_buf, sizeof(value_buf)); batch->Put(m_system_cfh, key, value); } @@ -4085,19 +4078,19 @@ void Rdb_dict_manager::delete_index_info(rocksdb::WriteBatch* batch, bool Rdb_dict_manager::get_index_info(const GL_INDEX_ID &gl_index_id, uint16_t *m_index_dict_version, uchar *m_index_type, - uint16_t *kv_version) + uint16_t *kv_version) const { bool found= false; bool error= false; std::string value; uchar key_buf[Rdb_key_def::INDEX_NUMBER_SIZE*3]= {0}; dump_index_id(key_buf, Rdb_key_def::INDEX_INFO, gl_index_id); - rocksdb::Slice key= rocksdb::Slice((char*)key_buf, sizeof(key_buf)); + const rocksdb::Slice &key= rocksdb::Slice((char*)key_buf, sizeof(key_buf)); - rocksdb::Status status= get_value(key, &value); + const rocksdb::Status &status= get_value(key, &value); if (status.ok()) { - const uchar* val= (const uchar*)value.c_str(); + const uchar* const val= (const uchar*)value.c_str(); const uchar* ptr= val; *m_index_dict_version= rdb_netbuf_to_uint16(val); *kv_version= 0; @@ -4148,16 +4141,17 @@ bool Rdb_dict_manager::get_index_info(const GL_INDEX_ID &gl_index_id, return found; } -bool Rdb_dict_manager::get_cf_flags(const uint32_t cf_id, uint32_t *cf_flags) +bool Rdb_dict_manager::get_cf_flags(const uint32_t &cf_id, + uint32_t* const cf_flags) const { bool found= false; std::string value; uchar key_buf[Rdb_key_def::INDEX_NUMBER_SIZE*2]= {0}; rdb_netbuf_store_uint32(key_buf, Rdb_key_def::CF_DEFINITION); rdb_netbuf_store_uint32(key_buf + Rdb_key_def::INDEX_NUMBER_SIZE, cf_id); - rocksdb::Slice key= rocksdb::Slice((char*)key_buf, sizeof(key_buf)); + const rocksdb::Slice key= rocksdb::Slice((char*)key_buf, sizeof(key_buf)); - rocksdb::Status status= get_value(key, &value); + const rocksdb::Status status= get_value(key, &value); if (status.ok()) { const uchar* val= (const uchar*)value.c_str(); @@ -4177,22 +4171,22 @@ bool Rdb_dict_manager::get_cf_flags(const uint32_t cf_id, uint32_t *cf_flags) ongoing creation. */ void Rdb_dict_manager::get_ongoing_index_operation( - std::vector<GL_INDEX_ID>* gl_index_ids, - Rdb_key_def::DATA_DICT_TYPE dd_type) + std::vector<GL_INDEX_ID>* const gl_index_ids, + Rdb_key_def::DATA_DICT_TYPE dd_type) const { DBUG_ASSERT(dd_type == Rdb_key_def::DDL_DROP_INDEX_ONGOING || dd_type == Rdb_key_def::DDL_CREATE_INDEX_ONGOING); uchar index_buf[Rdb_key_def::INDEX_NUMBER_SIZE]; rdb_netbuf_store_uint32(index_buf, dd_type); - rocksdb::Slice index_slice(reinterpret_cast<char*>(index_buf), + const rocksdb::Slice index_slice(reinterpret_cast<char*>(index_buf), Rdb_key_def::INDEX_NUMBER_SIZE); rocksdb::Iterator* it= new_iterator(); for (it->Seek(index_slice); it->Valid(); it->Next()) { rocksdb::Slice key= it->key(); - const uchar* ptr= (const uchar*)key.data(); + const uchar* const ptr= (const uchar*)key.data(); /* Ongoing drop/create index operations require key to be of the form: @@ -4226,7 +4220,7 @@ void Rdb_dict_manager::get_ongoing_index_operation( */ bool Rdb_dict_manager::is_index_operation_ongoing( const GL_INDEX_ID& gl_index_id, - Rdb_key_def::DATA_DICT_TYPE dd_type) + Rdb_key_def::DATA_DICT_TYPE dd_type) const { DBUG_ASSERT(dd_type == Rdb_key_def::DDL_DROP_INDEX_ONGOING || dd_type == Rdb_key_def::DDL_CREATE_INDEX_ONGOING); @@ -4235,9 +4229,9 @@ bool Rdb_dict_manager::is_index_operation_ongoing( std::string value; uchar key_buf[Rdb_key_def::INDEX_NUMBER_SIZE*3]= {0}; dump_index_id(key_buf, dd_type, gl_index_id); - rocksdb::Slice key= rocksdb::Slice((char*)key_buf, sizeof(key_buf)); + const rocksdb::Slice key= rocksdb::Slice((char*)key_buf, sizeof(key_buf)); - rocksdb::Status status= get_value(key, &value); + const rocksdb::Status status= get_value(key, &value); if (status.ok()) { found= true; @@ -4250,9 +4244,9 @@ bool Rdb_dict_manager::is_index_operation_ongoing( by drop_index_thread, or to track online index creation. */ void Rdb_dict_manager::start_ongoing_index_operation( - rocksdb::WriteBatch* batch, + rocksdb::WriteBatch* const batch, const GL_INDEX_ID& gl_index_id, - Rdb_key_def::DATA_DICT_TYPE dd_type) + Rdb_key_def::DATA_DICT_TYPE dd_type) const { DBUG_ASSERT(dd_type == Rdb_key_def::DDL_DROP_INDEX_ONGOING || dd_type == Rdb_key_def::DDL_CREATE_INDEX_ONGOING); @@ -4273,8 +4267,9 @@ void Rdb_dict_manager::start_ongoing_index_operation( Rdb_key_def::DDL_CREATE_INDEX_ONGOING_VERSION); } - rocksdb::Slice key= rocksdb::Slice((char*)key_buf, sizeof(key_buf)); - rocksdb::Slice value= rocksdb::Slice((char*)value_buf, sizeof(value_buf)); + const rocksdb::Slice key= rocksdb::Slice((char*)key_buf, sizeof(key_buf)); + const rocksdb::Slice value= + rocksdb::Slice((char*)value_buf, sizeof(value_buf)); batch->Put(m_system_cfh, key, value); } @@ -4282,9 +4277,10 @@ void Rdb_dict_manager::start_ongoing_index_operation( Removing index_id from data dictionary to confirm drop_index_thread completed dropping entire key/values of the index_id */ -void Rdb_dict_manager::end_ongoing_index_operation(rocksdb::WriteBatch* batch, - const GL_INDEX_ID& gl_index_id, - Rdb_key_def::DATA_DICT_TYPE dd_type) +void Rdb_dict_manager::end_ongoing_index_operation( + rocksdb::WriteBatch* const batch, + const GL_INDEX_ID& gl_index_id, + Rdb_key_def::DATA_DICT_TYPE dd_type) const { DBUG_ASSERT(dd_type == Rdb_key_def::DDL_DROP_INDEX_ONGOING || dd_type == Rdb_key_def::DDL_CREATE_INDEX_ONGOING); @@ -4296,7 +4292,7 @@ void Rdb_dict_manager::end_ongoing_index_operation(rocksdb::WriteBatch* batch, Returning true if there is no target index ids to be removed by drop_index_thread */ -bool Rdb_dict_manager::is_drop_index_empty() +bool Rdb_dict_manager::is_drop_index_empty() const { std::vector<GL_INDEX_ID> gl_index_ids; get_ongoing_drop_indexes(&gl_index_ids); @@ -4308,9 +4304,9 @@ bool Rdb_dict_manager::is_drop_index_empty() that dropping indexes started, and adding data dictionary so that all associated indexes to be removed */ -void Rdb_dict_manager::add_drop_table(std::shared_ptr<Rdb_key_def>* key_descr, - uint32 n_keys, - rocksdb::WriteBatch *batch) +void Rdb_dict_manager::add_drop_table(std::shared_ptr<Rdb_key_def>* const key_descr, + const uint32 &n_keys, + rocksdb::WriteBatch* const batch) const { std::unordered_set<GL_INDEX_ID> dropped_index_ids; for (uint32 i = 0; i < n_keys; i++) @@ -4328,7 +4324,7 @@ void Rdb_dict_manager::add_drop_table(std::shared_ptr<Rdb_key_def>* key_descr, */ void Rdb_dict_manager::add_drop_index( const std::unordered_set<GL_INDEX_ID>& gl_index_ids, - rocksdb::WriteBatch *batch) + rocksdb::WriteBatch* const batch) const { for (const auto& gl_index_id : gl_index_ids) { @@ -4344,7 +4340,7 @@ void Rdb_dict_manager::add_drop_index( */ void Rdb_dict_manager::add_create_index( const std::unordered_set<GL_INDEX_ID>& gl_index_ids, - rocksdb::WriteBatch *batch) + rocksdb::WriteBatch* const batch) const { for (const auto& gl_index_id : gl_index_ids) { @@ -4361,13 +4357,13 @@ void Rdb_dict_manager::add_create_index( */ void Rdb_dict_manager::finish_indexes_operation( const std::unordered_set<GL_INDEX_ID>& gl_index_ids, - Rdb_key_def::DATA_DICT_TYPE dd_type) + Rdb_key_def::DATA_DICT_TYPE dd_type) const { DBUG_ASSERT(dd_type == Rdb_key_def::DDL_DROP_INDEX_ONGOING || dd_type == Rdb_key_def::DDL_CREATE_INDEX_ONGOING); - std::unique_ptr<rocksdb::WriteBatch> wb= begin(); - rocksdb::WriteBatch *batch= wb.get(); + const std::unique_ptr<rocksdb::WriteBatch> wb= begin(); + rocksdb::WriteBatch* const batch= wb.get(); for (const auto& gl_index_id : gl_index_ids) { @@ -4395,7 +4391,7 @@ void Rdb_dict_manager::finish_indexes_operation( Rdb_dict_manager (at startup). If there is any index ids that are drop ongoing, printing out messages for diagnostics purposes. */ -void Rdb_dict_manager::resume_drop_indexes() +void Rdb_dict_manager::resume_drop_indexes() const { std::vector<GL_INDEX_ID> gl_index_ids; get_ongoing_drop_indexes(&gl_index_ids); @@ -4418,10 +4414,10 @@ void Rdb_dict_manager::resume_drop_indexes() } } -void Rdb_dict_manager::rollback_ongoing_index_creation() +void Rdb_dict_manager::rollback_ongoing_index_creation() const { - std::unique_ptr<rocksdb::WriteBatch> wb= begin(); - rocksdb::WriteBatch *batch= wb.get(); + const std::unique_ptr<rocksdb::WriteBatch> wb= begin(); + rocksdb::WriteBatch* const batch= wb.get(); std::vector<GL_INDEX_ID> gl_index_ids; get_ongoing_create_indexes(&gl_index_ids); @@ -4441,9 +4437,9 @@ void Rdb_dict_manager::rollback_ongoing_index_creation() } void Rdb_dict_manager::log_start_drop_table( - const std::shared_ptr<Rdb_key_def>* key_descr, - uint32 n_keys, - const char* log_action) + const std::shared_ptr<Rdb_key_def>* const key_descr, + const uint32 &n_keys, + const char* const log_action) const { for (uint32 i = 0; i < n_keys; i++) { log_start_drop_index(key_descr[i]->get_gl_index_id(), log_action); @@ -4451,7 +4447,7 @@ void Rdb_dict_manager::log_start_drop_table( } void Rdb_dict_manager::log_start_drop_index(GL_INDEX_ID gl_index_id, - const char* log_action) + const char* log_action) const { uint16 m_index_dict_version= 0; uchar m_index_type= 0; @@ -4468,16 +4464,16 @@ void Rdb_dict_manager::log_start_drop_index(GL_INDEX_ID gl_index_id, log_action, gl_index_id.cf_id, gl_index_id.index_id); } -bool Rdb_dict_manager::get_max_index_id(uint32_t *index_id) +bool Rdb_dict_manager::get_max_index_id(uint32_t* const index_id) const { bool found= false; std::string value; - rocksdb::Status status= get_value(m_key_slice_max_index_id, &value); + const rocksdb::Status status= get_value(m_key_slice_max_index_id, &value); if (status.ok()) { - const uchar* val= (const uchar*)value.c_str(); - uint16_t version= rdb_netbuf_to_uint16(val); + const uchar* const val= (const uchar*)value.c_str(); + const uint16_t &version= rdb_netbuf_to_uint16(val); if (version == Rdb_key_def::MAX_INDEX_ID_VERSION) { *index_id= rdb_netbuf_to_uint32(val+Rdb_key_def::VERSION_SIZE); @@ -4487,8 +4483,8 @@ bool Rdb_dict_manager::get_max_index_id(uint32_t *index_id) return found; } -bool Rdb_dict_manager::update_max_index_id(rocksdb::WriteBatch* batch, - const uint32_t index_id) +bool Rdb_dict_manager::update_max_index_id(rocksdb::WriteBatch* const batch, + const uint32_t &index_id) const { DBUG_ASSERT(batch != nullptr); @@ -4509,15 +4505,14 @@ bool Rdb_dict_manager::update_max_index_id(rocksdb::WriteBatch* batch, {0}; rdb_netbuf_store_uint16(value_buf, Rdb_key_def::MAX_INDEX_ID_VERSION); rdb_netbuf_store_uint32(value_buf + Rdb_key_def::VERSION_SIZE, index_id); - rocksdb::Slice value= rocksdb::Slice((char*)value_buf, sizeof(value_buf)); + const rocksdb::Slice value= + rocksdb::Slice((char*)value_buf, sizeof(value_buf)); batch->Put(m_system_cfh, m_key_slice_max_index_id, value); return false; } -void Rdb_dict_manager::add_stats( - rocksdb::WriteBatch* batch, - const std::vector<Rdb_index_stats>& stats -) +void Rdb_dict_manager::add_stats(rocksdb::WriteBatch* const batch, + const std::vector<Rdb_index_stats>& stats) const { DBUG_ASSERT(batch != nullptr); @@ -4527,7 +4522,7 @@ void Rdb_dict_manager::add_stats( // IndexStats::materialize takes complete care of serialization including // storing the version - auto value = Rdb_index_stats::materialize( + const auto value = Rdb_index_stats::materialize( std::vector<Rdb_index_stats>{it}, 1.); batch->Put( @@ -4538,13 +4533,13 @@ void Rdb_dict_manager::add_stats( } } -Rdb_index_stats Rdb_dict_manager::get_stats(GL_INDEX_ID gl_index_id) +Rdb_index_stats Rdb_dict_manager::get_stats(GL_INDEX_ID gl_index_id) const { uchar key_buf[Rdb_key_def::INDEX_NUMBER_SIZE*3]= {0}; dump_index_id(key_buf, Rdb_key_def::INDEX_STATISTICS, gl_index_id); std::string value; - rocksdb::Status status= get_value( + const rocksdb::Status status= get_value( rocksdb::Slice(reinterpret_cast<char*>(key_buf), sizeof(key_buf)), &value); if (status.ok()) @@ -4560,7 +4555,8 @@ Rdb_index_stats Rdb_dict_manager::get_stats(GL_INDEX_ID gl_index_id) return Rdb_index_stats(); } -uint Rdb_seq_generator::get_and_update_next_number(Rdb_dict_manager *dict) +uint +Rdb_seq_generator::get_and_update_next_number(Rdb_dict_manager* const dict) { DBUG_ASSERT(dict != nullptr); @@ -4569,8 +4565,8 @@ uint Rdb_seq_generator::get_and_update_next_number(Rdb_dict_manager *dict) res= m_next_number++; - std::unique_ptr<rocksdb::WriteBatch> wb= dict->begin(); - rocksdb::WriteBatch *batch= wb.get(); + const std::unique_ptr<rocksdb::WriteBatch> wb= dict->begin(); + rocksdb::WriteBatch* const batch= wb.get(); DBUG_ASSERT(batch != nullptr); dict->update_max_index_id(batch, res); diff --git a/storage/rocksdb/rdb_datadic.h b/storage/rocksdb/rdb_datadic.h index 7c277ddad7d..91cb9b8d563 100644 --- a/storage/rocksdb/rdb_datadic.h +++ b/storage/rocksdb/rdb_datadic.h @@ -59,7 +59,10 @@ class Rdb_ddl_manager; class Rdb_pack_field_context { public: - explicit Rdb_pack_field_context(Rdb_string_writer *writer_arg) : + Rdb_pack_field_context(const Rdb_pack_field_context&) = delete; + Rdb_pack_field_context& operator=(const Rdb_pack_field_context&) = delete; + + explicit Rdb_pack_field_context(Rdb_string_writer* const writer_arg) : writer(writer_arg) {} @@ -164,44 +167,50 @@ class Rdb_key_def { public: /* Convert a key from KeyTupleFormat to mem-comparable form */ - uint pack_index_tuple(TABLE *tbl, uchar *pack_buffer, uchar *packed_tuple, - const uchar *key_tuple, key_part_map keypart_map) const; + uint pack_index_tuple(TABLE* const tbl, uchar* const pack_buffer, + uchar* const packed_tuple, + const uchar* const key_tuple, + const key_part_map &keypart_map) const; /* Convert a key from Table->record format to mem-comparable form */ - uint pack_record(const TABLE *tbl, uchar *pack_buffer, const uchar *record, - uchar *packed_tuple, Rdb_string_writer *unpack_info, - bool should_store_checksums, - longlong hidden_pk_id= 0, uint n_key_parts= 0, - uint *n_null_fields= nullptr) const; + uint pack_record(const TABLE* const tbl, uchar* const pack_buffer, + const uchar* const record, + uchar* const packed_tuple, + Rdb_string_writer* const unpack_info, + const bool &should_store_row_debug_checksums, + const longlong &hidden_pk_id= 0, uint n_key_parts= 0, + uint* const n_null_fields= nullptr) const; /* Pack the hidden primary key into mem-comparable form. */ - uint pack_hidden_pk(longlong hidden_pk_id, - uchar *packed_tuple) const; - int unpack_record(TABLE *table, uchar *buf, const rocksdb::Slice *packed_key, - const rocksdb::Slice *unpack_info, bool verify_checksums) + uint pack_hidden_pk(const longlong &hidden_pk_id, + uchar* const packed_tuple) const; + int unpack_record(TABLE* const table, uchar* const buf, + const rocksdb::Slice* const packed_key, + const rocksdb::Slice* const unpack_info, + const bool &verify_row_debug_checksums) const; static bool unpack_info_has_checksum(const rocksdb::Slice& unpack_info); int compare_keys(const rocksdb::Slice *key1, const rocksdb::Slice *key2, - std::size_t* column_index) const; + std::size_t* const column_index) const; - size_t key_length(TABLE *table, const rocksdb::Slice &key) const; + size_t key_length(const TABLE* const table, const rocksdb::Slice &key) const; /* Get the key that is the "infimum" for this index */ - inline void get_infimum_key(uchar *key, uint *size) const + inline void get_infimum_key(uchar* const key, uint* const size) const { rdb_netbuf_store_index(key, m_index_number); *size= INDEX_NUMBER_SIZE; } /* Get the key that is a "supremum" for this index */ - inline void get_supremum_key(uchar *key, uint *size) const + inline void get_supremum_key(uchar* const key, uint* const size) const { rdb_netbuf_store_index(key, m_index_number+1); *size= INDEX_NUMBER_SIZE; } /* Make a key that is right after the given key. */ - static int successor(uchar *packed_tuple, uint len); + static int successor(uchar* const packed_tuple, const uint &len); /* This can be used to compare prefixes. @@ -252,14 +261,15 @@ public: GL_INDEX_ID get_gl_index_id() const { - GL_INDEX_ID gl_index_id = { m_cf_handle->GetID(), m_index_number }; + const GL_INDEX_ID gl_index_id = { m_cf_handle->GetID(), m_index_number }; return gl_index_id; } /* Must only be called for secondary keys: */ - uint get_primary_key_tuple(TABLE *tbl, - const std::shared_ptr<const Rdb_key_def>& pk_descr, - const rocksdb::Slice *key, uchar *pk_buffer) const; + uint get_primary_key_tuple(const TABLE* const tbl, + const Rdb_key_def& pk_descr, + const rocksdb::Slice* const key, + uchar* const pk_buffer) const; /* Return max length of mem-comparable form */ uint max_storage_fmt_length() const @@ -288,6 +298,7 @@ public: return m_name; } + Rdb_key_def& operator=(const Rdb_key_def&) = delete; Rdb_key_def(const Rdb_key_def& k); Rdb_key_def(uint indexnr_arg, uint keyno_arg, rocksdb::ColumnFamilyHandle* cf_handle_arg, @@ -379,31 +390,31 @@ public: SECONDARY_FORMAT_VERSION_LATEST= SECONDARY_FORMAT_VERSION_UPDATE1, }; - void setup(const TABLE *table, const Rdb_tbl_def *tbl_def); + void setup(const TABLE* const table, const Rdb_tbl_def* const tbl_def); rocksdb::ColumnFamilyHandle *get_cf() const { return m_cf_handle; } /* Check if keypart #kp can be unpacked from index tuple */ - inline bool can_unpack(uint kp) const; + inline bool can_unpack(const uint &kp) const; /* Check if keypart #kp needs unpack info */ - inline bool has_unpack_info(uint kp) const; + inline bool has_unpack_info(const uint &kp) const; /* Check if given table has a primary key */ - static bool table_has_hidden_pk(const TABLE* table); + static bool table_has_hidden_pk(const TABLE* const table); - void report_checksum_mismatch(bool is_key, const char *data, - size_t data_size) const; + void report_checksum_mismatch(const bool &is_key, const char* const data, + const size_t data_size) const; /* Check if index is at least pk_min if it is a PK, or at least sk_min if SK.*/ - bool index_format_min_check(int pk_min, int sk_min) const; + bool index_format_min_check(const int &pk_min, const int &sk_min) const; private: #ifndef DBUG_OFF - inline bool is_storage_available(int offset, int needed) const + inline bool is_storage_available(const int &offset, const int &needed) const { - int storage_length= static_cast<int>(max_storage_fmt_length()); + const int storage_length= static_cast<int>(max_storage_fmt_length()); return (storage_length - offset) >= needed; } #endif // DBUG_OFF @@ -497,6 +508,10 @@ extern std::array<const Rdb_collation_codec*, MY_ALL_CHARSETS_SIZE> class Rdb_field_packing { public: + Rdb_field_packing(const Rdb_field_packing&) = delete; + Rdb_field_packing& operator=(const Rdb_field_packing&) = delete; + Rdb_field_packing() = default; + /* Length of mem-comparable image of the field, in bytes */ int m_max_image_len; @@ -577,10 +592,11 @@ private: uint m_keynr; uint m_key_part; public: - bool setup(const Rdb_key_def *key_descr, const Field *field, - uint keynr_arg, uint key_part_arg, uint16 key_length); - Field *get_field_in_table(const TABLE *tbl) const; - void fill_hidden_pk_val(uchar **dst, longlong hidden_pk_id) const; + bool setup(const Rdb_key_def* const key_descr, const Field* const field, + const uint &keynr_arg, const uint &key_part_arg, + const uint16 &key_length); + Field *get_field_in_table(const TABLE* const tbl) const; + void fill_hidden_pk_val(uchar **dst, const longlong &hidden_pk_id) const; }; /* @@ -593,6 +609,8 @@ public: class Rdb_field_encoder { public: + Rdb_field_encoder(const Rdb_field_encoder&) = delete; + Rdb_field_encoder& operator=(const Rdb_field_encoder&) = delete; /* STORE_NONE is set when a column can be decoded solely from their mem-comparable form. @@ -633,13 +651,13 @@ inline Field* Rdb_key_def::get_table_field_for_part_no(TABLE *table, return m_pack_info[part_no].get_field_in_table(table); } -inline bool Rdb_key_def::can_unpack(uint kp) const +inline bool Rdb_key_def::can_unpack(const uint &kp) const { DBUG_ASSERT(kp < m_key_parts); return (m_pack_info[kp].m_unpack_func != nullptr); } -inline bool Rdb_key_def::has_unpack_info(uint kp) const +inline bool Rdb_key_def::has_unpack_info(const uint &kp) const { DBUG_ASSERT(kp < m_key_parts); return m_pack_info[kp].uses_unpack_info(); @@ -671,19 +689,22 @@ class Rdb_tbl_def void set_name(const std::string& name); public: + Rdb_tbl_def(const Rdb_tbl_def&) = delete; + Rdb_tbl_def& operator=(const Rdb_tbl_def&) = delete; + explicit Rdb_tbl_def(const std::string& name) : m_key_descr_arr(nullptr), m_hidden_pk_val(1), m_auto_incr_val(1) { set_name(name); } - Rdb_tbl_def(const char* name, size_t len) : + Rdb_tbl_def(const char* const name, const size_t &len) : m_key_descr_arr(nullptr), m_hidden_pk_val(1), m_auto_incr_val(1) { set_name(std::string(name, len)); } - explicit Rdb_tbl_def(const rocksdb::Slice& slice, size_t pos= 0) : + explicit Rdb_tbl_def(const rocksdb::Slice& slice, const size_t &pos= 0) : m_key_descr_arr(nullptr), m_hidden_pk_val(1), m_auto_incr_val(1) { set_name(std::string(slice.data() + pos, slice.size() - pos)); @@ -703,8 +724,8 @@ class Rdb_tbl_def /* Is this a system table */ bool m_is_mysql_system_table; - bool put_dict(Rdb_dict_manager *dict, rocksdb::WriteBatch *batch, - uchar *key, size_t keylen); + bool put_dict(Rdb_dict_manager* const dict, rocksdb::WriteBatch* const batch, + uchar* const key, const size_t &keylen); const std::string& full_tablename() const { return m_dbname_tablename; } const std::string& base_dbname() const { return m_dbname; } @@ -724,13 +745,17 @@ class Rdb_seq_generator mysql_mutex_t m_mutex; public: - void init(uint initial_number) + Rdb_seq_generator(const Rdb_seq_generator&) = delete; + Rdb_seq_generator& operator=(const Rdb_seq_generator&) = delete; + Rdb_seq_generator() = default; + + void init(const uint &initial_number) { mysql_mutex_init(0 , &m_mutex, MY_MUTEX_INIT_FAST); m_next_number= initial_number; } - uint get_and_update_next_number(Rdb_dict_manager *dict); + uint get_and_update_next_number(Rdb_dict_manager* const dict); void cleanup() { @@ -766,46 +791,54 @@ class Rdb_ddl_manager // It is produced by event listener (ie compaction and flush threads) // and consumed by the rocksdb background thread std::map<GL_INDEX_ID, Rdb_index_stats> m_stats2store; + + const std::shared_ptr<Rdb_key_def>& find( + GL_INDEX_ID gl_index_id); public: + Rdb_ddl_manager(const Rdb_ddl_manager&) = delete; + Rdb_ddl_manager& operator=(const Rdb_ddl_manager&) = delete; + Rdb_ddl_manager() {} + /* Load the data dictionary from on-disk storage */ - bool init(Rdb_dict_manager *dict_arg, Rdb_cf_manager *cf_manager, - uint32_t validate_tables); + bool init(Rdb_dict_manager* const dict_arg, Rdb_cf_manager* const cf_manager, + const uint32_t &validate_tables); void cleanup(); - Rdb_tbl_def* find(const std::string& table_name, bool lock= true); - const std::shared_ptr<Rdb_key_def>& find(GL_INDEX_ID gl_index_id); - std::shared_ptr<Rdb_key_def> safe_find(GL_INDEX_ID gl_index_id); + Rdb_tbl_def* find(const std::string& table_name, const bool &lock= true); + std::shared_ptr<const Rdb_key_def> safe_find(GL_INDEX_ID gl_index_id); void set_stats( const std::unordered_map<GL_INDEX_ID, Rdb_index_stats>& stats); void adjust_stats( const std::vector<Rdb_index_stats>& new_data, const std::vector<Rdb_index_stats>& deleted_data =std::vector<Rdb_index_stats>()); - void persist_stats(bool sync = false); + void persist_stats(const bool &sync = false); /* Modify the mapping and write it to on-disk storage */ - int put_and_write(Rdb_tbl_def *key_descr, rocksdb::WriteBatch *batch); - void remove(Rdb_tbl_def *rec, rocksdb::WriteBatch *batch, bool lock= true); + int put_and_write(Rdb_tbl_def* const key_descr, + rocksdb::WriteBatch* const batch); + void remove(Rdb_tbl_def* const rec, rocksdb::WriteBatch* const batch, + const bool &lock= true); bool rename(const std::string& from, const std::string& to, - rocksdb::WriteBatch *batch); + rocksdb::WriteBatch* const batch); - uint get_and_update_next_number(Rdb_dict_manager *dict) + uint get_and_update_next_number(Rdb_dict_manager* const dict) { return m_sequence.get_and_update_next_number(dict); } /* Walk the data dictionary */ int scan_for_tables(Rdb_tables_scanner* tables_scanner); - void erase_index_num(GL_INDEX_ID gl_index_id); + void erase_index_num(const GL_INDEX_ID &gl_index_id); private: /* Put the data into in-memory table (only) */ - int put(Rdb_tbl_def *key_descr, bool lock= true); + int put(Rdb_tbl_def* const key_descr, const bool &lock= true); /* Helper functions to be passed to my_core::HASH object */ - static const uchar* get_hash_key(Rdb_tbl_def *rec, size_t *length, + static const uchar* get_hash_key(Rdb_tbl_def* const rec, size_t* const length, my_bool not_used __attribute__((unused))); - static void free_hash_elem(void* data); + static void free_hash_elem(void* const data); bool validate_schemas(); }; @@ -829,25 +862,32 @@ private: class Rdb_binlog_manager { public: - bool init(Rdb_dict_manager *dict); + Rdb_binlog_manager(const Rdb_binlog_manager&) = delete; + Rdb_binlog_manager& operator=(const Rdb_binlog_manager&) = delete; + Rdb_binlog_manager() = default; + + bool init(Rdb_dict_manager* const dict); void cleanup(); - void update(const char* binlog_name, const my_off_t binlog_pos, - const char* binlog_gtid, rocksdb::WriteBatchBase* batch); - bool read(char* binlog_name, my_off_t* binlog_pos, char* binlog_gtid); - void update_slave_gtid_info(uint id, const char* db, const char* gtid, - rocksdb::WriteBatchBase *write_batch); + void update(const char* const binlog_name, const my_off_t binlog_pos, + const char* const binlog_max_gtid, + rocksdb::WriteBatchBase* const batch); + bool read(char* const binlog_name, my_off_t* const binlog_pos, + char* const binlog_gtid) const; + void update_slave_gtid_info(const uint &id, const char* const db, + const char* const gtid, + rocksdb::WriteBatchBase* const write_batch); private: Rdb_dict_manager *m_dict= nullptr; uchar m_key_buf[Rdb_key_def::INDEX_NUMBER_SIZE]= {0}; rocksdb::Slice m_key_slice; - rocksdb::Slice pack_value(uchar *buf, - const char *binlog_name, - const my_off_t binlog_pos, - const char *binlog_gtid); - bool unpack_value(const uchar *value, char *binlog_name, - my_off_t *binlog_pos, char *binlog_gtid); + rocksdb::Slice pack_value(uchar* const buf, + const char* const binlog_name, + const my_off_t &binlog_pos, + const char* const binlog_gtid) const; + bool unpack_value(const uchar* const value, char* const binlog_name, + my_off_t* const binlog_pos, char* const binlog_gtid) const; std::atomic<Rdb_tbl_def*> m_slave_gtid_info_tbl; }; @@ -915,21 +955,25 @@ private: uchar m_key_buf_max_index_id[Rdb_key_def::INDEX_NUMBER_SIZE]= {0}; rocksdb::Slice m_key_slice_max_index_id; - static void dump_index_id(uchar *netbuf, + static void dump_index_id(uchar* const netbuf, Rdb_key_def::DATA_DICT_TYPE dict_type, const GL_INDEX_ID &gl_index_id); - void delete_with_prefix(rocksdb::WriteBatch* batch, + void delete_with_prefix(rocksdb::WriteBatch* const batch, Rdb_key_def::DATA_DICT_TYPE dict_type, const GL_INDEX_ID &gl_index_id) const; /* Functions for fast DROP TABLE/INDEX */ - void resume_drop_indexes(); - void log_start_drop_table(const std::shared_ptr<Rdb_key_def>* key_descr, - uint32 n_keys, - const char* log_action); + void resume_drop_indexes() const; + void log_start_drop_table(const std::shared_ptr<Rdb_key_def>* const key_descr, + const uint32 &n_keys, + const char* const log_action) const; void log_start_drop_index(GL_INDEX_ID gl_index_id, - const char* log_action); + const char* log_action) const; public: - bool init(rocksdb::DB *rdb_dict, Rdb_cf_manager *cf_manager); + Rdb_dict_manager(const Rdb_dict_manager&) = delete; + Rdb_dict_manager& operator=(const Rdb_dict_manager&) = delete; + Rdb_dict_manager() = default; + + bool init(rocksdb::DB* const rdb_dict, Rdb_cf_manager* const cf_manager); inline void cleanup() { @@ -947,108 +991,111 @@ public: } /* Raw RocksDB operations */ - std::unique_ptr<rocksdb::WriteBatch> begin(); - int commit(rocksdb::WriteBatch *batch, bool sync = true); + std::unique_ptr<rocksdb::WriteBatch> begin() const; + int commit(rocksdb::WriteBatch* const batch, const bool &sync = true) const; rocksdb::Status get_value(const rocksdb::Slice& key, - std::string *value) const; - void put_key(rocksdb::WriteBatchBase *batch, const rocksdb::Slice &key, - const rocksdb::Slice &value); + std::string* const value) const; + void put_key(rocksdb::WriteBatchBase* const batch, const rocksdb::Slice &key, + const rocksdb::Slice &value) const; void delete_key(rocksdb::WriteBatchBase *batch, const rocksdb::Slice &key) const; - rocksdb::Iterator *new_iterator(); + rocksdb::Iterator *new_iterator() const; /* Internal Index id => CF */ void add_or_update_index_cf_mapping(rocksdb::WriteBatch *batch, const uchar index_type, const uint16_t kv_version, const uint index_id, - const uint cf_id); + const uint cf_id) const; void delete_index_info(rocksdb::WriteBatch* batch, const GL_INDEX_ID &index_id) const; bool get_index_info(const GL_INDEX_ID &gl_index_id, uint16_t *index_dict_version, - uchar *index_type, uint16_t *kv_version); + uchar *index_type, uint16_t *kv_version) const; /* CF id => CF flags */ - void add_cf_flags(rocksdb::WriteBatch *batch, - const uint cf_id, - const uint cf_flags); - bool get_cf_flags(const uint cf_id, uint *cf_flags); + void add_cf_flags(rocksdb::WriteBatch* const batch, + const uint &cf_id, + const uint &cf_flags) const; + bool get_cf_flags(const uint &cf_id, uint* const cf_flags) const; /* Functions for fast CREATE/DROP TABLE/INDEX */ void get_ongoing_index_operation(std::vector<GL_INDEX_ID>* gl_index_ids, - Rdb_key_def::DATA_DICT_TYPE dd_type); + Rdb_key_def::DATA_DICT_TYPE dd_type) const; bool is_index_operation_ongoing(const GL_INDEX_ID& gl_index_id, - Rdb_key_def::DATA_DICT_TYPE dd_type); + Rdb_key_def::DATA_DICT_TYPE dd_type) const; void start_ongoing_index_operation(rocksdb::WriteBatch* batch, const GL_INDEX_ID& gl_index_id, - Rdb_key_def::DATA_DICT_TYPE dd_type); - void end_ongoing_index_operation(rocksdb::WriteBatch* batch, + Rdb_key_def::DATA_DICT_TYPE dd_type) const; + void end_ongoing_index_operation(rocksdb::WriteBatch* const batch, const GL_INDEX_ID& gl_index_id, - Rdb_key_def::DATA_DICT_TYPE dd_type); - bool is_drop_index_empty(); - void add_drop_table(std::shared_ptr<Rdb_key_def>* key_descr, uint32 n_keys, - rocksdb::WriteBatch *batch); + Rdb_key_def::DATA_DICT_TYPE dd_type) const; + bool is_drop_index_empty() const; + void add_drop_table(std::shared_ptr<Rdb_key_def>* const key_descr, + const uint32 &n_keys, + rocksdb::WriteBatch* const batch) const; void add_drop_index(const std::unordered_set<GL_INDEX_ID>& gl_index_ids, - rocksdb::WriteBatch *batch); + rocksdb::WriteBatch* const batch) const; void add_create_index(const std::unordered_set<GL_INDEX_ID>& gl_index_ids, - rocksdb::WriteBatch *batch); + rocksdb::WriteBatch* const batch) const; void finish_indexes_operation( const std::unordered_set<GL_INDEX_ID>& gl_index_ids, - Rdb_key_def::DATA_DICT_TYPE dd_type); - void rollback_ongoing_index_creation(); + Rdb_key_def::DATA_DICT_TYPE dd_type) const; + void rollback_ongoing_index_creation() const; - inline void get_ongoing_drop_indexes(std::vector<GL_INDEX_ID>* gl_index_ids) + inline void + get_ongoing_drop_indexes(std::vector<GL_INDEX_ID>* gl_index_ids) const { get_ongoing_index_operation(gl_index_ids, Rdb_key_def::DDL_DROP_INDEX_ONGOING); } - inline void get_ongoing_create_indexes(std::vector<GL_INDEX_ID>* gl_index_ids) + inline void + get_ongoing_create_indexes(std::vector<GL_INDEX_ID>* gl_index_ids) const { get_ongoing_index_operation(gl_index_ids, Rdb_key_def::DDL_CREATE_INDEX_ONGOING); } inline void start_drop_index(rocksdb::WriteBatch *wb, - const GL_INDEX_ID& gl_index_id) + const GL_INDEX_ID& gl_index_id) const { start_ongoing_index_operation(wb, gl_index_id, Rdb_key_def::DDL_DROP_INDEX_ONGOING); } inline void start_create_index(rocksdb::WriteBatch *wb, - const GL_INDEX_ID& gl_index_id) + const GL_INDEX_ID& gl_index_id) const { start_ongoing_index_operation(wb, gl_index_id, Rdb_key_def::DDL_CREATE_INDEX_ONGOING); } inline void finish_drop_indexes( - const std::unordered_set<GL_INDEX_ID>& gl_index_ids) + const std::unordered_set<GL_INDEX_ID>& gl_index_ids) const { finish_indexes_operation(gl_index_ids, Rdb_key_def::DDL_DROP_INDEX_ONGOING); } inline void finish_create_indexes( - const std::unordered_set<GL_INDEX_ID>& gl_index_ids) + const std::unordered_set<GL_INDEX_ID>& gl_index_ids) const { finish_indexes_operation(gl_index_ids, Rdb_key_def::DDL_CREATE_INDEX_ONGOING); } - inline bool is_drop_index_ongoing(const GL_INDEX_ID& gl_index_id) + inline bool is_drop_index_ongoing(const GL_INDEX_ID& gl_index_id) const { return is_index_operation_ongoing(gl_index_id, Rdb_key_def::DDL_DROP_INDEX_ONGOING); } - inline bool is_create_index_ongoing(const GL_INDEX_ID& gl_index_id) + inline bool is_create_index_ongoing(const GL_INDEX_ID& gl_index_id) const { return is_index_operation_ongoing(gl_index_id, Rdb_key_def::DDL_CREATE_INDEX_ONGOING); } - bool get_max_index_id(uint32_t *index_id); - bool update_max_index_id(rocksdb::WriteBatch* batch, - const uint32_t index_id); - void add_stats(rocksdb::WriteBatch* batch, - const std::vector<Rdb_index_stats>& stats); - Rdb_index_stats get_stats(GL_INDEX_ID gl_index_id); + bool get_max_index_id(uint32_t* const index_id) const; + bool update_max_index_id(rocksdb::WriteBatch* const batch, + const uint32_t &index_id) const; + void add_stats(rocksdb::WriteBatch* const batch, + const std::vector<Rdb_index_stats>& stats) const; + Rdb_index_stats get_stats(GL_INDEX_ID gl_index_id) const; }; } // namespace myrocks diff --git a/storage/rocksdb/rdb_i_s.cc b/storage/rocksdb/rdb_i_s.cc index 78af6eff3a5..c35116e10a9 100644 --- a/storage/rocksdb/rdb_i_s.cc +++ b/storage/rocksdb/rdb_i_s.cc @@ -28,12 +28,14 @@ #include "rocksdb/memtablerep.h" #include "rocksdb/merge_operator.h" #include "rocksdb/slice_transform.h" +#include "rocksdb/utilities/transaction_db.h" /* MyRocks header files */ #include "./ha_rocksdb.h" #include "./ha_rocksdb_proto.h" #include "./rdb_cf_manager.h" #include "./rdb_datadic.h" +#include "./rdb_utils.h" namespace myrocks { @@ -50,17 +52,35 @@ namespace myrocks { /* Support for INFORMATION_SCHEMA.ROCKSDB_CFSTATS dynamic table */ +namespace RDB_CFSTATS_FIELD +{ + enum + { + CF_NAME= 0, + STAT_TYPE, + VALUE + }; +} // namespace RDB_CFSTATS_FIELD + +static ST_FIELD_INFO rdb_i_s_cfstats_fields_info[]= +{ + ROCKSDB_FIELD_INFO("CF_NAME", NAME_LEN+1, MYSQL_TYPE_STRING, 0), + ROCKSDB_FIELD_INFO("STAT_TYPE", NAME_LEN+1, MYSQL_TYPE_STRING, 0), + ROCKSDB_FIELD_INFO("VALUE", sizeof(uint64_t), MYSQL_TYPE_LONGLONG, 0), + ROCKSDB_FIELD_INFO_END +}; + static int rdb_i_s_cfstats_fill_table( - my_core::THD *thd, - my_core::TABLE_LIST *tables, - my_core::Item *cond __attribute__((__unused__))) + my_core::THD* const thd, + my_core::TABLE_LIST* const tables, + my_core::Item* const cond __attribute__((__unused__))) { bool ret; uint64_t val; DBUG_ENTER("rdb_i_s_cfstats_fill_table"); - std::vector<std::pair<const std::string, std::string>> cf_properties = { + const std::vector<std::pair<const std::string, std::string>> cf_properties = { {rocksdb::DB::Properties::kNumImmutableMemTable, "NUM_IMMUTABLE_MEM_TABLE"}, {rocksdb::DB::Properties::kMemTableFlushPending, "MEM_TABLE_FLUSH_PENDING"}, @@ -77,11 +97,11 @@ static int rdb_i_s_cfstats_fill_table( {rocksdb::DB::Properties::kNumLiveVersions, "NUM_LIVE_VERSIONS"} }; - rocksdb::DB *rdb= rdb_get_rocksdb_db(); - Rdb_cf_manager& cf_manager= rdb_get_cf_manager(); + rocksdb::DB* const rdb= rdb_get_rocksdb_db(); + const Rdb_cf_manager& cf_manager= rdb_get_cf_manager(); DBUG_ASSERT(rdb != nullptr); - for (auto cf_name : cf_manager.get_cf_names()) + for (const auto &cf_name : cf_manager.get_cf_names()) { rocksdb::ColumnFamilyHandle* cfh; bool is_automatic; @@ -94,19 +114,22 @@ static int rdb_i_s_cfstats_fill_table( if (cfh == nullptr) continue; - for (auto property : cf_properties) + for (const auto &property : cf_properties) { if (!rdb->GetIntProperty(cfh, property.first, &val)) continue; DBUG_ASSERT(tables != nullptr); - tables->table->field[0]->store(cf_name.c_str(), cf_name.size(), - system_charset_info); - tables->table->field[1]->store(property.second.c_str(), - property.second.size(), - system_charset_info); - tables->table->field[2]->store(val, true); + tables->table->field[RDB_CFSTATS_FIELD::CF_NAME]->store( + cf_name.c_str(), + cf_name.size(), + system_charset_info); + tables->table->field[RDB_CFSTATS_FIELD::STAT_TYPE]->store( + property.second.c_str(), + property.second.size(), + system_charset_info); + tables->table->field[RDB_CFSTATS_FIELD::VALUE]->store(val, true); ret= my_core::schema_table_store_record(thd, tables->table); @@ -117,14 +140,6 @@ static int rdb_i_s_cfstats_fill_table( DBUG_RETURN(0); } -static ST_FIELD_INFO rdb_i_s_cfstats_fields_info[]= -{ - ROCKSDB_FIELD_INFO("CF_NAME", NAME_LEN+1, MYSQL_TYPE_STRING, 0), - ROCKSDB_FIELD_INFO("STAT_TYPE", NAME_LEN+1, MYSQL_TYPE_STRING, 0), - ROCKSDB_FIELD_INFO("VALUE", sizeof(uint64_t), MYSQL_TYPE_LONGLONG, 0), - ROCKSDB_FIELD_INFO_END -}; - static int rdb_i_s_cfstats_init(void *p) { my_core::ST_SCHEMA_TABLE *schema; @@ -143,37 +158,54 @@ static int rdb_i_s_cfstats_init(void *p) /* Support for INFORMATION_SCHEMA.ROCKSDB_DBSTATS dynamic table */ +namespace RDB_DBSTATS_FIELD +{ + enum + { + STAT_TYPE= 0, + VALUE + }; +} // namespace RDB_DBSTATS_FIELD + +static ST_FIELD_INFO rdb_i_s_dbstats_fields_info[]= +{ + ROCKSDB_FIELD_INFO("STAT_TYPE", NAME_LEN+1, MYSQL_TYPE_STRING, 0), + ROCKSDB_FIELD_INFO("VALUE", sizeof(uint64_t), MYSQL_TYPE_LONGLONG, 0), + ROCKSDB_FIELD_INFO_END +}; + static int rdb_i_s_dbstats_fill_table( - my_core::THD *thd, - my_core::TABLE_LIST *tables, - my_core::Item *cond __attribute__((__unused__))) + my_core::THD* const thd, + my_core::TABLE_LIST* const tables, + my_core::Item* const cond __attribute__((__unused__))) { bool ret; uint64_t val; DBUG_ENTER("rdb_i_s_dbstats_fill_table"); - std::vector<std::pair<std::string, std::string>> db_properties = { + const std::vector<std::pair<std::string, std::string>> db_properties = { {rocksdb::DB::Properties::kBackgroundErrors, "DB_BACKGROUND_ERRORS"}, {rocksdb::DB::Properties::kNumSnapshots, "DB_NUM_SNAPSHOTS"}, {rocksdb::DB::Properties::kOldestSnapshotTime, "DB_OLDEST_SNAPSHOT_TIME"} }; - rocksdb::DB *rdb= rdb_get_rocksdb_db(); + rocksdb::DB* const rdb= rdb_get_rocksdb_db(); const rocksdb::BlockBasedTableOptions& table_options= rdb_get_table_options(); - for (auto property : db_properties) + for (const auto &property : db_properties) { if (!rdb->GetIntProperty(property.first, &val)) continue; DBUG_ASSERT(tables != nullptr); - tables->table->field[0]->store(property.second.c_str(), - property.second.size(), - system_charset_info); - tables->table->field[1]->store(val, true); + tables->table->field[RDB_DBSTATS_FIELD::STAT_TYPE]->store( + property.second.c_str(), + property.second.size(), + system_charset_info); + tables->table->field[RDB_DBSTATS_FIELD::VALUE]->store(val, true); ret= my_core::schema_table_store_record(thd, tables->table); @@ -192,23 +224,16 @@ static int rdb_i_s_dbstats_fill_table( information from the column family. */ val= (table_options.block_cache ? table_options.block_cache->GetUsage() : 0); - tables->table->field[0]->store(STRING_WITH_LEN("DB_BLOCK_CACHE_USAGE"), - system_charset_info); - tables->table->field[1]->store(val, true); + tables->table->field[RDB_DBSTATS_FIELD::STAT_TYPE]->store( + STRING_WITH_LEN("DB_BLOCK_CACHE_USAGE"), system_charset_info); + tables->table->field[RDB_DBSTATS_FIELD::VALUE]->store(val, true); ret= my_core::schema_table_store_record(thd, tables->table); DBUG_RETURN(ret); } -static ST_FIELD_INFO rdb_i_s_dbstats_fields_info[]= -{ - ROCKSDB_FIELD_INFO("STAT_TYPE", NAME_LEN+1, MYSQL_TYPE_STRING, 0), - ROCKSDB_FIELD_INFO("VALUE", sizeof(uint64_t), MYSQL_TYPE_LONGLONG, 0), - ROCKSDB_FIELD_INFO_END -}; - -static int rdb_i_s_dbstats_init(void *p) +static int rdb_i_s_dbstats_init(void* const p) { DBUG_ASSERT(p != nullptr); @@ -227,11 +252,34 @@ static int rdb_i_s_dbstats_init(void *p) /* Support for INFORMATION_SCHEMA.ROCKSDB_PERF_CONTEXT dynamic table */ +namespace RDB_PERF_CONTEXT_FIELD +{ + enum + { + TABLE_SCHEMA= 0, + TABLE_NAME, + PARTITION_NAME, + STAT_TYPE, + VALUE + }; +} // namespace RDB_PERF_CONTEXT_FIELD + +static ST_FIELD_INFO rdb_i_s_perf_context_fields_info[]= +{ + ROCKSDB_FIELD_INFO("TABLE_SCHEMA", NAME_LEN+1, MYSQL_TYPE_STRING, 0), + ROCKSDB_FIELD_INFO("TABLE_NAME", NAME_LEN+1, MYSQL_TYPE_STRING, 0), + ROCKSDB_FIELD_INFO("PARTITION_NAME", NAME_LEN+1, MYSQL_TYPE_STRING, + MY_I_S_MAYBE_NULL), + ROCKSDB_FIELD_INFO("STAT_TYPE", NAME_LEN+1, MYSQL_TYPE_STRING, 0), + ROCKSDB_FIELD_INFO("VALUE", sizeof(uint64_t), MYSQL_TYPE_LONGLONG, + 0), + ROCKSDB_FIELD_INFO_END +}; static int rdb_i_s_perf_context_fill_table( - my_core::THD *thd, - my_core::TABLE_LIST *tables, - my_core::Item *cond __attribute__((__unused__))) + my_core::THD* const thd, + my_core::TABLE_LIST* const tables, + my_core::Item* const cond __attribute__((__unused__))) { DBUG_ASSERT(thd != nullptr); DBUG_ASSERT(tables != nullptr); @@ -241,7 +289,7 @@ static int rdb_i_s_perf_context_fill_table( DBUG_ENTER("rdb_i_s_perf_context_fill_table"); - std::vector<std::string> tablenames= rdb_get_open_table_names(); + const std::vector<std::string> tablenames= rdb_get_open_table_names(); for (const auto& it : tablenames) { std::string str, dbname, tablename, partname; @@ -263,23 +311,28 @@ static int rdb_i_s_perf_context_fill_table( DBUG_ASSERT(field != nullptr); - field[0]->store(dbname.c_str(), dbname.size(), system_charset_info); - field[1]->store(tablename.c_str(), tablename.size(), system_charset_info); + field[RDB_PERF_CONTEXT_FIELD::TABLE_SCHEMA]->store( + dbname.c_str(), dbname.size(), system_charset_info); + field[RDB_PERF_CONTEXT_FIELD::TABLE_NAME]->store( + tablename.c_str(), tablename.size(), system_charset_info); if (partname.size() == 0) { - field[2]->set_null(); + field[RDB_PERF_CONTEXT_FIELD::PARTITION_NAME]->set_null(); } else { - field[2]->set_notnull(); - field[2]->store(partname.c_str(), partname.size(), system_charset_info); + field[RDB_PERF_CONTEXT_FIELD::PARTITION_NAME]->set_notnull(); + field[RDB_PERF_CONTEXT_FIELD::PARTITION_NAME]->store( + partname.c_str(), partname.size(), system_charset_info); } for (int i= 0; i < PC_MAX_IDX; i++) { - field[3]->store(rdb_pc_stat_types[i].c_str(), rdb_pc_stat_types[i].size(), - system_charset_info); - field[4]->store(counters.m_value[i], true); + field[RDB_PERF_CONTEXT_FIELD::STAT_TYPE]->store( + rdb_pc_stat_types[i].c_str(), + rdb_pc_stat_types[i].size(), + system_charset_info); + field[RDB_PERF_CONTEXT_FIELD::VALUE]->store(counters.m_value[i], true); ret= my_core::schema_table_store_record(thd, tables->table); if (ret) @@ -290,19 +343,7 @@ static int rdb_i_s_perf_context_fill_table( DBUG_RETURN(0); } -static ST_FIELD_INFO rdb_i_s_perf_context_fields_info[]= -{ - ROCKSDB_FIELD_INFO("TABLE_SCHEMA", NAME_LEN+1, MYSQL_TYPE_STRING, 0), - ROCKSDB_FIELD_INFO("TABLE_NAME", NAME_LEN+1, MYSQL_TYPE_STRING, 0), - ROCKSDB_FIELD_INFO("PARTITION_NAME", NAME_LEN+1, MYSQL_TYPE_STRING, - MY_I_S_MAYBE_NULL), - ROCKSDB_FIELD_INFO("STAT_TYPE", NAME_LEN+1, MYSQL_TYPE_STRING, 0), - ROCKSDB_FIELD_INFO("VALUE", sizeof(uint64_t), MYSQL_TYPE_LONGLONG, - 0), - ROCKSDB_FIELD_INFO_END -}; - -static int rdb_i_s_perf_context_init(void *p) +static int rdb_i_s_perf_context_init(void* const p) { DBUG_ASSERT(p != nullptr); @@ -318,10 +359,29 @@ static int rdb_i_s_perf_context_init(void *p) DBUG_RETURN(0); } +/* + Support for INFORMATION_SCHEMA.ROCKSDB_PERF_CONTEXT_GLOBAL dynamic table + */ +namespace RDB_PERF_CONTEXT_GLOBAL_FIELD +{ + enum + { + STAT_TYPE= 0, + VALUE + }; +} // namespace RDB_PERF_CONTEXT_GLOBAL_FIELD + +static ST_FIELD_INFO rdb_i_s_perf_context_global_fields_info[]= +{ + ROCKSDB_FIELD_INFO("STAT_TYPE", NAME_LEN+1, MYSQL_TYPE_STRING, 0), + ROCKSDB_FIELD_INFO("VALUE", sizeof(uint64_t), MYSQL_TYPE_LONGLONG, 0), + ROCKSDB_FIELD_INFO_END +}; + static int rdb_i_s_perf_context_global_fill_table( - my_core::THD *thd, - my_core::TABLE_LIST *tables, - my_core::Item *cond __attribute__((__unused__))) + my_core::THD* const thd, + my_core::TABLE_LIST* const tables, + my_core::Item* const cond __attribute__((__unused__))) { DBUG_ASSERT(thd != nullptr); DBUG_ASSERT(tables != nullptr); @@ -337,10 +397,12 @@ static int rdb_i_s_perf_context_global_fill_table( DBUG_ASSERT(tables->table != nullptr); DBUG_ASSERT(tables->table->field != nullptr); - tables->table->field[0]->store(rdb_pc_stat_types[i].c_str(), - rdb_pc_stat_types[i].size(), - system_charset_info); - tables->table->field[1]->store(global_counters.m_value[i], true); + tables->table->field[RDB_PERF_CONTEXT_GLOBAL_FIELD::STAT_TYPE]->store( + rdb_pc_stat_types[i].c_str(), + rdb_pc_stat_types[i].size(), + system_charset_info); + tables->table->field[RDB_PERF_CONTEXT_GLOBAL_FIELD::VALUE]->store( + global_counters.m_value[i], true); ret= my_core::schema_table_store_record(thd, tables->table); if (ret) @@ -350,14 +412,7 @@ static int rdb_i_s_perf_context_global_fill_table( DBUG_RETURN(0); } -static ST_FIELD_INFO rdb_i_s_perf_context_global_fields_info[]= -{ - ROCKSDB_FIELD_INFO("STAT_TYPE", NAME_LEN+1, MYSQL_TYPE_STRING, 0), - ROCKSDB_FIELD_INFO("VALUE", sizeof(uint64_t), MYSQL_TYPE_LONGLONG, 0), - ROCKSDB_FIELD_INFO_END -}; - -static int rdb_i_s_perf_context_global_init(void *p) +static int rdb_i_s_perf_context_global_init(void* const p) { DBUG_ASSERT(p != nullptr); @@ -376,10 +431,28 @@ static int rdb_i_s_perf_context_global_init(void *p) /* Support for INFORMATION_SCHEMA.ROCKSDB_CFOPTIONS dynamic table */ +namespace RDB_CFOPTIONS_FIELD +{ + enum + { + CF_NAME= 0, + OPTION_TYPE, + VALUE + }; +} // namespace RDB_CFOPTIONS_FIELD + +static ST_FIELD_INFO rdb_i_s_cfoptions_fields_info[] = +{ + ROCKSDB_FIELD_INFO("CF_NAME", NAME_LEN+1, MYSQL_TYPE_STRING, 0), + ROCKSDB_FIELD_INFO("OPTION_TYPE", NAME_LEN+1, MYSQL_TYPE_STRING, 0), + ROCKSDB_FIELD_INFO("VALUE", NAME_LEN+1, MYSQL_TYPE_STRING, 0), + ROCKSDB_FIELD_INFO_END +}; + static int rdb_i_s_cfoptions_fill_table( - my_core::THD *thd, - my_core::TABLE_LIST *tables, - my_core::Item *cond __attribute__((__unused__))) + my_core::THD* const thd, + my_core::TABLE_LIST* const tables, + my_core::Item* const cond __attribute__((__unused__))) { DBUG_ASSERT(thd != nullptr); DBUG_ASSERT(tables != nullptr); @@ -390,7 +463,7 @@ static int rdb_i_s_cfoptions_fill_table( Rdb_cf_manager& cf_manager= rdb_get_cf_manager(); - for (auto cf_name : cf_manager.get_cf_names()) + for (const auto &cf_name : cf_manager.get_cf_names()) { std::string val; rocksdb::ColumnFamilyOptions opts; @@ -460,7 +533,7 @@ static int rdb_i_s_cfoptions_fill_table( // get MAX_BYTES_FOR_LEVEL_MULTIPLIER_ADDITIONAL option value val = opts.max_bytes_for_level_multiplier_additional.empty() ? "NULL" : ""; - for (auto level : opts.max_bytes_for_level_multiplier_additional) + for (const auto &level : opts.max_bytes_for_level_multiplier_additional) { val.append(std::to_string(level) + ":"); } @@ -477,7 +550,7 @@ static int rdb_i_s_cfoptions_fill_table( // get COMPRESSION_PER_LEVEL option value val = opts.compression_per_level.empty() ? "NULL" : ""; - for (auto compression_type : opts.compression_per_level) + for (const auto &compression_type : opts.compression_per_level) { std::string res; GetStringFromCompressionType(&res, compression_type); @@ -523,7 +596,8 @@ static int rdb_i_s_cfoptions_fill_table( cf_option_types.push_back({"COMPACTION_STYLE", val}); // get COMPACTION_OPTIONS_UNIVERSAL related options - rocksdb::CompactionOptionsUniversal compac_opts = opts.compaction_options_universal; + const rocksdb::CompactionOptionsUniversal compac_opts = + opts.compaction_options_universal; val = "{SIZE_RATIO="; val.append(std::to_string(compac_opts.size_ratio)); val.append("; MIN_MERGE_WIDTH="); @@ -550,8 +624,7 @@ static int rdb_i_s_cfoptions_fill_table( std::to_string(opts.compaction_options_fifo.max_table_files_size)}); // get block-based table related options - const rocksdb::BlockBasedTableOptions& table_options= - rdb_get_table_options(); + const rocksdb::BlockBasedTableOptions& table_options= rdb_get_table_options(); // get BLOCK_BASED_TABLE_FACTORY::CACHE_INDEX_AND_FILTER_BLOCKS option cf_option_types.push_back( @@ -620,19 +693,21 @@ static int rdb_i_s_cfoptions_fill_table( cf_option_types.push_back({"BLOCK_BASED_TABLE_FACTORY::FORMAT_VERSION", std::to_string(table_options.format_version)}); - for (auto cf_option_type : cf_option_types) + for (const auto &cf_option_type : cf_option_types) { DBUG_ASSERT(tables->table != nullptr); DBUG_ASSERT(tables->table->field != nullptr); - tables->table->field[0]->store(cf_name.c_str(), cf_name.size(), - system_charset_info); - tables->table->field[1]->store(cf_option_type.first.c_str(), - cf_option_type.first.size(), - system_charset_info); - tables->table->field[2]->store(cf_option_type.second.c_str(), - cf_option_type.second.size(), - system_charset_info); + tables->table->field[RDB_CFOPTIONS_FIELD::CF_NAME]->store( + cf_name.c_str(), cf_name.size(), system_charset_info); + tables->table->field[RDB_CFOPTIONS_FIELD::OPTION_TYPE]->store( + cf_option_type.first.c_str(), + cf_option_type.first.size(), + system_charset_info); + tables->table->field[RDB_CFOPTIONS_FIELD::VALUE]->store( + cf_option_type.second.c_str(), + cf_option_type.second.size(), + system_charset_info); ret = my_core::schema_table_store_record(thd, tables->table); @@ -643,11 +718,24 @@ static int rdb_i_s_cfoptions_fill_table( DBUG_RETURN(0); } -static ST_FIELD_INFO rdb_i_s_cfoptions_fields_info[] = +/* + Support for INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO dynamic table + */ +namespace RDB_GLOBAL_INFO_FIELD { - ROCKSDB_FIELD_INFO("CF_NAME", NAME_LEN+1, MYSQL_TYPE_STRING, 0), - ROCKSDB_FIELD_INFO("OPTION_TYPE", NAME_LEN+1, MYSQL_TYPE_STRING, 0), - ROCKSDB_FIELD_INFO("VALUE", NAME_LEN+1, MYSQL_TYPE_STRING, 0), + enum + { + TYPE= 0, + NAME, + VALUE + }; +} + +static ST_FIELD_INFO rdb_i_s_global_info_fields_info[] = +{ + ROCKSDB_FIELD_INFO("TYPE", FN_REFLEN+1, MYSQL_TYPE_STRING, 0), + ROCKSDB_FIELD_INFO("NAME", FN_REFLEN+1, MYSQL_TYPE_STRING, 0), + ROCKSDB_FIELD_INFO("VALUE", FN_REFLEN+1, MYSQL_TYPE_STRING, 0), ROCKSDB_FIELD_INFO_END }; @@ -656,11 +744,11 @@ static ST_FIELD_INFO rdb_i_s_cfoptions_fields_info[] = * to insert (TYPE, KEY, VALUE) rows into * information_schema.rocksdb_global_info */ -static int rdb_global_info_fill_row(my_core::THD *thd, - my_core::TABLE_LIST *tables, - const char *type, - const char *name, - const char *value) +static int rdb_global_info_fill_row(my_core::THD* const thd, + my_core::TABLE_LIST* const tables, + const char* const type, + const char* const name, + const char* const value) { DBUG_ASSERT(thd != nullptr); DBUG_ASSERT(tables != nullptr); @@ -672,20 +760,20 @@ static int rdb_global_info_fill_row(my_core::THD *thd, Field **field= tables->table->field; DBUG_ASSERT(field != nullptr); - field[0]->store(type, strlen(type), system_charset_info); - field[1]->store(name, strlen(name), system_charset_info); - field[2]->store(value, strlen(value), system_charset_info); + field[RDB_GLOBAL_INFO_FIELD::TYPE]->store( + type, strlen(type), system_charset_info); + field[RDB_GLOBAL_INFO_FIELD::NAME]->store( + name, strlen(name), system_charset_info); + field[RDB_GLOBAL_INFO_FIELD::VALUE]->store( + value, strlen(value), system_charset_info); return my_core::schema_table_store_record(thd, tables->table); } -/* - Support for INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO dynamic table - */ static int rdb_i_s_global_info_fill_table( - my_core::THD *thd, - my_core::TABLE_LIST *tables, - my_core::Item *cond __attribute__((__unused__))) + my_core::THD* const thd, + my_core::TABLE_LIST* const tables, + my_core::Item* const cond __attribute__((__unused__))) { DBUG_ASSERT(thd != nullptr); DBUG_ASSERT(tables != nullptr); @@ -698,7 +786,7 @@ static int rdb_i_s_global_info_fill_table( int ret= 0; /* binlog info */ - Rdb_binlog_manager *blm= rdb_get_binlog_manager(); + Rdb_binlog_manager* const blm= rdb_get_binlog_manager(); DBUG_ASSERT(blm != nullptr); char file_buf[FN_REFLEN+1]= {0}; @@ -714,7 +802,7 @@ static int rdb_i_s_global_info_fill_table( } /* max index info */ - Rdb_dict_manager *dict_manager= rdb_get_dict_manager(); + const Rdb_dict_manager* const dict_manager= rdb_get_dict_manager(); DBUG_ASSERT(dict_manager != nullptr); uint32_t max_index_id; @@ -729,8 +817,8 @@ static int rdb_i_s_global_info_fill_table( /* cf_id -> cf_flags */ char cf_id_buf[INT_BUF_LEN]= {0}; char cf_value_buf[FN_REFLEN+1] = {0}; - Rdb_cf_manager& cf_manager= rdb_get_cf_manager(); - for (auto cf_handle : cf_manager.get_all_cf()) { + const Rdb_cf_manager& cf_manager= rdb_get_cf_manager(); + for (const auto &cf_handle : cf_manager.get_all_cf()) { uint flags; dict_manager->get_cf_flags(cf_handle->GetID(), &flags); snprintf(cf_id_buf, INT_BUF_LEN, "%u", cf_handle->GetID()); @@ -761,14 +849,6 @@ static int rdb_i_s_global_info_fill_table( DBUG_RETURN(ret); } -static ST_FIELD_INFO rdb_i_s_global_info_fields_info[] = -{ - ROCKSDB_FIELD_INFO("TYPE", FN_REFLEN+1, MYSQL_TYPE_STRING, 0), - ROCKSDB_FIELD_INFO("NAME", FN_REFLEN+1, MYSQL_TYPE_STRING, 0), - ROCKSDB_FIELD_INFO("VALUE", FN_REFLEN+1, MYSQL_TYPE_STRING, 0), - ROCKSDB_FIELD_INFO_END -}; - namespace // anonymous namespace = not visible outside this source file { @@ -781,6 +861,40 @@ struct Rdb_ddl_scanner : public Rdb_tables_scanner }; } // anonymous namespace +/* + Support for INFORMATION_SCHEMA.ROCKSDB_DDL dynamic table + */ +namespace RDB_DDL_FIELD +{ + enum + { + TABLE_SCHEMA= 0, + TABLE_NAME, + PARTITION_NAME, + INDEX_NAME, + COLUMN_FAMILY, + INDEX_NUMBER, + INDEX_TYPE, + KV_FORMAT_VERSION, + CF + }; +} // namespace RDB_DDL_FIELD + +static ST_FIELD_INFO rdb_i_s_ddl_fields_info[] = +{ + ROCKSDB_FIELD_INFO("TABLE_SCHEMA", NAME_LEN+1, MYSQL_TYPE_STRING, 0), + ROCKSDB_FIELD_INFO("TABLE_NAME", NAME_LEN+1, MYSQL_TYPE_STRING, 0), + ROCKSDB_FIELD_INFO("PARTITION_NAME", NAME_LEN+1, MYSQL_TYPE_STRING, + MY_I_S_MAYBE_NULL), + ROCKSDB_FIELD_INFO("INDEX_NAME", NAME_LEN+1, MYSQL_TYPE_STRING, 0), + ROCKSDB_FIELD_INFO("COLUMN_FAMILY", sizeof(uint32_t), MYSQL_TYPE_LONG, 0), + ROCKSDB_FIELD_INFO("INDEX_NUMBER", sizeof(uint32_t), MYSQL_TYPE_LONG, 0), + ROCKSDB_FIELD_INFO("INDEX_TYPE", sizeof(uint16_t), MYSQL_TYPE_SHORT, 0), + ROCKSDB_FIELD_INFO("KV_FORMAT_VERSION", sizeof(uint16_t), + MYSQL_TYPE_SHORT, 0), + ROCKSDB_FIELD_INFO("CF", NAME_LEN+1, MYSQL_TYPE_STRING, 0), + ROCKSDB_FIELD_INFO_END +}; int Rdb_ddl_scanner::add_table(Rdb_tbl_def *tdef) { @@ -793,37 +907,42 @@ int Rdb_ddl_scanner::add_table(Rdb_tbl_def *tdef) DBUG_ASSERT(field != nullptr); const std::string& dbname= tdef->base_dbname(); - field[0]->store(dbname.c_str(), dbname.size(), system_charset_info); + field[RDB_DDL_FIELD::TABLE_SCHEMA]->store( + dbname.c_str(), dbname.size(), system_charset_info); const std::string& tablename= tdef->base_tablename(); - field[1]->store(tablename.c_str(), tablename.size(), system_charset_info); + field[RDB_DDL_FIELD::TABLE_NAME]->store( + tablename.c_str(), tablename.size(), system_charset_info); const std::string& partname= tdef->base_partition(); if (partname.length() == 0) { - field[2]->set_null(); + field[RDB_DDL_FIELD::PARTITION_NAME]->set_null(); } else { - field[2]->set_notnull(); - field[2]->store(partname.c_str(), partname.size(), system_charset_info); + field[RDB_DDL_FIELD::PARTITION_NAME]->set_notnull(); + field[RDB_DDL_FIELD::PARTITION_NAME]->store( + partname.c_str(), partname.size(), system_charset_info); } for (uint i= 0; i < tdef->m_key_count; i++) { - const std::shared_ptr<const Rdb_key_def>& kd= tdef->m_key_descr_arr[i]; - DBUG_ASSERT(kd != nullptr); + const Rdb_key_def& kd= *tdef->m_key_descr_arr[i]; - field[3]->store(kd->m_name.c_str(), kd->m_name.size(), system_charset_info); + field[RDB_DDL_FIELD::INDEX_NAME]->store( + kd.m_name.c_str(), kd.m_name.size(), system_charset_info); - GL_INDEX_ID gl_index_id = kd->get_gl_index_id(); - field[4]->store(gl_index_id.cf_id, true); - field[5]->store(gl_index_id.index_id, true); - field[6]->store(kd->m_index_type, true); - field[7]->store(kd->m_kv_format_version, true); + GL_INDEX_ID gl_index_id = kd.get_gl_index_id(); + field[RDB_DDL_FIELD::COLUMN_FAMILY]->store(gl_index_id.cf_id, true); + field[RDB_DDL_FIELD::INDEX_NUMBER]->store(gl_index_id.index_id, true); + field[RDB_DDL_FIELD::INDEX_TYPE]->store(kd.m_index_type, true); + field[RDB_DDL_FIELD::KV_FORMAT_VERSION]->store( + kd.m_kv_format_version, true); - std::string cf_name= kd->get_cf()->GetName(); - field[8]->store(cf_name.c_str(), cf_name.size(), system_charset_info); + std::string cf_name= kd.get_cf()->GetName(); + field[RDB_DDL_FIELD::CF]->store( + cf_name.c_str(), cf_name.size(), system_charset_info); ret= my_core::schema_table_store_record(m_thd, m_table); if (ret) @@ -832,9 +951,9 @@ int Rdb_ddl_scanner::add_table(Rdb_tbl_def *tdef) return 0; } -static int rdb_i_s_ddl_fill_table(my_core::THD *thd, - my_core::TABLE_LIST *tables, - my_core::Item *cond) +static int rdb_i_s_ddl_fill_table(my_core::THD* const thd, + my_core::TABLE_LIST* const tables, + my_core::Item* const cond) { DBUG_ENTER("rdb_i_s_ddl_fill_table"); @@ -852,23 +971,7 @@ static int rdb_i_s_ddl_fill_table(my_core::THD *thd, DBUG_RETURN(ret); } -static ST_FIELD_INFO rdb_i_s_ddl_fields_info[] = -{ - ROCKSDB_FIELD_INFO("TABLE_SCHEMA", NAME_LEN+1, MYSQL_TYPE_STRING, 0), - ROCKSDB_FIELD_INFO("TABLE_NAME", NAME_LEN+1, MYSQL_TYPE_STRING, 0), - ROCKSDB_FIELD_INFO("PARTITION_NAME", NAME_LEN+1, MYSQL_TYPE_STRING, - MY_I_S_MAYBE_NULL), - ROCKSDB_FIELD_INFO("INDEX_NAME", NAME_LEN+1, MYSQL_TYPE_STRING, 0), - ROCKSDB_FIELD_INFO("COLUMN_FAMILY", sizeof(uint32_t), MYSQL_TYPE_LONG, 0), - ROCKSDB_FIELD_INFO("INDEX_NUMBER", sizeof(uint32_t), MYSQL_TYPE_LONG, 0), - ROCKSDB_FIELD_INFO("INDEX_TYPE", sizeof(uint16_t), MYSQL_TYPE_SHORT, 0), - ROCKSDB_FIELD_INFO("KV_FORMAT_VERSION", sizeof(uint16_t), - MYSQL_TYPE_SHORT, 0), - ROCKSDB_FIELD_INFO("CF", NAME_LEN+1, MYSQL_TYPE_STRING, 0), - ROCKSDB_FIELD_INFO_END -}; - -static int rdb_i_s_ddl_init(void *p) +static int rdb_i_s_ddl_init(void* const p) { my_core::ST_SCHEMA_TABLE *schema; @@ -883,7 +986,7 @@ static int rdb_i_s_ddl_init(void *p) DBUG_RETURN(0); } -static int rdb_i_s_cfoptions_init(void *p) +static int rdb_i_s_cfoptions_init(void* const p) { my_core::ST_SCHEMA_TABLE *schema; @@ -898,7 +1001,7 @@ static int rdb_i_s_cfoptions_init(void *p) DBUG_RETURN(0); } -static int rdb_i_s_global_info_init(void *p) +static int rdb_i_s_global_info_init(void* const p) { my_core::ST_SCHEMA_TABLE *schema; @@ -918,7 +1021,7 @@ static std::string rdb_filename_without_path( const std::string& path) { /* Find last slash in path */ - size_t pos = path.rfind('/'); + const size_t pos = path.rfind('/'); /* None found? Just return the original string */ if (pos == std::string::npos) { @@ -929,11 +1032,52 @@ static std::string rdb_filename_without_path( return path.substr(pos + 1); } +/* + Support for INFORMATION_SCHEMA.ROCKSDB_INDEX_FILE_MAP dynamic table + */ +namespace RDB_INDEX_FILE_MAP_FIELD +{ + enum + { + COLUMN_FAMILY= 0, + INDEX_NUMBER, + SST_NAME, + NUM_ROWS, + DATA_SIZE, + ENTRY_DELETES, + ENTRY_SINGLEDELETES, + ENTRY_MERGES, + ENTRY_OTHERS + }; +} // namespace RDB_INDEX_FILE_MAP_FIELD + +static ST_FIELD_INFO rdb_i_s_index_file_map_fields_info[] = +{ + /* The information_schema.rocksdb_index_file_map virtual table has four + * fields: + * COLUMN_FAMILY => the index's column family contained in the SST file + * INDEX_NUMBER => the index id contained in the SST file + * SST_NAME => the name of the SST file containing some indexes + * NUM_ROWS => the number of entries of this index id in this SST file + * DATA_SIZE => the data size stored in this SST file for this index id */ + ROCKSDB_FIELD_INFO("COLUMN_FAMILY", sizeof(uint32_t), MYSQL_TYPE_LONG, 0), + ROCKSDB_FIELD_INFO("INDEX_NUMBER", sizeof(uint32_t), MYSQL_TYPE_LONG, 0), + ROCKSDB_FIELD_INFO("SST_NAME", NAME_LEN+1, MYSQL_TYPE_STRING, 0), + ROCKSDB_FIELD_INFO("NUM_ROWS", sizeof(int64_t), MYSQL_TYPE_LONGLONG, 0), + ROCKSDB_FIELD_INFO("DATA_SIZE", sizeof(int64_t), MYSQL_TYPE_LONGLONG, 0), + ROCKSDB_FIELD_INFO("ENTRY_DELETES", sizeof(int64_t), MYSQL_TYPE_LONGLONG, 0), + ROCKSDB_FIELD_INFO("ENTRY_SINGLEDELETES", sizeof(int64_t), + MYSQL_TYPE_LONGLONG, 0), + ROCKSDB_FIELD_INFO("ENTRY_MERGES", sizeof(int64_t), MYSQL_TYPE_LONGLONG, 0), + ROCKSDB_FIELD_INFO("ENTRY_OTHERS", sizeof(int64_t), MYSQL_TYPE_LONGLONG, 0), + ROCKSDB_FIELD_INFO_END +}; + /* Fill the information_schema.rocksdb_index_file_map virtual table */ static int rdb_i_s_index_file_map_fill_table( - my_core::THD *thd, - my_core::TABLE_LIST *tables, - my_core::Item *cond __attribute__((__unused__))) + my_core::THD* const thd, + my_core::TABLE_LIST* const tables, + my_core::Item* const cond __attribute__((__unused__))) { DBUG_ASSERT(thd != nullptr); DBUG_ASSERT(tables != nullptr); @@ -946,14 +1090,14 @@ static int rdb_i_s_index_file_map_fill_table( DBUG_ENTER("rdb_i_s_index_file_map_fill_table"); /* Iterate over all the column families */ - rocksdb::DB *rdb= rdb_get_rocksdb_db(); + rocksdb::DB* const rdb= rdb_get_rocksdb_db(); DBUG_ASSERT(rdb != nullptr); - Rdb_cf_manager& cf_manager= rdb_get_cf_manager(); - for (auto cf_handle : cf_manager.get_all_cf()) { + const Rdb_cf_manager& cf_manager= rdb_get_cf_manager(); + for (const auto &cf_handle : cf_manager.get_all_cf()) { /* Grab the the properties of all the tables in the column family */ rocksdb::TablePropertiesCollection table_props_collection; - rocksdb::Status s = rdb->GetPropertiesOfAllTables(cf_handle, + const rocksdb::Status s = rdb->GetPropertiesOfAllTables(cf_handle, &table_props_collection); if (!s.ok()) { continue; @@ -961,35 +1105,43 @@ static int rdb_i_s_index_file_map_fill_table( /* Iterate over all the items in the collection, each of which contains a * name and the actual properties */ - for (auto props : table_props_collection) { + for (const auto &props : table_props_collection) { /* Add the SST name into the output */ - std::string sst_name = rdb_filename_without_path(props.first); - field[2]->store(sst_name.data(), sst_name.size(), system_charset_info); + const std::string sst_name = rdb_filename_without_path(props.first); + field[RDB_INDEX_FILE_MAP_FIELD::SST_NAME]->store( + sst_name.data(), sst_name.size(), system_charset_info); /* Get the __indexstats__ data out of the table property */ std::vector<Rdb_index_stats> stats; Rdb_tbl_prop_coll::read_stats_from_tbl_props(props.second, &stats); if (stats.empty()) { - field[0]->store(-1, true); - field[1]->store(-1, true); - field[3]->store(-1, true); - field[4]->store(-1, true); - field[5]->store(-1, true); - field[6]->store(-1, true); - field[7]->store(-1, true); - field[8]->store(-1, true); + field[RDB_INDEX_FILE_MAP_FIELD::COLUMN_FAMILY]->store(-1, true); + field[RDB_INDEX_FILE_MAP_FIELD::INDEX_NUMBER]->store(-1, true); + field[RDB_INDEX_FILE_MAP_FIELD::NUM_ROWS]->store(-1, true); + field[RDB_INDEX_FILE_MAP_FIELD::DATA_SIZE]->store(-1, true); + field[RDB_INDEX_FILE_MAP_FIELD::ENTRY_DELETES]->store(-1, true); + field[RDB_INDEX_FILE_MAP_FIELD::ENTRY_SINGLEDELETES]->store(-1, true); + field[RDB_INDEX_FILE_MAP_FIELD::ENTRY_MERGES]->store(-1, true); + field[RDB_INDEX_FILE_MAP_FIELD::ENTRY_OTHERS]->store(-1, true); } else { for (auto it : stats) { /* Add the index number, the number of rows, and data size to the output */ - field[0]->store(it.m_gl_index_id.cf_id, true); - field[1]->store(it.m_gl_index_id.index_id, true); - field[3]->store(it.m_rows, true); - field[4]->store(it.m_data_size, true); - field[5]->store(it.m_entry_deletes, true); - field[6]->store(it.m_entry_single_deletes, true); - field[7]->store(it.m_entry_merges, true); - field[8]->store(it.m_entry_others, true); + field[RDB_INDEX_FILE_MAP_FIELD::COLUMN_FAMILY]->store( + it.m_gl_index_id.cf_id, true); + field[RDB_INDEX_FILE_MAP_FIELD::INDEX_NUMBER]->store( + it.m_gl_index_id.index_id, true); + field[RDB_INDEX_FILE_MAP_FIELD::NUM_ROWS]->store(it.m_rows, true); + field[RDB_INDEX_FILE_MAP_FIELD::DATA_SIZE]->store( + it.m_data_size, true); + field[RDB_INDEX_FILE_MAP_FIELD::ENTRY_DELETES]->store( + it.m_entry_deletes, true); + field[RDB_INDEX_FILE_MAP_FIELD::ENTRY_SINGLEDELETES]->store( + it.m_entry_single_deletes, true); + field[RDB_INDEX_FILE_MAP_FIELD::ENTRY_MERGES]->store( + it.m_entry_merges, true); + field[RDB_INDEX_FILE_MAP_FIELD::ENTRY_OTHERS]->store( + it.m_entry_others, true); /* Tell MySQL about this row in the virtual table */ ret= my_core::schema_table_store_record(thd, tables->table); @@ -1004,29 +1156,8 @@ static int rdb_i_s_index_file_map_fill_table( DBUG_RETURN(ret); } -static ST_FIELD_INFO rdb_i_s_index_file_map_fields_info[] = -{ - /* The information_schema.rocksdb_index_file_map virtual table has four fields: - * COLUMN_FAMILY => the index's column family contained in the SST file - * INDEX_NUMBER => the index id contained in the SST file - * SST_NAME => the name of the SST file containing some indexes - * NUM_ROWS => the number of entries of this index id in this SST file - * DATA_SIZE => the data size stored in this SST file for this index id */ - ROCKSDB_FIELD_INFO("COLUMN_FAMILY", sizeof(uint32_t), MYSQL_TYPE_LONG, 0), - ROCKSDB_FIELD_INFO("INDEX_NUMBER", sizeof(uint32_t), MYSQL_TYPE_LONG, 0), - ROCKSDB_FIELD_INFO("SST_NAME", NAME_LEN+1, MYSQL_TYPE_STRING, 0), - ROCKSDB_FIELD_INFO("NUM_ROWS", sizeof(int64_t), MYSQL_TYPE_LONGLONG, 0), - ROCKSDB_FIELD_INFO("DATA_SIZE", sizeof(int64_t), MYSQL_TYPE_LONGLONG, 0), - ROCKSDB_FIELD_INFO("ENTRY_DELETES", sizeof(int64_t), MYSQL_TYPE_LONGLONG, 0), - ROCKSDB_FIELD_INFO("ENTRY_SINGLEDELETES", sizeof(int64_t), - MYSQL_TYPE_LONGLONG, 0), - ROCKSDB_FIELD_INFO("ENTRY_MERGES", sizeof(int64_t), MYSQL_TYPE_LONGLONG, 0), - ROCKSDB_FIELD_INFO("ENTRY_OTHERS", sizeof(int64_t), MYSQL_TYPE_LONGLONG, 0), - ROCKSDB_FIELD_INFO_END -}; - /* Initialize the information_schema.rocksdb_index_file_map virtual table */ -static int rdb_i_s_index_file_map_init(void *p) +static int rdb_i_s_index_file_map_init(void* const p) { my_core::ST_SCHEMA_TABLE *schema; @@ -1041,6 +1172,221 @@ static int rdb_i_s_index_file_map_init(void *p) DBUG_RETURN(0); } +/* + Support for INFORMATION_SCHEMA.ROCKSDB_LOCKS dynamic table + */ +namespace RDB_LOCKS_FIELD +{ + enum + { + COLUMN_FAMILY_ID= 0, + TRANSACTION_ID, + KEY, + MODE + }; +} // namespace RDB_LOCKS_FIELD + +static ST_FIELD_INFO rdb_i_s_lock_info_fields_info[] = +{ + ROCKSDB_FIELD_INFO("COLUMN_FAMILY_ID", sizeof(uint32_t), MYSQL_TYPE_LONG, 0), + ROCKSDB_FIELD_INFO("TRANSACTION_ID", sizeof(uint32_t), MYSQL_TYPE_LONG, 0), + ROCKSDB_FIELD_INFO("KEY", FN_REFLEN+1, MYSQL_TYPE_STRING, 0), + ROCKSDB_FIELD_INFO("MODE", 32, MYSQL_TYPE_STRING, 0), + ROCKSDB_FIELD_INFO_END +}; + +/* Fill the information_schema.rocksdb_locks virtual table */ +static int rdb_i_s_lock_info_fill_table( + my_core::THD* const thd, + my_core::TABLE_LIST* const tables, + my_core::Item* const cond __attribute__((__unused__))) +{ + DBUG_ASSERT(thd != nullptr); + DBUG_ASSERT(tables != nullptr); + DBUG_ASSERT(tables->table != nullptr); + + int ret = 0; + + DBUG_ENTER("rdb_i_s_lock_info_fill_table"); + + rocksdb::TransactionDB* const rdb= rdb_get_rocksdb_db(); + DBUG_ASSERT(rdb != nullptr); + + /* cf id -> rocksdb::KeyLockInfo */ + std::unordered_multimap<uint32_t, rocksdb::KeyLockInfo> lock_info = + rdb->GetLockStatusData(); + + for (const auto& lock : lock_info) { + const uint32_t cf_id = lock.first; + const auto& key_lock_info = lock.second; + const auto key_hexstr = rdb_hexdump(key_lock_info.key.c_str(), + key_lock_info.key.length(), FN_REFLEN); + + for (const auto &id : key_lock_info.ids) { + tables->table->field[RDB_LOCKS_FIELD::COLUMN_FAMILY_ID]->store( + cf_id, true); + tables->table->field[RDB_LOCKS_FIELD::TRANSACTION_ID]->store(id, true); + + tables->table->field[RDB_LOCKS_FIELD::KEY]->store( + key_hexstr.c_str(), key_hexstr.size(), + system_charset_info); + tables->table->field[RDB_LOCKS_FIELD::MODE]->store( + key_lock_info.exclusive ? "X" : "S", + 1, system_charset_info); + + /* Tell MySQL about this row in the virtual table */ + ret= my_core::schema_table_store_record(thd, tables->table); + if (ret != 0) { + break; + } + } + } + DBUG_RETURN(ret); +} + +/* Initialize the information_schema.rocksdb_lock_info virtual table */ +static int rdb_i_s_lock_info_init(void* const p) +{ + my_core::ST_SCHEMA_TABLE *schema; + + DBUG_ENTER("rdb_i_s_lock_info_init"); + DBUG_ASSERT(p != nullptr); + + schema= (my_core::ST_SCHEMA_TABLE*) p; + + schema->fields_info= rdb_i_s_lock_info_fields_info; + schema->fill_table= rdb_i_s_lock_info_fill_table; + + DBUG_RETURN(0); +} + +/* + Support for INFORMATION_SCHEMA.ROCKSDB_TRX dynamic table + */ +namespace RDB_TRX_FIELD +{ + enum + { + TRANSACTION_ID= 0, + STATE, + NAME, + WRITE_COUNT, + LOCK_COUNT, + TIMEOUT_SEC, + WAITING_KEY, + WAITING_COLUMN_FAMILY_ID, + IS_REPLICATION, + SKIP_TRX_API, + READ_ONLY, + HAS_DEADLOCK_DETECTION, + NUM_ONGOING_BULKLOAD, + THREAD_ID, + QUERY + }; +} // namespace RDB_TRX_FIELD + +static ST_FIELD_INFO rdb_i_s_trx_info_fields_info[] = +{ + ROCKSDB_FIELD_INFO("TRANSACTION_ID", sizeof(ulonglong), + MYSQL_TYPE_LONGLONG, 0), + ROCKSDB_FIELD_INFO("STATE", NAME_LEN+1, MYSQL_TYPE_STRING, 0), + ROCKSDB_FIELD_INFO("NAME", NAME_LEN+1, MYSQL_TYPE_STRING, 0), + ROCKSDB_FIELD_INFO("WRITE_COUNT", sizeof(ulonglong), MYSQL_TYPE_LONGLONG, 0), + ROCKSDB_FIELD_INFO("LOCK_COUNT", sizeof(ulonglong), MYSQL_TYPE_LONGLONG, 0), + ROCKSDB_FIELD_INFO("TIMEOUT_SEC", sizeof(uint32_t), MYSQL_TYPE_LONG, 0), + ROCKSDB_FIELD_INFO("WAITING_KEY", FN_REFLEN+1, MYSQL_TYPE_STRING, 0), + ROCKSDB_FIELD_INFO("WAITING_COLUMN_FAMILY_ID", sizeof(uint32_t), + MYSQL_TYPE_LONG, 0), + ROCKSDB_FIELD_INFO("IS_REPLICATION", sizeof(uint32_t), MYSQL_TYPE_LONG, 0), + ROCKSDB_FIELD_INFO("SKIP_TRX_API", sizeof(uint32_t), MYSQL_TYPE_LONG, 0), + ROCKSDB_FIELD_INFO("READ_ONLY", sizeof(uint32_t), MYSQL_TYPE_LONG, 0), + ROCKSDB_FIELD_INFO("HAS_DEADLOCK_DETECTION", sizeof(uint32_t), + MYSQL_TYPE_LONG, 0), + ROCKSDB_FIELD_INFO("NUM_ONGOING_BULKLOAD", sizeof(uint32_t), + MYSQL_TYPE_LONG, 0), + ROCKSDB_FIELD_INFO("THREAD_ID", sizeof(ulong), MYSQL_TYPE_LONG, 0), + ROCKSDB_FIELD_INFO("QUERY", NAME_LEN+1, MYSQL_TYPE_STRING, 0), + ROCKSDB_FIELD_INFO_END +}; + +/* Fill the information_schema.rocksdb_trx virtual table */ +static int rdb_i_s_trx_info_fill_table( + my_core::THD* const thd, + my_core::TABLE_LIST* const tables, + my_core::Item* const cond __attribute__((__unused__))) +{ + DBUG_ASSERT(thd != nullptr); + DBUG_ASSERT(tables != nullptr); + DBUG_ASSERT(tables->table != nullptr); + + int ret = 0; + + DBUG_ENTER("rdb_i_s_trx_info_fill_table"); + + const std::vector<Rdb_trx_info> &all_trx_info = rdb_get_all_trx_info(); + + for (const auto &info : all_trx_info) { + auto name_hexstr = rdb_hexdump(info.name.c_str(), info.name.length(), + NAME_LEN); + auto key_hexstr = rdb_hexdump(info.waiting_key.c_str(), + info.waiting_key.length(), FN_REFLEN); + tables->table->field[RDB_TRX_FIELD::TRANSACTION_ID]->store( + info.trx_id, true); + tables->table->field[RDB_TRX_FIELD::STATE]->store( + info.state.c_str(), info.state.length(), system_charset_info); + tables->table->field[RDB_TRX_FIELD::NAME]->store( + name_hexstr.c_str(), name_hexstr.length(), system_charset_info); + tables->table->field[RDB_TRX_FIELD::WRITE_COUNT]->store( + info.write_count, true); + tables->table->field[RDB_TRX_FIELD::LOCK_COUNT]->store( + info.lock_count, true); + tables->table->field[RDB_TRX_FIELD::TIMEOUT_SEC]->store( + info.timeout_sec, false); + tables->table->field[RDB_TRX_FIELD::WAITING_KEY]->store( + key_hexstr.c_str(), key_hexstr.length(), system_charset_info); + tables->table->field[RDB_TRX_FIELD::WAITING_COLUMN_FAMILY_ID]->store( + info.waiting_cf_id, true); + tables->table->field[RDB_TRX_FIELD::IS_REPLICATION]->store( + info.is_replication, false); + tables->table->field[RDB_TRX_FIELD::SKIP_TRX_API]->store( + info.skip_trx_api, false); + tables->table->field[RDB_TRX_FIELD::READ_ONLY]->store( + info.read_only, false); + tables->table->field[RDB_TRX_FIELD::HAS_DEADLOCK_DETECTION]->store( + info.deadlock_detect, false); + tables->table->field[RDB_TRX_FIELD::NUM_ONGOING_BULKLOAD]->store( + info.num_ongoing_bulk_load, false); + tables->table->field[RDB_TRX_FIELD::THREAD_ID]->store( + info.thread_id, true); + tables->table->field[RDB_TRX_FIELD::QUERY]->store( + info.query_str.c_str(), info.query_str.length(), system_charset_info); + + /* Tell MySQL about this row in the virtual table */ + ret= my_core::schema_table_store_record(thd, tables->table); + if (ret != 0) { + break; + } + } + + DBUG_RETURN(ret); +} + +/* Initialize the information_schema.rocksdb_trx_info virtual table */ +static int rdb_i_s_trx_info_init(void* const p) +{ + my_core::ST_SCHEMA_TABLE *schema; + + DBUG_ENTER("rdb_i_s_trx_info_init"); + DBUG_ASSERT(p != nullptr); + + schema= (my_core::ST_SCHEMA_TABLE*) p; + + schema->fields_info= rdb_i_s_trx_info_fields_info; + schema->fill_table= rdb_i_s_trx_info_fill_table; + + DBUG_RETURN(0); +} + static int rdb_i_s_deinit(void *p __attribute__((__unused__))) { DBUG_ENTER("rdb_i_s_deinit"); @@ -1186,4 +1532,37 @@ struct st_mysql_plugin rdb_i_s_index_file_map= 0, /* flags */ }; +struct st_mysql_plugin rdb_i_s_lock_info= +{ + MYSQL_INFORMATION_SCHEMA_PLUGIN, + &rdb_i_s_info, + "ROCKSDB_LOCKS", + "Facebook", + "RocksDB lock information", + PLUGIN_LICENSE_GPL, + rdb_i_s_lock_info_init, + nullptr, + 0x0001, /* version number (0.1) */ + nullptr, /* status variables */ + nullptr, /* system variables */ + nullptr, /* config options */ + 0, /* flags */ +}; + +struct st_mysql_plugin rdb_i_s_trx_info= +{ + MYSQL_INFORMATION_SCHEMA_PLUGIN, + &rdb_i_s_info, + "ROCKSDB_TRX", + "Facebook", + "RocksDB transaction information", + PLUGIN_LICENSE_GPL, + rdb_i_s_trx_info_init, + nullptr, + 0x0001, /* version number (0.1) */ + nullptr, /* status variables */ + nullptr, /* system variables */ + nullptr, /* config options */ + 0, /* flags */ +}; } // namespace myrocks diff --git a/storage/rocksdb/rdb_i_s.h b/storage/rocksdb/rdb_i_s.h index 846defab961..5384d237d3b 100644 --- a/storage/rocksdb/rdb_i_s.h +++ b/storage/rocksdb/rdb_i_s.h @@ -29,6 +29,7 @@ extern struct st_mysql_plugin rdb_i_s_cfoptions; extern struct st_mysql_plugin rdb_i_s_global_info; extern struct st_mysql_plugin rdb_i_s_ddl; extern struct st_mysql_plugin rdb_i_s_index_file_map; - +extern struct st_mysql_plugin rdb_i_s_lock_info; +extern struct st_mysql_plugin rdb_i_s_trx_info; } // namespace myrocks diff --git a/storage/rocksdb/rdb_index_merge.cc b/storage/rocksdb/rdb_index_merge.cc index dc85db4d356..8544bdf52fd 100644 --- a/storage/rocksdb/rdb_index_merge.cc +++ b/storage/rocksdb/rdb_index_merge.cc @@ -23,9 +23,11 @@ namespace myrocks { -Rdb_index_merge::Rdb_index_merge(const ulonglong merge_buf_size, - const ulonglong merge_combine_read_size, - const rocksdb::Comparator* comparator) : +Rdb_index_merge::Rdb_index_merge(const char* const tmpfile_path, + const ulonglong &merge_buf_size, + const ulonglong &merge_combine_read_size, + const rocksdb::Comparator* const comparator) : + m_tmpfile_path(tmpfile_path), m_merge_buf_size(merge_buf_size), m_merge_combine_read_size(merge_combine_read_size), m_comparator(comparator), @@ -64,13 +66,15 @@ int Rdb_index_merge::init() to disk. They will be written to disk sorted. A sorted tree is used to keep track of the offset of each record within the unsorted buffer. */ - m_rec_buf_unsorted= std::make_shared<merge_buf_info>(m_merge_buf_size); + m_rec_buf_unsorted= std::shared_ptr<merge_buf_info>( + new merge_buf_info(m_merge_buf_size)); /* Allocate output buffer that will contain sorted block that is written to disk. */ - m_output_buf= std::make_shared<merge_buf_info>(m_merge_buf_size); + m_output_buf= std::shared_ptr<merge_buf_info>( + new merge_buf_info(m_merge_buf_size)); return 0; } @@ -82,7 +86,16 @@ int Rdb_index_merge::merge_file_create() { DBUG_ASSERT(m_merge_file.fd == -1); - int fd = mysql_tmpfile("myrocks"); + int fd; + /* If no path set for tmpfile, use mysql_tmpdir by default */ + if (m_tmpfile_path == nullptr) + { + fd = mysql_tmpfile("myrocks"); + } + else + { + fd = mysql_tmpfile_path(m_tmpfile_path, "myrocks"); + } if (fd < 0) { @@ -112,11 +125,24 @@ int Rdb_index_merge::add(const rocksdb::Slice& key, Check if sort buffer is going to be out of space, if so write it out to disk in sorted order using offset tree. */ - uint total_offset= RDB_MERGE_CHUNK_LEN + m_rec_buf_unsorted->curr_offset + + const uint total_offset= + RDB_MERGE_CHUNK_LEN + m_rec_buf_unsorted->curr_offset + RDB_MERGE_KEY_DELIMITER + RDB_MERGE_VAL_DELIMITER + key.size() + val.size(); if (total_offset >= m_rec_buf_unsorted->total_size) { + /* + If the offset tree is empty here, that means that the proposed key to + add is too large for the buffer. + */ + if (m_offset_tree.empty()) + { + // NO_LINT_DEBUG + sql_print_error("Sort buffer size is too small to process merge. " + "Please set merge buffer size to a higher value."); + return HA_ERR_INTERNAL_ERROR; + } + if (merge_buf_write()) { // NO_LINT_DEBUG @@ -125,7 +151,7 @@ int Rdb_index_merge::add(const rocksdb::Slice& key, } } - ulonglong rec_offset= m_rec_buf_unsorted->curr_offset; + const ulonglong rec_offset= m_rec_buf_unsorted->curr_offset; /* Store key and value in temporary unsorted in memory buffer pointed to by @@ -159,7 +185,7 @@ int Rdb_index_merge::merge_buf_write() Iterate through the offset tree. Should be ordered by the secondary key at this point. */ - for (auto& rec : m_offset_tree) + for (const auto& rec : m_offset_tree) { DBUG_ASSERT(m_output_buf->curr_offset <= m_merge_buf_size); @@ -188,8 +214,14 @@ int Rdb_index_merge::merge_buf_write() return HA_ERR_INTERNAL_ERROR; } + /* + Add a file sync call here to flush the data out. Otherwise, the filesystem + cache can flush out all of the files at the same time, causing a write + burst. + */ if (my_write(m_merge_file.fd, m_output_buf->block.get(), - m_output_buf->total_size, MYF(MY_WME | MY_NABP))) + m_output_buf->total_size, MYF(MY_WME | MY_NABP)) || + mysql_file_sync(m_merge_file.fd, MYF(MY_WME))) { // NO_LINT_DEBUG sql_print_error("Error writing sorted merge buffer to disk."); @@ -238,13 +270,13 @@ int Rdb_index_merge::merge_heap_prepare() /* Allocate buffers for each chunk */ for (ulonglong i = 0; i < m_merge_file.num_sort_buffers; i++) { - auto entry= std::make_shared<merge_heap_entry>(m_comparator); + const auto entry= std::make_shared<merge_heap_entry>(m_comparator); /* Read chunk_size bytes from each chunk on disk, and place inside respective chunk buffer. */ - size_t total_size= + const size_t total_size= entry->prepare(m_merge_file.fd, i * m_merge_buf_size, chunk_size); if (total_size == (size_t) - 1) @@ -275,7 +307,7 @@ int Rdb_index_merge::merge_heap_prepare() /** Create and/or iterate through keys in the merge heap. */ -int Rdb_index_merge::next(rocksdb::Slice* key, rocksdb::Slice* val) +int Rdb_index_merge::next(rocksdb::Slice* const key, rocksdb::Slice* const val) { /* If table fits in one sort buffer, we can optimize by writing @@ -292,7 +324,7 @@ int Rdb_index_merge::next(rocksdb::Slice* key, rocksdb::Slice* val) return -1; } - auto rec= m_offset_tree.begin(); + const auto rec= m_offset_tree.begin(); /* Read record from offset */ merge_read_rec(rec->block, key, val); @@ -332,8 +364,8 @@ int Rdb_index_merge::next(rocksdb::Slice* key, rocksdb::Slice* val) /** Get current top record from the heap. */ -void Rdb_index_merge::merge_heap_top(rocksdb::Slice* key, - rocksdb::Slice* val) +void Rdb_index_merge::merge_heap_top(rocksdb::Slice* const key, + rocksdb::Slice* const val) { DBUG_ASSERT(!m_merge_min_heap.empty()); @@ -348,8 +380,8 @@ void Rdb_index_merge::merge_heap_top(rocksdb::Slice* key, Returns -1 when there are no more records in the heap. */ -int Rdb_index_merge::merge_heap_pop_and_get_next(rocksdb::Slice* key, - rocksdb::Slice* val) +int Rdb_index_merge::merge_heap_pop_and_get_next(rocksdb::Slice* const key, + rocksdb::Slice* const val) { /* Make a new reference to shared ptr so it doesn't get destroyed @@ -430,7 +462,7 @@ int Rdb_index_merge::merge_buf_info::read_next_chunk_from_disk(File fd) } /* Overwrite the old block */ - size_t bytes_read= my_read(fd, block.get(), block_len, MYF(MY_WME)); + const size_t bytes_read= my_read(fd, block.get(), block_len, MYF(MY_WME)); if (bytes_read == (size_t) -1) { // NO_LINT_DEBUG @@ -446,8 +478,8 @@ int Rdb_index_merge::merge_buf_info::read_next_chunk_from_disk(File fd) Get records from offset within sort buffer and compare them. Sort by least to greatest. */ -int Rdb_index_merge::merge_record_compare(const uchar* a_block, - const uchar* b_block, +int Rdb_index_merge::merge_record_compare(const uchar* const a_block, + const uchar* const b_block, const rocksdb::Comparator* const comparator) { return comparator->Compare(as_slice(a_block), as_slice(b_block)); @@ -457,9 +489,9 @@ int Rdb_index_merge::merge_record_compare(const uchar* a_block, Given an offset in a merge sort buffer, read out the keys + values. After this, block will point to the next record in the buffer. **/ -void Rdb_index_merge::merge_read_rec(const uchar* block, - rocksdb::Slice* key, - rocksdb::Slice* val) +void Rdb_index_merge::merge_read_rec(const uchar* const block, + rocksdb::Slice* const key, + rocksdb::Slice* const val) { /* Read key at block offset into key slice and the value into value slice*/ read_slice(key, block); @@ -474,13 +506,15 @@ void Rdb_index_merge::read_slice(rocksdb::Slice* slice, const uchar* block_ptr) *slice= rocksdb::Slice(reinterpret_cast<const char*>(block_ptr), slice_len); } -int Rdb_index_merge::merge_heap_entry::read_rec(rocksdb::Slice *key, - rocksdb::Slice *val) +int Rdb_index_merge::merge_heap_entry::read_rec(rocksdb::Slice* const key, + rocksdb::Slice* const val) { const uchar* block_ptr= block; + const auto orig_offset = chunk_info->curr_offset; + const auto orig_block = block; /* Read key at block offset into key slice and the value into value slice*/ - if (read_slice(key, &block_ptr) != 0 || read_slice(val, &block_ptr) != 0) + if (read_slice(key, &block_ptr) != 0) { return 1; } @@ -488,10 +522,20 @@ int Rdb_index_merge::merge_heap_entry::read_rec(rocksdb::Slice *key, chunk_info->curr_offset += (uintptr_t) block_ptr - (uintptr_t) block; block += (uintptr_t) block_ptr - (uintptr_t) block; + if (read_slice(val, &block_ptr) != 0) + { + chunk_info->curr_offset= orig_offset; + block= orig_block; + return 1; + } + + chunk_info->curr_offset += (uintptr_t) block_ptr - (uintptr_t) block; + block += (uintptr_t) block_ptr - (uintptr_t) block; + return 0; } -int Rdb_index_merge::merge_heap_entry::read_slice(rocksdb::Slice* slice, +int Rdb_index_merge::merge_heap_entry::read_slice(rocksdb::Slice* const slice, const uchar** block_ptr) { if (!chunk_info->has_space(RDB_MERGE_REC_DELIMITER)) @@ -515,7 +559,7 @@ size_t Rdb_index_merge::merge_heap_entry::prepare(File fd, ulonglong f_offset, ulonglong chunk_size) { chunk_info= std::make_shared<merge_buf_info>(chunk_size); - size_t res = chunk_info->prepare(fd, f_offset); + const size_t res = chunk_info->prepare(fd, f_offset); if (res != (size_t) - 1) { block= chunk_info->block.get() + RDB_MERGE_CHUNK_LEN; @@ -540,7 +584,7 @@ size_t Rdb_index_merge::merge_buf_info::prepare(File fd, ulonglong f_offset) return (size_t) - 1; } - size_t bytes_read= my_read(fd, block.get(), total_size, MYF(MY_WME)); + const size_t bytes_read= my_read(fd, block.get(), total_size, MYF(MY_WME)); if (bytes_read == (size_t) - 1) { // NO_LINT_DEBUG @@ -601,4 +645,3 @@ void Rdb_index_merge::merge_reset() } } // namespace myrocks - diff --git a/storage/rocksdb/rdb_index_merge.h b/storage/rocksdb/rdb_index_merge.h index 24090c335ac..86893bf316c 100644 --- a/storage/rocksdb/rdb_index_merge.h +++ b/storage/rocksdb/rdb_index_merge.h @@ -60,7 +60,7 @@ class Rdb_index_merge { struct merge_buf_info { /* heap memory allocated for main memory sort/merge */ std::unique_ptr<uchar[]> block; - ulonglong block_len; /* amount of data bytes allocated for block above */ + const ulonglong block_len; /* amount of data bytes allocated for block above */ ulonglong curr_offset; /* offset of the record pointer for the block */ ulonglong disk_start_offset; /* where the chunk starts on disk */ ulonglong disk_curr_offset; /* current offset on disk */ @@ -78,12 +78,12 @@ class Rdb_index_merge { int read_next_chunk_from_disk(File fd) __attribute__((__nonnull__, __warn_unused_result__)); - inline bool is_chunk_finished() + inline bool is_chunk_finished() const { return curr_offset + disk_curr_offset - disk_start_offset == total_size; } - inline bool has_space(uint64 needed) + inline bool has_space(uint64 needed) const { return curr_offset + needed <= block_len; } @@ -115,10 +115,10 @@ class Rdb_index_merge { int read_next_chunk_from_disk(File fd) __attribute__((__nonnull__, __warn_unused_result__)); - int read_rec(rocksdb::Slice *key, rocksdb::Slice *val) + int read_rec(rocksdb::Slice* const key, rocksdb::Slice* const val) __attribute__((__nonnull__, __warn_unused_result__)); - int read_slice(rocksdb::Slice* slice, const uchar** block_ptr) + int read_slice(rocksdb::Slice* const slice, const uchar** block_ptr) __attribute__((__nonnull__, __warn_unused_result__)); explicit merge_heap_entry(const rocksdb::Comparator* const comparator) : @@ -145,11 +145,13 @@ class Rdb_index_merge { return merge_record_compare(this->block, record.block, comparator) < 0; } - merge_record(uchar* block, const rocksdb::Comparator* const comparator) : + merge_record(uchar* const block, + const rocksdb::Comparator* const comparator) : block(block), comparator(comparator) {} }; private: + const char* m_tmpfile_path; const ulonglong m_merge_buf_size; const ulonglong m_merge_combine_read_size; const rocksdb::Comparator* m_comparator; @@ -161,12 +163,12 @@ class Rdb_index_merge { std::vector<std::shared_ptr<merge_heap_entry>>, merge_heap_comparator> m_merge_min_heap; - static inline void merge_store_uint64(uchar *dst, uint64 n) + static inline void merge_store_uint64(uchar* const dst, uint64 n) { memcpy(dst, &n, sizeof(n)); } - static inline void merge_read_uint64(const uchar **buf_ptr, uint64 *dst) + static inline void merge_read_uint64(const uchar **buf_ptr, uint64* const dst) { DBUG_ASSERT(buf_ptr != nullptr); memcpy(dst, *buf_ptr, sizeof(uint64)); @@ -185,17 +187,18 @@ class Rdb_index_merge { const rocksdb::Comparator* const comparator) __attribute__((__nonnull__, __warn_unused_result__)); - void merge_read_rec(const uchar* block, rocksdb::Slice* key, - rocksdb::Slice* val) + void merge_read_rec(const uchar* const block, rocksdb::Slice* const key, + rocksdb::Slice* const val) __attribute__((__nonnull__)); void read_slice(rocksdb::Slice* slice, const uchar* block_ptr) __attribute__((__nonnull__)); public: - Rdb_index_merge(const ulonglong merge_buf_size, - const ulonglong merge_combine_read_size, - const rocksdb::Comparator* comparator); + Rdb_index_merge(const char* const tmpfile_path, + const ulonglong &merge_buf_size, + const ulonglong &merge_combine_read_size, + const rocksdb::Comparator* const comparator); ~Rdb_index_merge(); int init() @@ -210,7 +213,7 @@ class Rdb_index_merge { int merge_buf_write() __attribute__((__nonnull__, __warn_unused_result__)); - int next(rocksdb::Slice* key, rocksdb::Slice* val) + int next(rocksdb::Slice* const key, rocksdb::Slice* const val) __attribute__((__nonnull__, __warn_unused_result__)); int merge_heap_prepare() @@ -219,11 +222,11 @@ class Rdb_index_merge { void merge_heap_top(rocksdb::Slice* key, rocksdb::Slice* val) __attribute__((__nonnull__)); - int merge_heap_pop_and_get_next(rocksdb::Slice* key, rocksdb::Slice* val) + int merge_heap_pop_and_get_next(rocksdb::Slice* const key, + rocksdb::Slice* const val) __attribute__((__nonnull__, __warn_unused_result__)); void merge_reset(); }; } // namespace myrocks - diff --git a/storage/rocksdb/rdb_mutex_wrapper.cc b/storage/rocksdb/rdb_mutex_wrapper.cc index e8077e2fd89..5b1c9ba4c22 100644 --- a/storage/rocksdb/rdb_mutex_wrapper.cc +++ b/storage/rocksdb/rdb_mutex_wrapper.cc @@ -39,7 +39,7 @@ PSI_stage_info stage_waiting_on_row_lock2= { 0, "Waiting for row lock", 0}; static const int64_t MICROSECS= 1000*1000; // A timeout as long as one full non-leap year worth of microseconds is as // good as infinite timeout. -static const int64_t BIG_TIMEOUT= MICROSECS * 60 * 60 * 24 * 7 * 365; +static const int64_t BIG_TIMEOUT= MICROSECS * 60 * 60 * 24 * 365; Rdb_cond_var::Rdb_cond_var() { mysql_cond_init(0, &m_cond, nullptr); @@ -49,7 +49,7 @@ Rdb_cond_var::~Rdb_cond_var() { mysql_cond_destroy(&m_cond); } -Status Rdb_cond_var::Wait(std::shared_ptr<TransactionDBMutex> mutex_arg) { +Status Rdb_cond_var::Wait(const std::shared_ptr<TransactionDBMutex> mutex_arg) { return WaitFor(mutex_arg, BIG_TIMEOUT); } @@ -69,7 +69,7 @@ Status Rdb_cond_var::Wait(std::shared_ptr<TransactionDBMutex> mutex_arg) { */ Status -Rdb_cond_var::WaitFor(std::shared_ptr<TransactionDBMutex> mutex_arg, +Rdb_cond_var::WaitFor(const std::shared_ptr<TransactionDBMutex> mutex_arg, int64_t timeout_micros) { auto *mutex_obj= reinterpret_cast<Rdb_mutex*>(mutex_arg.get()); @@ -204,7 +204,7 @@ Status Rdb_mutex::TryLockFor(int64_t timeout_time __attribute__((__unused__))) #ifndef STANDALONE_UNITTEST -void Rdb_mutex::set_unlock_action(PSI_stage_info *old_stage_arg) +void Rdb_mutex::set_unlock_action(const PSI_stage_info* const old_stage_arg) { DBUG_ASSERT(old_stage_arg != nullptr); @@ -221,7 +221,8 @@ void Rdb_mutex::UnLock() { #ifndef STANDALONE_UNITTEST if (m_old_stage_info.count(current_thd) > 0) { - std::shared_ptr<PSI_stage_info> old_stage = m_old_stage_info[current_thd]; + const std::shared_ptr<PSI_stage_info> old_stage = + m_old_stage_info[current_thd]; m_old_stage_info.erase(current_thd); /* The following will call mysql_mutex_unlock */ my_core::thd_exit_cond(current_thd, old_stage.get()); diff --git a/storage/rocksdb/rdb_mutex_wrapper.h b/storage/rocksdb/rdb_mutex_wrapper.h index 7d0e4169ade..6edd78a1167 100644 --- a/storage/rocksdb/rdb_mutex_wrapper.h +++ b/storage/rocksdb/rdb_mutex_wrapper.h @@ -64,13 +64,15 @@ class Rdb_mutex: public rocksdb::TransactionDBMutex { friend class Rdb_cond_var; #ifndef STANDALONE_UNITTEST - void set_unlock_action(PSI_stage_info *old_stage_arg); + void set_unlock_action(const PSI_stage_info* const old_stage_arg); std::unordered_map<THD*, std::shared_ptr<PSI_stage_info>> m_old_stage_info; #endif }; class Rdb_cond_var: public rocksdb::TransactionDBCondVar { + Rdb_cond_var(const Rdb_cond_var&) = delete; + Rdb_cond_var& operator=(const Rdb_cond_var&) = delete; public: Rdb_cond_var(); virtual ~Rdb_cond_var(); @@ -85,7 +87,7 @@ class Rdb_cond_var: public rocksdb::TransactionDBCondVar { // Returns non-OK if TransactionDB should stop waiting and fail the operation. // May return OK spuriously even if not notified. virtual rocksdb::Status - Wait(std::shared_ptr<rocksdb::TransactionDBMutex> mutex) override; + Wait(const std::shared_ptr<rocksdb::TransactionDBMutex> mutex) override; // Block current thread until condition variable is notifiesd by a call to // Notify() or NotifyAll(), or if the timeout is reached. @@ -100,7 +102,7 @@ class Rdb_cond_var: public rocksdb::TransactionDBCondVar { // fail the operation. // May return OK spuriously even if not notified. virtual rocksdb::Status - WaitFor(std::shared_ptr<rocksdb::TransactionDBMutex> mutex, + WaitFor(const std::shared_ptr<rocksdb::TransactionDBMutex> mutex, int64_t timeout_time) override; // If any threads are waiting on *this, unblock at least one of the @@ -117,6 +119,9 @@ class Rdb_cond_var: public rocksdb::TransactionDBCondVar { class Rdb_mutex_factory : public rocksdb::TransactionDBMutexFactory { public: + Rdb_mutex_factory(const Rdb_mutex_factory&) = delete; + Rdb_mutex_factory& operator=(const Rdb_mutex_factory&) = delete; + Rdb_mutex_factory() {} /* Override parent class's virtual methods of interrest. */ diff --git a/storage/rocksdb/rdb_perf_context.cc b/storage/rocksdb/rdb_perf_context.cc index cd0d9e57c2b..88d84061789 100644 --- a/storage/rocksdb/rdb_perf_context.cc +++ b/storage/rocksdb/rdb_perf_context.cc @@ -96,7 +96,7 @@ std::string rdb_pc_stat_types[]= idx++; \ } while (0) -static void harvest_diffs(Rdb_atomic_perf_counters *counters) +static void harvest_diffs(Rdb_atomic_perf_counters * const counters) { // (C) These should be in the same order as the PC enum size_t idx= 0; @@ -151,7 +151,7 @@ static void harvest_diffs(Rdb_atomic_perf_counters *counters) static Rdb_atomic_perf_counters rdb_global_perf_counters; -void rdb_get_global_perf_counters(Rdb_perf_counters *counters) +void rdb_get_global_perf_counters(Rdb_perf_counters* const counters) { DBUG_ASSERT(counters != nullptr); @@ -165,9 +165,9 @@ void Rdb_perf_counters::load(const Rdb_atomic_perf_counters &atomic_counters) } } -bool Rdb_io_perf::start(uint32_t perf_context_level) +bool Rdb_io_perf::start(const uint32_t perf_context_level) { - rocksdb::PerfLevel perf_level= + const rocksdb::PerfLevel perf_level= static_cast<rocksdb::PerfLevel>(perf_context_level); if (rocksdb::GetPerfLevel() != perf_level) @@ -185,9 +185,9 @@ bool Rdb_io_perf::start(uint32_t perf_context_level) return true; } -void Rdb_io_perf::end_and_record(uint32_t perf_context_level) +void Rdb_io_perf::end_and_record(const uint32_t perf_context_level) { - rocksdb::PerfLevel perf_level= + const rocksdb::PerfLevel perf_level= static_cast<rocksdb::PerfLevel>(perf_context_level); if (perf_level == rocksdb::kDisable) @@ -208,7 +208,7 @@ void Rdb_io_perf::end_and_record(uint32_t perf_context_level) { my_io_perf_t io_perf_read; - my_io_perf_init(&io_perf_read); + io_perf_read.init(); io_perf_read.bytes= rocksdb::perf_context.block_read_byte; io_perf_read.requests= rocksdb::perf_context.block_read_count; @@ -219,8 +219,8 @@ void Rdb_io_perf::end_and_record(uint32_t perf_context_level) io_perf_read.svc_time_max= io_perf_read.svc_time= rocksdb::perf_context.block_read_time; - my_io_perf_sum_atomic_helper(m_shared_io_perf_read, &io_perf_read); - my_io_perf_sum(&m_stats->table_io_perf_read, &io_perf_read); + m_shared_io_perf_read->sum(io_perf_read); + m_stats->table_io_perf_read.sum(io_perf_read); } if (m_stats) { diff --git a/storage/rocksdb/rdb_perf_context.h b/storage/rocksdb/rdb_perf_context.h index 1e01e933895..e6439c2e613 100644 --- a/storage/rocksdb/rdb_perf_context.h +++ b/storage/rocksdb/rdb_perf_context.h @@ -91,7 +91,10 @@ struct Rdb_atomic_perf_counters */ class Rdb_perf_counters { + Rdb_perf_counters(const Rdb_perf_counters&) = delete; + Rdb_perf_counters& operator=(const Rdb_perf_counters&) = delete; public: + Rdb_perf_counters() = default; uint64_t m_value[PC_MAX_IDX]; void load(const Rdb_atomic_perf_counters &atomic_counters); @@ -110,9 +113,12 @@ class Rdb_io_perf ha_statistics *m_stats= nullptr; public: - void init(Rdb_atomic_perf_counters *atomic_counters, - my_io_perf_atomic_t *shared_io_perf_read, - ha_statistics *stats) + Rdb_io_perf(const Rdb_io_perf&) = delete; + Rdb_io_perf& operator=(const Rdb_io_perf&) = delete; + + void init(Rdb_atomic_perf_counters* const atomic_counters, + my_io_perf_atomic_t* const shared_io_perf_read, + ha_statistics* const stats) { DBUG_ASSERT(atomic_counters != nullptr); DBUG_ASSERT(shared_io_perf_read != nullptr); @@ -123,8 +129,8 @@ class Rdb_io_perf m_stats= stats; } - bool start(uint32_t perf_context_level); - void end_and_record(uint32_t perf_context_level); + bool start(const uint32_t perf_context_level); + void end_and_record(const uint32_t perf_context_level); explicit Rdb_io_perf() : m_atomic_counters(nullptr), m_shared_io_perf_read(nullptr), diff --git a/storage/rocksdb/rdb_sst_info.cc b/storage/rocksdb/rdb_sst_info.cc index d131545e476..ce457cc73a7 100644 --- a/storage/rocksdb/rdb_sst_info.cc +++ b/storage/rocksdb/rdb_sst_info.cc @@ -28,6 +28,7 @@ /* RocksDB header files */ #include "rocksdb/db.h" +#include "rocksdb/options.h" /* MyRocks header files */ #include "./ha_rocksdb.h" @@ -36,14 +37,16 @@ namespace myrocks { -Rdb_sst_file::Rdb_sst_file(rocksdb::DB* db, rocksdb::ColumnFamilyHandle* cf, +Rdb_sst_file::Rdb_sst_file(rocksdb::DB* const db, + rocksdb::ColumnFamilyHandle* const cf, const rocksdb::DBOptions& db_options, - const std::string& name) : + const std::string& name, const bool tracing) : m_db(db), m_cf(cf), m_db_options(db_options), m_sst_file_writer(nullptr), - m_name(name) + m_name(name), + m_tracing(tracing) { DBUG_ASSERT(db != nullptr); DBUG_ASSERT(cf != nullptr); @@ -76,13 +79,20 @@ rocksdb::Status Rdb_sst_file::open() // Create an sst file writer with the current options and comparator const rocksdb::Comparator* comparator= m_cf->GetComparator(); - rocksdb::EnvOptions env_options(m_db_options); - rocksdb::Options options(m_db_options, cf_descr.options); + const rocksdb::EnvOptions env_options(m_db_options); + const rocksdb::Options options(m_db_options, cf_descr.options); m_sst_file_writer= - new rocksdb::SstFileWriter(env_options, options, comparator); + new rocksdb::SstFileWriter(env_options, options, comparator, m_cf); s= m_sst_file_writer->Open(m_name); + if (m_tracing) + { + // NO_LINT_DEBUG + sql_print_information("SST Tracing: Open(%s) returned %s", m_name.c_str(), + s.ok() ? "ok" : "not ok"); + } + if (!s.ok()) { delete m_sst_file_writer; @@ -101,22 +111,73 @@ rocksdb::Status Rdb_sst_file::put(const rocksdb::Slice& key, return m_sst_file_writer->Add(key, value); } +std::string Rdb_sst_file::generateKey(const std::string& key) +{ + static char const hexdigit[]= { + '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' + }; + + std::string res; + + res.reserve(key.size() * 2); + + for (auto ch : key) + { + res += hexdigit[((uint8_t) ch) >> 4]; + res += hexdigit[((uint8_t) ch) & 0x0F]; + } + + return res; +} + // This function is run by the background thread rocksdb::Status Rdb_sst_file::commit() { DBUG_ASSERT(m_sst_file_writer != nullptr); rocksdb::Status s; + rocksdb::ExternalSstFileInfo fileinfo; ///Finish may should be modified // Close out the sst file - s= m_sst_file_writer->Finish(); + s= m_sst_file_writer->Finish(&fileinfo); + if (m_tracing) + { + // NO_LINT_DEBUG + sql_print_information("SST Tracing: Finish returned %s", + s.ok() ? "ok" : "not ok"); + } + if (s.ok()) { - std::vector<std::string> files = { m_name }; + if (m_tracing) + { + // NO_LINT_DEBUG + sql_print_information("SST Tracing: Adding file %s, smallest key: %s, " + "largest key: %s, file size: %" PRIu64 ", " + "num_entries: %" PRIu64, fileinfo.file_path.c_str(), + generateKey(fileinfo.smallest_key).c_str(), + generateKey(fileinfo.largest_key).c_str(), + fileinfo.file_size, fileinfo.num_entries); + } + // Add the file to the database - // Set the skip_snapshot_check parameter to true since no one + // Set the snapshot_consistency parameter to false since no one // should be accessing the table we are bulk loading - s= m_db->AddFile(m_cf, files, true, true); + rocksdb::IngestExternalFileOptions opts; + opts.move_files = true; + opts.snapshot_consistency = false; + opts.allow_global_seqno = false; + opts.allow_blocking_flush = false; + s= m_db->IngestExternalFile(m_cf, { m_name }, opts); + + if (m_tracing) + { + // NO_LINT_DEBUG + sql_print_information("SST Tracing: AddFile(%s) returned %s", + fileinfo.file_path.c_str(), + s.ok() ? "ok" : "not ok"); + } } delete m_sst_file_writer; @@ -125,10 +186,11 @@ rocksdb::Status Rdb_sst_file::commit() return s; } -Rdb_sst_info::Rdb_sst_info(rocksdb::DB* db, const std::string& tablename, +Rdb_sst_info::Rdb_sst_info(rocksdb::DB* const db, const std::string& tablename, const std::string& indexname, - rocksdb::ColumnFamilyHandle* cf, - const rocksdb::DBOptions& db_options) : + rocksdb::ColumnFamilyHandle* const cf, + const rocksdb::DBOptions& db_options, + const bool& tracing) : m_db(db), m_cf(cf), m_db_options(db_options), @@ -142,7 +204,8 @@ Rdb_sst_info::Rdb_sst_info(rocksdb::DB* db, const std::string& tablename, m_thread(nullptr), m_finished(false), #endif - m_sst_file(nullptr) + m_sst_file(nullptr), + m_tracing(tracing) { m_prefix= db->GetName() + "/"; @@ -162,7 +225,7 @@ Rdb_sst_info::Rdb_sst_info(rocksdb::DB* db, const std::string& tablename, } rocksdb::ColumnFamilyDescriptor cf_descr; - rocksdb::Status s= m_cf->GetDescriptor(&cf_descr); + const rocksdb::Status s= m_cf->GetDescriptor(&cf_descr); if (!s.ok()) { // Default size if we can't get the cf's target size @@ -188,13 +251,13 @@ int Rdb_sst_info::open_new_sst_file() DBUG_ASSERT(m_sst_file == nullptr); // Create the new sst file's name - std::string name= m_prefix + std::to_string(m_sst_count++) + m_suffix; + const std::string name= m_prefix + std::to_string(m_sst_count++) + m_suffix; // Create the new sst file object - m_sst_file= new Rdb_sst_file(m_db, m_cf, m_db_options, name); + m_sst_file= new Rdb_sst_file(m_db, m_cf, m_db_options, name, m_tracing); // Open the sst file - rocksdb::Status s= m_sst_file->open(); + const rocksdb::Status s= m_sst_file->open(); if (!s.ok()) { set_error_msg(s.ToString()); @@ -224,14 +287,14 @@ void Rdb_sst_info::close_curr_sst_file() { // Add this finished sst file to the queue (while holding mutex) - std::lock_guard<std::mutex> guard(m_mutex); + const std::lock_guard<std::mutex> guard(m_mutex); m_queue.push(m_sst_file); } // Notify the background thread that there is a new entry in the queue m_cond.notify_one(); #else - rocksdb::Status s= m_sst_file->commit(); + const rocksdb::Status s= m_sst_file->commit(); if (!s.ok()) { set_error_msg(s.ToString()); @@ -276,7 +339,7 @@ int Rdb_sst_info::put(const rocksdb::Slice& key, DBUG_ASSERT(m_sst_file != nullptr); // Add the key/value to the current sst file - rocksdb::Status s= m_sst_file->put(key, value); + const rocksdb::Status s= m_sst_file->put(key, value); if (!s.ok()) { set_error_msg(s.ToString()); @@ -325,7 +388,7 @@ void Rdb_sst_info::set_error_msg(const std::string& msg) // Both the foreground and background threads can set the error message // so lock the mutex to protect it. We only want the first error that // we encounter. - std::lock_guard<std::mutex> guard(m_mutex); + const std::lock_guard<std::mutex> guard(m_mutex); #endif my_printf_error(ER_UNKNOWN_ERROR, "bulk load error: %s", MYF(0), msg.c_str()); if (m_error_msg.empty()) @@ -343,7 +406,7 @@ void Rdb_sst_info::thread_fcn(void* object) void Rdb_sst_info::run_thread() { - std::unique_lock<std::mutex> lk(m_mutex); + const std::unique_lock<std::mutex> lk(m_mutex); do { @@ -353,14 +416,14 @@ void Rdb_sst_info::run_thread() // Inner loop pulls off all Rdb_sst_file entries and processes them while (!m_queue.empty()) { - Rdb_sst_file* sst_file= m_queue.front(); + const Rdb_sst_file* const sst_file= m_queue.front(); m_queue.pop(); // Release the lock - we don't want to hold it while committing the file lk.unlock(); // Close out the sst file and add it to the database - rocksdb::Status s= sst_file->commit(); + const rocksdb::Status s= sst_file->commit(); if (!s.ok()) { set_error_msg(s.ToString()); @@ -380,10 +443,10 @@ void Rdb_sst_info::run_thread() } #endif -void Rdb_sst_info::init(rocksdb::DB* db) +void Rdb_sst_info::init(const rocksdb::DB* const db) { - std::string path= db->GetName() + FN_DIRSEP; - struct st_my_dir* dir_info= my_dir(path.c_str(), MYF(MY_DONT_SORT)); + const std::string path= db->GetName() + FN_DIRSEP; + struct st_my_dir* const dir_info= my_dir(path.c_str(), MYF(MY_DONT_SORT)); // Access the directory if (dir_info == nullptr) @@ -395,16 +458,16 @@ void Rdb_sst_info::init(rocksdb::DB* db) } // Scan through the files in the directory - struct fileinfo* file_info= dir_info->dir_entry; + const struct fileinfo* file_info= dir_info->dir_entry; for (uint ii= 0; ii < dir_info->number_off_files; ii++, file_info++) { // find any files ending with m_suffix ... - std::string name= file_info->name; - size_t pos= name.find(m_suffix); + const std::string name= file_info->name; + const size_t pos= name.find(m_suffix); if (pos != std::string::npos && name.size() - pos == m_suffix.size()) { // ... and remove them - std::string fullname= path + name; + const std::string fullname= path + name; my_delete(fullname.c_str(), MYF(0)); } } diff --git a/storage/rocksdb/rdb_sst_info.h b/storage/rocksdb/rdb_sst_info.h index 8845ec98122..933357c8f08 100644 --- a/storage/rocksdb/rdb_sst_info.h +++ b/storage/rocksdb/rdb_sst_info.h @@ -33,18 +33,24 @@ namespace myrocks { class Rdb_sst_file { + private: Rdb_sst_file(const Rdb_sst_file& p)= delete; Rdb_sst_file& operator=(const Rdb_sst_file& p)= delete; - rocksdb::DB* m_db; - rocksdb::ColumnFamilyHandle* m_cf; - const rocksdb::DBOptions& m_db_options; - rocksdb::SstFileWriter* m_sst_file_writer; - std::string m_name; + rocksdb::DB* const m_db; + rocksdb::ColumnFamilyHandle* const m_cf; + const rocksdb::DBOptions& m_db_options; + rocksdb::SstFileWriter* m_sst_file_writer; + const std::string m_name; + const bool m_tracing; + + std::string generateKey(const std::string& key); public: - Rdb_sst_file(rocksdb::DB* db, rocksdb::ColumnFamilyHandle* cf, - const rocksdb::DBOptions& db_options, const std::string& name); + Rdb_sst_file(rocksdb::DB* const db, + rocksdb::ColumnFamilyHandle* const cf, + const rocksdb::DBOptions& db_options, const std::string& name, + const bool tracing); ~Rdb_sst_file(); rocksdb::Status open(); @@ -53,26 +59,28 @@ class Rdb_sst_file { }; class Rdb_sst_info { + private: Rdb_sst_info(const Rdb_sst_info& p)= delete; Rdb_sst_info& operator=(const Rdb_sst_info& p)= delete; - rocksdb::DB* m_db; - rocksdb::ColumnFamilyHandle* m_cf; - const rocksdb::DBOptions& m_db_options; - uint64_t m_curr_size; - uint64_t m_max_size; - uint m_sst_count; - std::string m_error_msg; - std::string m_prefix; - static std::string m_suffix; + rocksdb::DB* const m_db; + rocksdb::ColumnFamilyHandle* const m_cf; + const rocksdb::DBOptions& m_db_options; + uint64_t m_curr_size; + uint64_t m_max_size; + uint m_sst_count; + std::string m_error_msg; + std::string m_prefix; + static std::string m_suffix; #if defined(RDB_SST_INFO_USE_THREAD) - std::queue<Rdb_sst_file*> m_queue; - std::mutex m_mutex; - std::condition_variable m_cond; - std::thread* m_thread; - bool m_finished; + std::queue<Rdb_sst_file*> m_queue; + std::mutex m_mutex; + std::condition_variable m_cond; + std::thread* m_thread; + bool m_finished; #endif - Rdb_sst_file* m_sst_file; + Rdb_sst_file* m_sst_file; + const bool m_tracing; int open_new_sst_file(); void close_curr_sst_file(); @@ -85,9 +93,10 @@ class Rdb_sst_info { #endif public: - Rdb_sst_info(rocksdb::DB* db, const std::string& tablename, - const std::string& indexname, rocksdb::ColumnFamilyHandle* cf, - const rocksdb::DBOptions& db_options); + Rdb_sst_info(rocksdb::DB* const db, const std::string& tablename, + const std::string& indexname, + rocksdb::ColumnFamilyHandle* const cf, + const rocksdb::DBOptions& db_options, const bool &tracing); ~Rdb_sst_info(); int put(const rocksdb::Slice& key, const rocksdb::Slice& value); @@ -95,7 +104,7 @@ class Rdb_sst_info { const std::string& error_message() const { return m_error_msg; } - static void init(rocksdb::DB* db); + static void init(const rocksdb::DB* const db); }; } // namespace myrocks diff --git a/storage/rocksdb/rdb_threads.cc b/storage/rocksdb/rdb_threads.cc index 1538404ea56..3f00bc13325 100644 --- a/storage/rocksdb/rdb_threads.cc +++ b/storage/rocksdb/rdb_threads.cc @@ -24,10 +24,10 @@ namespace myrocks { -void* Rdb_thread::thread_func(void* thread_ptr) +void* Rdb_thread::thread_func(void* const thread_ptr) { DBUG_ASSERT(thread_ptr != nullptr); - Rdb_thread* thread= static_cast<Rdb_thread*>(thread_ptr); + Rdb_thread* const thread= static_cast<Rdb_thread* const>(thread_ptr); if (!thread->m_run_once.exchange(true)) { thread->run(); @@ -68,7 +68,7 @@ int Rdb_thread::create_thread( } -void Rdb_thread::signal(bool stop_thread) +void Rdb_thread::signal(const bool &stop_thread) { mysql_mutex_lock(&m_signal_mutex); if (stop_thread) { diff --git a/storage/rocksdb/rdb_threads.h b/storage/rocksdb/rdb_threads.h index cba03b4cdba..c06dba438c0 100644 --- a/storage/rocksdb/rdb_threads.h +++ b/storage/rocksdb/rdb_threads.h @@ -58,7 +58,7 @@ class Rdb_thread virtual void run(void) = 0; - void signal(bool stop_thread= false); + void signal(const bool &stop_thread= false); int join() { @@ -70,7 +70,7 @@ class Rdb_thread virtual ~Rdb_thread() {} private: - static void* thread_func(void* thread_ptr); + static void* thread_func(void* const thread_ptr); }; diff --git a/storage/rocksdb/rdb_utils.cc b/storage/rocksdb/rdb_utils.cc index 05f0104d483..599f11c5681 100644 --- a/storage/rocksdb/rdb_utils.cc +++ b/storage/rocksdb/rdb_utils.cc @@ -32,7 +32,8 @@ namespace myrocks { /* Skip past any spaces in the input */ -const char* rdb_skip_spaces(struct charset_info_st* cs, const char *str) +const char* rdb_skip_spaces(const struct charset_info_st* const cs, + const char *str) { DBUG_ASSERT(cs != nullptr); DBUG_ASSERT(str != nullptr); @@ -50,7 +51,7 @@ const char* rdb_skip_spaces(struct charset_info_st* cs, const char *str) Note that str1 can be longer but we only compare up to the number of characters in str2. */ -bool rdb_compare_strings_ic(const char *str1, const char *str2) +bool rdb_compare_strings_ic(const char* const str1, const char* const str2) { DBUG_ASSERT(str1 != nullptr); DBUG_ASSERT(str2 != nullptr); @@ -74,7 +75,7 @@ bool rdb_compare_strings_ic(const char *str1, const char *str2) and skipping all data enclosed in quotes. */ const char* rdb_find_in_string(const char *str, const char *pattern, - bool *succeeded) + bool * const succeeded) { char quote = '\0'; bool escape = false; @@ -131,8 +132,9 @@ const char* rdb_find_in_string(const char *str, const char *pattern, /* See if the next valid token matches the specified string */ -const char* rdb_check_next_token(struct charset_info_st* cs, const char *str, - const char *pattern, bool *succeeded) +const char* rdb_check_next_token(const struct charset_info_st* const cs, + const char *str, const char* const pattern, + bool* const succeeded) { DBUG_ASSERT(cs != nullptr); DBUG_ASSERT(str != nullptr); @@ -156,8 +158,8 @@ const char* rdb_check_next_token(struct charset_info_st* cs, const char *str, /* Parse id */ -const char* rdb_parse_id(struct charset_info_st* cs, const char *str, - std::string *id) +const char* rdb_parse_id(const struct charset_info_st* const cs, + const char *str, std::string * const id) { DBUG_ASSERT(cs != nullptr); DBUG_ASSERT(str != nullptr); @@ -232,7 +234,7 @@ const char* rdb_parse_id(struct charset_info_st* cs, const char *str, /* Skip id */ -const char* rdb_skip_id(struct charset_info_st* cs, const char *str) +const char* rdb_skip_id(const struct charset_info_st* const cs, const char *str) { DBUG_ASSERT(cs != nullptr); DBUG_ASSERT(str != nullptr); @@ -251,8 +253,8 @@ static const std::array<char, 16> rdb_hexdigit = Convert data into a hex string with optional maximum length. If the data is larger than the maximum length trancate it and append "..". */ -std::string rdb_hexdump(const char *data, std::size_t data_len, - std::size_t maxsize) +std::string rdb_hexdump(const char *data, const std::size_t data_len, + const std::size_t maxsize) { DBUG_ASSERT(data != nullptr); @@ -296,8 +298,9 @@ std::string rdb_hexdump(const char *data, std::size_t data_len, */ bool rdb_database_exists(const std::string& db_name) { - std::string dir = std::string(mysql_real_data_home) + FN_DIRSEP + db_name; - struct st_my_dir* dir_info = my_dir(dir.c_str(), + const std::string dir = std::string(mysql_real_data_home) + FN_DIRSEP + + db_name; + struct st_my_dir* const dir_info = my_dir(dir.c_str(), MYF(MY_DONT_SORT | MY_WANT_STAT)); if (dir_info == nullptr) { diff --git a/storage/rocksdb/rdb_utils.h b/storage/rocksdb/rdb_utils.h index 5630dc1c20f..7d63ff9c220 100644 --- a/storage/rocksdb/rdb_utils.h +++ b/storage/rocksdb/rdb_utils.h @@ -16,6 +16,7 @@ #pragma once /* C++ standard header files */ +#include <chrono> #include <string> /* MySQL header files */ @@ -101,6 +102,11 @@ namespace myrocks { #endif /* + Generic constant. +*/ +const size_t RDB_MAX_HEXDUMP_LEN= 1000; + +/* Helper function to get an NULL terminated uchar* out of a given MySQL String. */ @@ -121,6 +127,15 @@ inline const uchar* rdb_std_str_to_uchar_ptr(const std::string &str) } /* + Helper function to convert seconds to milliseconds. +*/ + +constexpr int rdb_convert_sec_to_ms(int sec) +{ + return std::chrono::milliseconds(std::chrono::seconds(sec)).count(); +} + +/* Helper function to get plain (not necessary NULL terminated) uchar* out of a given RocksDB item. */ @@ -169,33 +184,35 @@ inline int purge_all_jemalloc_arenas() Helper functions to parse strings. */ -const char* rdb_skip_spaces(struct charset_info_st* cs, const char *str) +const char* rdb_skip_spaces(const struct charset_info_st* const cs, + const char *str) __attribute__((__nonnull__, __warn_unused_result__)); -bool rdb_compare_strings_ic(const char *str1, const char *str2) +bool rdb_compare_strings_ic(const char* const str1, const char* const str2) __attribute__((__nonnull__, __warn_unused_result__)); const char* rdb_find_in_string(const char *str, const char *pattern, - bool *succeeded) + bool * const succeeded) __attribute__((__nonnull__, __warn_unused_result__)); -const char* rdb_check_next_token(struct charset_info_st* cs, const char *str, - const char *pattern, bool *succeeded) +const char* rdb_check_next_token(const struct charset_info_st* const cs, + const char *str, const char* const pattern, + bool * const succeeded) __attribute__((__nonnull__, __warn_unused_result__)); -const char* rdb_parse_id(struct charset_info_st* cs, const char *str, - std::string *id) +const char* rdb_parse_id(const struct charset_info_st* const cs, + const char *str, std::string * const id) __attribute__((__nonnull__(1, 2), __warn_unused_result__)); -const char* rdb_skip_id(struct charset_info_st* cs, const char *str) +const char* rdb_skip_id(const struct charset_info_st* const cs, const char *str) __attribute__((__nonnull__, __warn_unused_result__)); /* Helper functions to populate strings. */ -std::string rdb_hexdump(const char *data, std::size_t data_len, - std::size_t maxsize = 0) +std::string rdb_hexdump(const char *data, const std::size_t data_len, + const std::size_t maxsize = 0) __attribute__((__nonnull__)); /* diff --git a/storage/rocksdb/tools/mysql_ldb.cc b/storage/rocksdb/tools/mysql_ldb.cc index ce51481690b..52d23f20a32 100644 --- a/storage/rocksdb/tools/mysql_ldb.cc +++ b/storage/rocksdb/tools/mysql_ldb.cc @@ -8,7 +8,7 @@ int main(int argc, char** argv) { rocksdb::Options db_options; - myrocks::Rdb_pk_comparator pk_comparator; + const myrocks::Rdb_pk_comparator pk_comparator; db_options.comparator= &pk_comparator; rocksdb::LDBTool tool; |