diff options
Diffstat (limited to 'storage/tokudb/ft-index/src')
46 files changed, 1835 insertions, 155 deletions
diff --git a/storage/tokudb/ft-index/src/CMakeLists.txt b/storage/tokudb/ft-index/src/CMakeLists.txt index 394d120beaa..7487d1880aa 100644 --- a/storage/tokudb/ft-index/src/CMakeLists.txt +++ b/storage/tokudb/ft-index/src/CMakeLists.txt @@ -42,8 +42,14 @@ endif () # detect when we are being built as a subproject if (NOT DEFINED MYSQL_PROJECT_NAME_DOCSTRING) install( - TARGETS ${LIBTOKUDB} ${LIBTOKUDB}_static - DESTINATION lib + TARGETS ${LIBTOKUDB} + DESTINATION ${INSTALL_LIBDIR} + COMPONENT tokukv_libs_shared + ) + install( + TARGETS ${LIBTOKUDB}_static + DESTINATION ${INSTALL_LIBDIR} + COMPONENT tokukv_libs_static ) endif () diff --git a/storage/tokudb/ft-index/src/export.map b/storage/tokudb/ft-index/src/export.map index 566e751bd74..3f2c7569ea4 100644 --- a/storage/tokudb/ft-index/src/export.map +++ b/storage/tokudb/ft-index/src/export.map @@ -5,6 +5,7 @@ db_strerror; db_version; db_env_set_direct_io; + db_env_set_compress_buffers_before_eviction; db_env_set_func_fsync; db_env_set_func_malloc; db_env_set_func_realloc; diff --git a/storage/tokudb/ft-index/src/indexer-undo-do.cc b/storage/tokudb/ft-index/src/indexer-undo-do.cc index a9a2c52bf98..2201c4f37e3 100644 --- a/storage/tokudb/ft-index/src/indexer-undo-do.cc +++ b/storage/tokudb/ft-index/src/indexer-undo-do.cc @@ -615,7 +615,16 @@ indexer_ft_delete_committed(DB_INDEXER *indexer, DB *hotdb, DBT *hotkey, XIDS xi } else { result = toku_ydb_check_avail_fs_space(indexer->i->env); if (result == 0) { - toku_ft_send_delete(db_struct_i(hotdb)->ft_handle, hotkey, xids, TXNID_NONE, make_gc_info(true)); + FT_HANDLE ft_h = db_struct_i(hotdb)->ft_handle; + TXN_MANAGER txn_manager = toku_ft_get_txn_manager(ft_h); + txn_manager_state txn_state_for_gc(txn_manager); + + TXNID oldest_referenced_xid_estimate = toku_ft_get_oldest_referenced_xid_estimate(ft_h); + txn_gc_info gc_info(&txn_state_for_gc, + oldest_referenced_xid_estimate, + oldest_referenced_xid_estimate, + true); + toku_ft_send_delete(db_struct_i(hotdb)->ft_handle, hotkey, xids, &gc_info); } } return result; @@ -651,7 +660,16 @@ indexer_ft_insert_committed(DB_INDEXER *indexer, DB *hotdb, DBT *hotkey, DBT *ho } else { result = toku_ydb_check_avail_fs_space(indexer->i->env); if (result == 0) { - toku_ft_send_insert(db_struct_i(hotdb)->ft_handle, hotkey, hotval, xids, FT_INSERT, TXNID_NONE, make_gc_info(true)); + FT_HANDLE ft_h = db_struct_i(hotdb)->ft_handle; + TXN_MANAGER txn_manager = toku_ft_get_txn_manager(ft_h); + txn_manager_state txn_state_for_gc(txn_manager); + + TXNID oldest_referenced_xid_estimate = toku_ft_get_oldest_referenced_xid_estimate(ft_h); + txn_gc_info gc_info(&txn_state_for_gc, + oldest_referenced_xid_estimate, + oldest_referenced_xid_estimate, + true); + toku_ft_send_insert(db_struct_i(hotdb)->ft_handle, hotkey, hotval, xids, FT_INSERT, &gc_info); } } return result; @@ -670,8 +688,18 @@ indexer_ft_commit(DB_INDEXER *indexer, DB *hotdb, DBT *hotkey, XIDS xids) { result = indexer->i->test_commit_any(indexer, hotdb, hotkey, xids); } else { result = toku_ydb_check_avail_fs_space(indexer->i->env); - if (result == 0) - toku_ft_send_commit_any(db_struct_i(hotdb)->ft_handle, hotkey, xids, TXNID_NONE, make_gc_info(true)); + if (result == 0) { + FT_HANDLE ft_h = db_struct_i(hotdb)->ft_handle; + TXN_MANAGER txn_manager = toku_ft_get_txn_manager(ft_h); + txn_manager_state txn_state_for_gc(txn_manager); + + TXNID oldest_referenced_xid_estimate = toku_ft_get_oldest_referenced_xid_estimate(ft_h); + txn_gc_info gc_info(&txn_state_for_gc, + oldest_referenced_xid_estimate, + oldest_referenced_xid_estimate, + true); + toku_ft_send_commit_any(db_struct_i(hotdb)->ft_handle, hotkey, xids, &gc_info); + } } } return result; diff --git a/storage/tokudb/ft-index/src/loader.cc b/storage/tokudb/ft-index/src/loader.cc index 57beee3810f..93ac551632e 100644 --- a/storage/tokudb/ft-index/src/loader.cc +++ b/storage/tokudb/ft-index/src/loader.cc @@ -319,8 +319,8 @@ toku_loader_create_loader(DB_ENV *env, } TOKUTXN ttxn = txn ? db_txn_struct_i(txn)->tokutxn : NULL; rval = toku_ft_loader_open(&loader->i->ft_loader, - loader->i->env->i->cachetable, - loader->i->env->i->generate_row_for_put, + env->i->cachetable, + env->i->generate_row_for_put, src_db, N, brts, dbs, @@ -330,7 +330,7 @@ toku_loader_create_loader(DB_ENV *env, load_lsn, ttxn, puts_allowed, - loader->i->env->i->loader_memory_size, + env->get_loader_memory_size(env), compress_intermediates); if ( rval!=0 ) { toku_free(new_inames_in_env); diff --git a/storage/tokudb/ft-index/src/tests/CMakeLists.txt b/storage/tokudb/ft-index/src/tests/CMakeLists.txt index af04123ca93..d85d24be001 100644 --- a/storage/tokudb/ft-index/src/tests/CMakeLists.txt +++ b/storage/tokudb/ft-index/src/tests/CMakeLists.txt @@ -67,10 +67,12 @@ if(BUILD_TESTING OR BUILD_SRC_TESTS) dump-env env_loader_memory env-put-multiple + env_nproc env_startup execute-updates filesize get_key_after_bytes_unit + get_last_key helgrind1 helgrind2 helgrind3 @@ -105,6 +107,7 @@ if(BUILD_TESTING OR BUILD_SRC_TESTS) loader-stress-del loader-stress-test loader-tpch-load + locktree_escalation_stalls lock-pressure manyfiles maxsize-for-loader @@ -146,6 +149,7 @@ if(BUILD_TESTING OR BUILD_SRC_TESTS) recover-2483 recover-3113 recover-5146 + recover-child-rollback recover-compare-db recover-compare-db-descriptor recover-del-multiple @@ -252,10 +256,12 @@ if(BUILD_TESTING OR BUILD_SRC_TESTS) test_abort1 test_abort4 test_abort5 + test_bad_implicit_promotion test_blobs_leaf_split test_bulk_fetch test_compression_methods test_cmp_descriptor + test_cursor_interrupt test_cursor_with_read_txn test_db_change_pagesize test_db_change_xxx @@ -264,6 +270,7 @@ if(BUILD_TESTING OR BUILD_SRC_TESTS) test_db_descriptor_named_db test_db_txn_locks_read_uncommitted test_get_max_row_size + test_insert_many_gc test_iterate_live_transactions test_iterate_pending_lock_requests test_large_update_broadcast_small_cachetable @@ -578,7 +585,7 @@ if(BUILD_TESTING OR BUILD_SRC_TESTS) foreach(oldver 4.2.0 5.0.8 5.2.7 6.0.0 6.1.0 6.5.1 6.6.3) set(versiondir ${TOKUDB_DATA}/old-stress-test-envs/${oldver}) - if (NOT EXISTS "${versiondir}/" AND NOT WARNED_ABOUT_DATA) + if (NOT EXISTS "${versiondir}/" AND NOT WARNED_ABOUT_DATA AND CMAKE_PROJECT_NAME STREQUAL TokuDB) message(WARNING "Test data for upgrade tests for version ${oldver} doesn't exist, check out ${versiondir}/*-2000-dir first or upgrade stress tests may fail.") set(WARNED_ABOUT_DATA 1) endif () @@ -612,7 +619,7 @@ if(BUILD_TESTING OR BUILD_SRC_TESTS) endif () endforeach(src) - if (NOT EXISTS "${TOKUDB_DATA}/test_5902/" AND NOT WARNED_ABOUT_DATA) + if (NOT EXISTS "${TOKUDB_DATA}/test_5902/" AND NOT WARNED_ABOUT_DATA AND CMAKE_PROJECT_NAME STREQUAL TokuDB) message(WARNING "Test data for dump-env.tdb doesn't exist, check out ${TOKUDB_DATA}/test_5902 first or dump-env.tdb may fail.") set(WARNED_ABOUT_DATA 1) endif () @@ -653,7 +660,7 @@ if(BUILD_TESTING OR BUILD_SRC_TESTS) endforeach(num) set(old_loader_upgrade_data "${TOKUDB_DATA}/env_preload.4.2.0.emptydictionaries.cleanshutdown") - if (NOT EXISTS "${old_loader_upgrade_data}/" AND NOT WARNED_ABOUT_DATA) + if (NOT EXISTS "${old_loader_upgrade_data}/" AND NOT WARNED_ABOUT_DATA AND CMAKE_PROJECT_NAME STREQUAL TokuDB) message(WARNING "Test data for loader upgrade tests doesn't exist, check out ${old_loader_upgrade_data} first, or loader-stress-test3.tdb may fail.") set(WARNED_ABOUT_DATA 1) endif () diff --git a/storage/tokudb/ft-index/src/tests/blocking-first-empty.cc b/storage/tokudb/ft-index/src/tests/blocking-first-empty.cc index 442ccfd112d..72267437a4a 100644 --- a/storage/tokudb/ft-index/src/tests/blocking-first-empty.cc +++ b/storage/tokudb/ft-index/src/tests/blocking-first-empty.cc @@ -220,7 +220,7 @@ int test_main(int argc, char * const argv[]) { } r = db_env->open(db_env, db_env_dir, db_env_open_flags, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); assert(r == 0); #if TOKUDB - r = db_env->set_lock_timeout(db_env, 30 * 1000); assert(r == 0); + r = db_env->set_lock_timeout(db_env, 30 * 1000, nullptr); assert(r == 0); #endif // create the db diff --git a/storage/tokudb/ft-index/src/tests/blocking-first.cc b/storage/tokudb/ft-index/src/tests/blocking-first.cc index 9be783d2a04..1bd414a90aa 100644 --- a/storage/tokudb/ft-index/src/tests/blocking-first.cc +++ b/storage/tokudb/ft-index/src/tests/blocking-first.cc @@ -237,7 +237,7 @@ int test_main(int argc, char * const argv[]) { } r = db_env->open(db_env, db_env_dir, db_env_open_flags, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); assert(r == 0); #if TOKUDB - r = db_env->set_lock_timeout(db_env, 30 * 1000); assert(r == 0); + r = db_env->set_lock_timeout(db_env, 30 * 1000, nullptr); assert(r == 0); #endif // create the db diff --git a/storage/tokudb/ft-index/src/tests/blocking-last.cc b/storage/tokudb/ft-index/src/tests/blocking-last.cc index 9166a07aa28..5ca6ce5ec6a 100644 --- a/storage/tokudb/ft-index/src/tests/blocking-last.cc +++ b/storage/tokudb/ft-index/src/tests/blocking-last.cc @@ -237,7 +237,7 @@ int test_main(int argc, char * const argv[]) { } r = db_env->open(db_env, db_env_dir, db_env_open_flags, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); assert(r == 0); #if TOKUDB - r = db_env->set_lock_timeout(db_env, 30 * 1000); assert(r == 0); + r = db_env->set_lock_timeout(db_env, 30 * 1000, nullptr); assert(r == 0); #endif // create the db diff --git a/storage/tokudb/ft-index/src/tests/blocking-next-prev-deadlock.cc b/storage/tokudb/ft-index/src/tests/blocking-next-prev-deadlock.cc index c85996275bd..fb591feff28 100644 --- a/storage/tokudb/ft-index/src/tests/blocking-next-prev-deadlock.cc +++ b/storage/tokudb/ft-index/src/tests/blocking-next-prev-deadlock.cc @@ -306,7 +306,7 @@ int test_main(int argc, char * const argv[]) { } r = db_env->open(db_env, db_env_dir, db_env_open_flags, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); assert(r == 0); #if TOKUDB - r = db_env->set_lock_timeout(db_env, 30 * 1000); assert(r == 0); + r = db_env->set_lock_timeout(db_env, 30 * 1000, nullptr); assert(r == 0); #else r = db_env->set_lk_detect(db_env, DB_LOCK_YOUNGEST); assert(r == 0); #endif diff --git a/storage/tokudb/ft-index/src/tests/blocking-prelock-range.cc b/storage/tokudb/ft-index/src/tests/blocking-prelock-range.cc index 561f2550bb9..cece47cbeec 100644 --- a/storage/tokudb/ft-index/src/tests/blocking-prelock-range.cc +++ b/storage/tokudb/ft-index/src/tests/blocking-prelock-range.cc @@ -182,7 +182,7 @@ int test_main(int argc, char * const argv[]) { } r = db_env->open(db_env, db_env_dir, db_env_open_flags, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); assert(r == 0); #if TOKUDB - r = db_env->set_lock_timeout(db_env, 30 * 1000); assert(r == 0); + r = db_env->set_lock_timeout(db_env, 30 * 1000, nullptr); assert(r == 0); #endif // create the db diff --git a/storage/tokudb/ft-index/src/tests/blocking-put-timeout.cc b/storage/tokudb/ft-index/src/tests/blocking-put-timeout.cc index cffb164fba3..c43bcfad5f0 100644 --- a/storage/tokudb/ft-index/src/tests/blocking-put-timeout.cc +++ b/storage/tokudb/ft-index/src/tests/blocking-put-timeout.cc @@ -233,7 +233,7 @@ int test_main(int argc, char * const argv[]) { uint64_t lock_timeout_msec; r = db_env->get_lock_timeout(db_env, &lock_timeout_msec); assert(r == 0); if (verbose) printf("lock timeout: %" PRIu64 "\n", lock_timeout_msec); - r = db_env->set_lock_timeout(db_env, 5000); assert(r == 0); + r = db_env->set_lock_timeout(db_env, 5000, nullptr); assert(r == 0); r = db_env->get_lock_timeout(db_env, &lock_timeout_msec); assert(r == 0); if (verbose) printf("lock timeout: %" PRIu64 "\n", lock_timeout_msec); #endif diff --git a/storage/tokudb/ft-index/src/tests/blocking-put-wakeup.cc b/storage/tokudb/ft-index/src/tests/blocking-put-wakeup.cc index f2f7ea0e825..9c8ad60787b 100644 --- a/storage/tokudb/ft-index/src/tests/blocking-put-wakeup.cc +++ b/storage/tokudb/ft-index/src/tests/blocking-put-wakeup.cc @@ -209,7 +209,7 @@ int test_main(int argc, char * const argv[]) { } r = db_env->open(db_env, db_env_dir, db_env_open_flags, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); assert(r == 0); #if TOKUDB - r = db_env->set_lock_timeout(db_env, 30 * 1000); assert(r == 0); + r = db_env->set_lock_timeout(db_env, 30 * 1000, nullptr); assert(r == 0); #endif // create the db diff --git a/storage/tokudb/ft-index/src/tests/blocking-put.cc b/storage/tokudb/ft-index/src/tests/blocking-put.cc index d377e86c236..afd083986d8 100644 --- a/storage/tokudb/ft-index/src/tests/blocking-put.cc +++ b/storage/tokudb/ft-index/src/tests/blocking-put.cc @@ -181,7 +181,7 @@ int test_main(int argc, char * const argv[]) { } r = db_env->open(db_env, db_env_dir, db_env_open_flags, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); assert(r == 0); #if TOKUDB - r = db_env->set_lock_timeout(db_env, 30 * 1000); assert(r == 0); + r = db_env->set_lock_timeout(db_env, 30 * 1000, nullptr); assert(r == 0); #endif // create the db diff --git a/storage/tokudb/ft-index/src/tests/blocking-set-range-0.cc b/storage/tokudb/ft-index/src/tests/blocking-set-range-0.cc index a97d6d2f935..8d2dad831a2 100644 --- a/storage/tokudb/ft-index/src/tests/blocking-set-range-0.cc +++ b/storage/tokudb/ft-index/src/tests/blocking-set-range-0.cc @@ -247,7 +247,7 @@ int test_main(int argc, char * const argv[]) { } r = db_env->open(db_env, db_env_dir, db_env_open_flags, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); assert(r == 0); #if TOKUDB - r = db_env->set_lock_timeout(db_env, 30 * 1000); assert(r == 0); + r = db_env->set_lock_timeout(db_env, 30 * 1000, nullptr); assert(r == 0); #endif // create the db diff --git a/storage/tokudb/ft-index/src/tests/blocking-set-range-n.cc b/storage/tokudb/ft-index/src/tests/blocking-set-range-n.cc index ee2787554f4..838d18ed6d5 100644 --- a/storage/tokudb/ft-index/src/tests/blocking-set-range-n.cc +++ b/storage/tokudb/ft-index/src/tests/blocking-set-range-n.cc @@ -242,7 +242,7 @@ int test_main(int argc, char * const argv[]) { } r = db_env->open(db_env, db_env_dir, db_env_open_flags, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); assert(r == 0); #if TOKUDB - r = db_env->set_lock_timeout(db_env, 30 * 1000); assert(r == 0); + r = db_env->set_lock_timeout(db_env, 30 * 1000, nullptr); assert(r == 0); #endif // create the db diff --git a/storage/tokudb/ft-index/src/tests/blocking-set-range-reverse-0.cc b/storage/tokudb/ft-index/src/tests/blocking-set-range-reverse-0.cc index 829672dedae..7d06dc4c9e0 100644 --- a/storage/tokudb/ft-index/src/tests/blocking-set-range-reverse-0.cc +++ b/storage/tokudb/ft-index/src/tests/blocking-set-range-reverse-0.cc @@ -245,7 +245,7 @@ int test_main(int argc, char * const argv[]) { } r = db_env->open(db_env, db_env_dir, db_env_open_flags, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); assert(r == 0); #if TOKUDB - r = db_env->set_lock_timeout(db_env, 30 * 1000); assert(r == 0); + r = db_env->set_lock_timeout(db_env, 30 * 1000, nullptr); assert(r == 0); #endif // create the db diff --git a/storage/tokudb/ft-index/src/tests/blocking-set.cc b/storage/tokudb/ft-index/src/tests/blocking-set.cc index 70c17da513f..d229321aa43 100644 --- a/storage/tokudb/ft-index/src/tests/blocking-set.cc +++ b/storage/tokudb/ft-index/src/tests/blocking-set.cc @@ -227,7 +227,7 @@ int test_main(int argc, char * const argv[]) { } r = db_env->open(db_env, db_env_dir, db_env_open_flags, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); assert(r == 0); #if TOKUDB - r = db_env->set_lock_timeout(db_env, 30 * 1000); assert(r == 0); + r = db_env->set_lock_timeout(db_env, 30 * 1000, nullptr); assert(r == 0); #endif // create the db diff --git a/storage/tokudb/ft-index/src/tests/blocking-table-lock.cc b/storage/tokudb/ft-index/src/tests/blocking-table-lock.cc index 91626773f35..2ae82319a7e 100644 --- a/storage/tokudb/ft-index/src/tests/blocking-table-lock.cc +++ b/storage/tokudb/ft-index/src/tests/blocking-table-lock.cc @@ -175,7 +175,7 @@ int test_main(int argc, char * const argv[]) { } r = db_env->open(db_env, db_env_dir, db_env_open_flags, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); assert(r == 0); #if TOKUDB - r = db_env->set_lock_timeout(db_env, 30 * 1000); assert(r == 0); + r = db_env->set_lock_timeout(db_env, 30 * 1000, nullptr); assert(r == 0); #endif // create the db diff --git a/storage/tokudb/ft-index/src/tests/db-put-simple-deadlock-threads.cc b/storage/tokudb/ft-index/src/tests/db-put-simple-deadlock-threads.cc index e78c95cb2e3..004a7aad12d 100644 --- a/storage/tokudb/ft-index/src/tests/db-put-simple-deadlock-threads.cc +++ b/storage/tokudb/ft-index/src/tests/db-put-simple-deadlock-threads.cc @@ -273,7 +273,7 @@ int test_main(int argc, char * const argv[]) { #endif r = db_env->open(db_env, db_env_dir, db_env_open_flags, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); assert(r == 0); #if defined(USE_TDB) - r = db_env->set_lock_timeout(db_env, 0); assert(r == 0); // no wait + r = db_env->set_lock_timeout(db_env, 0, nullptr); assert(r == 0); // no wait #elif defined(USE_BDB) r = db_env->set_lk_detect(db_env, DB_LOCK_YOUNGEST); assert(r == 0); r = db_env->set_timeout(db_env, 10000, DB_SET_LOCK_TIMEOUT); assert(r == 0); diff --git a/storage/tokudb/ft-index/src/tests/db-put-simple-lockwait.cc b/storage/tokudb/ft-index/src/tests/db-put-simple-lockwait.cc index d89235d7f59..54682db81af 100644 --- a/storage/tokudb/ft-index/src/tests/db-put-simple-lockwait.cc +++ b/storage/tokudb/ft-index/src/tests/db-put-simple-lockwait.cc @@ -214,7 +214,7 @@ int test_main(int argc, char * const argv[]) { db_env_open_flags &= ~(DB_INIT_TXN | DB_INIT_LOG); r = db_env->open(db_env, db_env_dir, db_env_open_flags, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); assert(r == 0); #if TOKUDB - r = db_env->set_lock_timeout(db_env, 30 * 1000); assert(r == 0); + r = db_env->set_lock_timeout(db_env, 30 * 1000, nullptr); assert(r == 0); #else r = db_env->set_lk_detect(db_env, DB_LOCK_YOUNGEST); assert(r == 0); #endif diff --git a/storage/tokudb/ft-index/src/tests/db-put-update-deadlock.cc b/storage/tokudb/ft-index/src/tests/db-put-update-deadlock.cc index 21ae8d853f1..1a546825fc4 100644 --- a/storage/tokudb/ft-index/src/tests/db-put-update-deadlock.cc +++ b/storage/tokudb/ft-index/src/tests/db-put-update-deadlock.cc @@ -275,7 +275,7 @@ int test_main(int argc, char * const argv[]) { db_env_open_flags &= ~(DB_INIT_TXN | DB_INIT_LOG); r = db_env->open(db_env, db_env_dir, db_env_open_flags, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); assert(r == 0); #if defined(TOKUDB) - r = db_env->set_lock_timeout(db_env, 30 * 1000); assert(r == 0); + r = db_env->set_lock_timeout(db_env, 30 * 1000, nullptr); assert(r == 0); #endif #if defined(USE_BDB) if (!poll_deadlock) { diff --git a/storage/tokudb/ft-index/src/tests/env_loader_memory.cc b/storage/tokudb/ft-index/src/tests/env_loader_memory.cc index 1cbdc1b16f5..106bdefd3a9 100644 --- a/storage/tokudb/ft-index/src/tests/env_loader_memory.cc +++ b/storage/tokudb/ft-index/src/tests/env_loader_memory.cc @@ -91,14 +91,21 @@ PATENT RIGHTS GRANT: #include "test.h" #include <db.h> +static uint64_t my_loader_memory_size; + +static uint64_t get_loader_memory_size(void) { + return my_loader_memory_size; +} + int test_main (int argc, char * const argv[]) { parse_args(argc, argv); int r; DB_ENV *env; r = db_env_create(&env, 0); assert_zero(r); + env->set_loader_memory_size(env, get_loader_memory_size); for (uint64_t n = 0 ; n < 10000000000; n += 1000000000) { - env->set_loader_memory_size(env, n); + my_loader_memory_size = n; assert(env->get_loader_memory_size(env) == n); } r = env->close(env, 0); diff --git a/storage/tokudb/ft-index/src/tests/env_nproc.cc b/storage/tokudb/ft-index/src/tests/env_nproc.cc new file mode 100644 index 00000000000..29bc216f9c8 --- /dev/null +++ b/storage/tokudb/ft-index/src/tests/env_nproc.cc @@ -0,0 +1,143 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4: + +/* +COPYING CONDITIONS NOTICE: + + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation, and provided that the + following conditions are met: + + * Redistributions of source code must retain this COPYING + CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the + DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the + PATENT MARKING NOTICE (below), and the PATENT RIGHTS + GRANT (below). + + * Redistributions in binary form must reproduce this COPYING + CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the + DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the + PATENT MARKING NOTICE (below), and the PATENT RIGHTS + GRANT (below) in the documentation and/or other materials + provided with the distribution. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + 02110-1301, USA. + +COPYRIGHT NOTICE: + + TokuDB, Tokutek Fractal Tree Indexing Library. + Copyright (C) 2007-2013 Tokutek, Inc. + +DISCLAIMER: + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + +UNIVERSITY PATENT NOTICE: + + The technology is licensed by the Massachusetts Institute of + Technology, Rutgers State University of New Jersey, and the Research + Foundation of State University of New York at Stony Brook under + United States of America Serial No. 11/760379 and to the patents + and/or patent applications resulting from it. + +PATENT MARKING NOTICE: + + This software is covered by US Patent No. 8,185,551. + This software is covered by US Patent No. 8,489,638. + +PATENT RIGHTS GRANT: + + "THIS IMPLEMENTATION" means the copyrightable works distributed by + Tokutek as part of the Fractal Tree project. + + "PATENT CLAIMS" means the claims of patents that are owned or + licensable by Tokutek, both currently or in the future; and that in + the absence of this license would be infringed by THIS + IMPLEMENTATION or by using or running THIS IMPLEMENTATION. + + "PATENT CHALLENGE" shall mean a challenge to the validity, + patentability, enforceability and/or non-infringement of any of the + PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS. + + Tokutek hereby grants to you, for the term and geographical scope of + the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free, + irrevocable (except as stated in this section) patent license to + make, have made, use, offer to sell, sell, import, transfer, and + otherwise run, modify, and propagate the contents of THIS + IMPLEMENTATION, where such license applies only to the PATENT + CLAIMS. This grant does not include claims that would be infringed + only as a consequence of further modifications of THIS + IMPLEMENTATION. If you or your agent or licensee institute or order + or agree to the institution of patent litigation against any entity + (including a cross-claim or counterclaim in a lawsuit) alleging that + THIS IMPLEMENTATION constitutes direct or contributory patent + infringement, or inducement of patent infringement, then any rights + granted to you under this License shall terminate as of the date + such litigation is filed. If you or your agent or exclusive + licensee institute or order or agree to the institution of a PATENT + CHALLENGE, then Tokutek may terminate any rights granted to you + under this License. +*/ + +#ident "Copyright (c) 2009-2013 Tokutek Inc. All rights reserved." +#ident "$Id$" + +#include "test.h" +#include <db.h> +#include <sys/resource.h> + +static void env_open_close(void) { + int r; + + DB_ENV *env = NULL; + r = db_env_create(&env, 0); + assert(r == 0); + env->set_errfile(env, stderr); + r = env->open(env, TOKU_TEST_FILENAME, DB_INIT_LOCK+DB_INIT_MPOOL+DB_INIT_TXN+DB_INIT_LOG + DB_CREATE + DB_PRIVATE, S_IRWXU+S_IRWXG+S_IRWXO); + if (r != 0) { + fprintf(stderr, "%s:%u r=%d\n", __FILE__, __LINE__, r); + } + r = env->close(env, 0); + assert(r == 0); +} + +int test_main (int argc, char * const argv[]) { + int r; + int limit = 1; + + // parse_args(argc, argv); + for (int i = 1; i < argc; i++) { + if (strcmp(argv[i], "-v") == 0) { + verbose++; + continue; + } + if (strcmp(argv[i], "-q") == 0) { + if (verbose > 0) verbose--; + continue; + } + limit = atoi(argv[i]); + continue; + } + + toku_os_recursive_delete(TOKU_TEST_FILENAME); + r = toku_os_mkdir(TOKU_TEST_FILENAME, S_IRWXU+S_IRWXG+S_IRWXO); + + struct rlimit nproc_rlimit; + r = getrlimit(RLIMIT_NPROC, &nproc_rlimit); + assert(r == 0); + + nproc_rlimit.rlim_cur = limit; + r = setrlimit(RLIMIT_NPROC, &nproc_rlimit); + assert(r == 0); + + env_open_close(); + + return 0; +} diff --git a/storage/tokudb/ft-index/src/tests/filesize.cc b/storage/tokudb/ft-index/src/tests/filesize.cc index e03bcee4a15..c1b7ad2df30 100644 --- a/storage/tokudb/ft-index/src/tests/filesize.cc +++ b/storage/tokudb/ft-index/src/tests/filesize.cc @@ -206,7 +206,8 @@ delete_n (uint32_t ah) static void optimize(void) { if (verbose) printf("Filesize: begin optimize dictionary\n"); - int r = db->hot_optimize(db, NULL, NULL, NULL, NULL); + uint64_t loops_run; + int r = db->hot_optimize(db, NULL, NULL, NULL, NULL, &loops_run); CKERR(r); if (verbose) printf("Filesize: end optimize dictionary\n"); } diff --git a/storage/tokudb/ft-index/src/tests/get_last_key.cc b/storage/tokudb/ft-index/src/tests/get_last_key.cc new file mode 100644 index 00000000000..36c7ab63259 --- /dev/null +++ b/storage/tokudb/ft-index/src/tests/get_last_key.cc @@ -0,0 +1,299 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4: +#ident "$Id$" +/* +COPYING CONDITIONS NOTICE: + + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation, and provided that the + following conditions are met: + + * Redistributions of source code must retain this COPYING + CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the + DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the + PATENT MARKING NOTICE (below), and the PATENT RIGHTS + GRANT (below). + + * Redistributions in binary form must reproduce this COPYING + CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the + DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the + PATENT MARKING NOTICE (below), and the PATENT RIGHTS + GRANT (below) in the documentation and/or other materials + provided with the distribution. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + 02110-1301, USA. + +COPYRIGHT NOTICE: + + TokuDB, Tokutek Fractal Tree Indexing Library. + Copyright (C) 2007-2013 Tokutek, Inc. + +DISCLAIMER: + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + +UNIVERSITY PATENT NOTICE: + + The technology is licensed by the Massachusetts Institute of + Technology, Rutgers State University of New Jersey, and the Research + Foundation of State University of New York at Stony Brook under + United States of America Serial No. 11/760379 and to the patents + and/or patent applications resulting from it. + +PATENT MARKING NOTICE: + + This software is covered by US Patent No. 8,185,551. + This software is covered by US Patent No. 8,489,638. + +PATENT RIGHTS GRANT: + + "THIS IMPLEMENTATION" means the copyrightable works distributed by + Tokutek as part of the Fractal Tree project. + + "PATENT CLAIMS" means the claims of patents that are owned or + licensable by Tokutek, both currently or in the future; and that in + the absence of this license would be infringed by THIS + IMPLEMENTATION or by using or running THIS IMPLEMENTATION. + + "PATENT CHALLENGE" shall mean a challenge to the validity, + patentability, enforceability and/or non-infringement of any of the + PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS. + + Tokutek hereby grants to you, for the term and geographical scope of + the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free, + irrevocable (except as stated in this section) patent license to + make, have made, use, offer to sell, sell, import, transfer, and + otherwise run, modify, and propagate the contents of THIS + IMPLEMENTATION, where such license applies only to the PATENT + CLAIMS. This grant does not include claims that would be infringed + only as a consequence of further modifications of THIS + IMPLEMENTATION. If you or your agent or licensee institute or order + or agree to the institution of patent litigation against any entity + (including a cross-claim or counterclaim in a lawsuit) alleging that + THIS IMPLEMENTATION constitutes direct or contributory patent + infringement, or inducement of patent infringement, then any rights + granted to you under this License shall terminate as of the date + such litigation is filed. If you or your agent or exclusive + licensee institute or order or agree to the institution of a PATENT + CHALLENGE, then Tokutek may terminate any rights granted to you + under this License. +*/ + +#ident "Copyright (c) 2007-2013 Tokutek Inc. All rights reserved." +#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it." +/** + * Test that various queries behave correctly + * + * Zardosht says: + * + * write a test that inserts a bunch of elements into the tree, + * and then verify that the following types of queries work: + * - db->get + * - next + * - prev + * - set_range + * - set_range_reverse + * - first + * - last + * - current + * + * do it on a table with: + * - just a leaf node + * - has internal nodes (make node size 4K and bn size 1K) + * - big cachetable such that everything fits + * - small cachetable such that not a lot fits + * + * make sure APIs are the callback APIs (getf_XXX) + * make sure your callbacks all return TOKUDB_CURSOR_CONTINUE, + * so we ensure that returning TOKUDB_CURSOR_CONTINUE does not + * mess anything up. + */ + +#include "test.h" + +/** + * Calculate or verify that a value for a given key is correct + * Returns 0 if the value is correct, nonzero otherwise. + */ +static void get_value_by_key(DBT * key, DBT * value) +{ + // keys/values are always stored in the DBT in net order + int * CAST_FROM_VOIDP(k, key->data); + int v = toku_ntohl(*k) * 2 + 1; + memcpy(value->data, &v, sizeof(int)); +} + +static void prepare_for_env(void) { + toku_os_recursive_delete(TOKU_TEST_FILENAME); + int r = toku_os_mkdir(TOKU_TEST_FILENAME, 0755); { int chk_r = r; CKERR(chk_r); } +} + +static void init_env(DB_ENV ** env, size_t ct_size) +{ + int r; + const int envflags = DB_INIT_MPOOL | DB_CREATE | DB_THREAD | + DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_TXN | DB_PRIVATE; + + printf("initializing environment\n"); + + r = db_env_create(env, 0); { int chk_r = r; CKERR(chk_r); } + assert(ct_size < 1024 * 1024 * 1024L); + r = (*env)->set_cachesize(*env, 0, ct_size, 1); { int chk_r = r; CKERR(chk_r); } + r = (*env)->open(*env, TOKU_TEST_FILENAME, envflags, 0755); { int chk_r = r; CKERR(chk_r); } +} + +static void init_db(DB_ENV * env, DB ** db) +{ + int r; + const int node_size = 4096; + const int bn_size = 1024; + + printf("initializing db\n"); + + DB_TXN * txn; + r = db_create(db, env, 0); { int chk_r = r; CKERR(chk_r); } + r = (*db)->set_readpagesize(*db, bn_size); { int chk_r = r; CKERR(chk_r); } + r = (*db)->set_pagesize(*db, node_size); { int chk_r = r; CKERR(chk_r); } + r = env->txn_begin(env, nullptr, &txn, 0); { int chk_r = r; CKERR(chk_r); } + r = (*db)->open(*db, txn, "db", nullptr, DB_BTREE, DB_CREATE, 0644); { int chk_r = r; CKERR(chk_r); } + r = txn->commit(txn, 0); { int chk_r = r; CKERR(chk_r); } +} + +static void cleanup_env_and_db(DB_ENV * env, DB * db) +{ + int r; + + printf("cleaning up environment and db\n"); + r = db->close(db, 0); { int chk_r = r; CKERR(chk_r); } + r = env->close(env, 0); { int chk_r = r; CKERR(chk_r); } +} + +static int get_last_key_cb(const DBT *key, const DBT *value, void *extra) { + if (key->data) { + invariant_null(value); + int expected_key = *(int*)extra; + int found_key = *(int*)key->data; + invariant(expected_key == (int)ntohl(found_key)); + } + return 0; +} + + +static void check_last_key_matches(DB *db, int expect_r, int key) { + int r = db->get_last_key(db, get_last_key_cb, &key); + CKERR2(r, expect_r); +} + +static void do_test(size_t ct_size, int num_keys) +{ + int i, r; + DB * db; + DB_ENV * env; + DB_TXN *txn = nullptr; + DB_TXN *txn2 = nullptr; + uint64_t loops_run = 0; + + + printf("doing tests for ct_size %lu, num_keys %d\n", + ct_size, num_keys); + + // initialize everything and insert data + prepare_for_env(); + init_env(&env, ct_size); + assert(env != nullptr); + init_db(env, &db); + assert(db != nullptr); + + r = env->txn_begin(env, nullptr, &txn, 0); + CKERR(r); + DBT key, value; + for (i = 0; i < num_keys; i++) { + int v, k = toku_htonl(i); + dbt_init(&key, &k, sizeof(int)); + dbt_init(&value, &v, sizeof(int)); + get_value_by_key(&key, &value); + r = db->put(db, txn, &key, &value, 0); + CKERR(r); + } + CKERR(r); + + int expect_r = num_keys == 0 ? DB_NOTFOUND : 0; + check_last_key_matches(db, expect_r, num_keys - 1); + + r = txn->commit(txn, 0); + check_last_key_matches(db, expect_r, num_keys - 1); + + if (num_keys == 0) { + goto cleanup; + } + r = env->txn_begin(env, nullptr, &txn2, 0); + CKERR(r); + r = env->txn_begin(env, nullptr, &txn, 0); + CKERR(r); + + r = db->del(db, txn, &key, 0); + check_last_key_matches(db, 0, num_keys - 1); + + r = txn->commit(txn, 0); + check_last_key_matches(db, 0, num_keys - 1); + + r = txn2->commit(txn2, 0); + check_last_key_matches(db, 0, num_keys - 1); + + //Run Garbage collection (NOTE does not work when everything fits in root??? WHY) + r = db->hot_optimize(db, nullptr, nullptr, nullptr, nullptr, &loops_run); + CKERR(r); + + r = env->txn_checkpoint(env, 0, 0, 0); + CKERR(r); + + //Run Garbage collection (NOTE does not work when everything fits in root??? WHY) + r = db->hot_optimize(db, nullptr, nullptr, nullptr, nullptr, &loops_run); + CKERR(r); + + r = env->txn_checkpoint(env, 0, 0, 0); + CKERR(r); + + //Fully close and reopen + //This clears cachetable + //note that closing a db and reopening may not flush the cachetable so we close env as well + cleanup_env_and_db(env, db); + init_env(&env, ct_size); + assert(env != nullptr); + init_db(env, &db); + assert(db != nullptr); + + //NOTE: tried overkill (double optimize, double checkpoint.. gc still doesn't happen for everything in root in single basement + + if (num_keys >= 2) { + // At least one key remains. + check_last_key_matches(db, 0, num_keys - 2); + } else { + //no key remains. Should find nothing. + check_last_key_matches(db, DB_NOTFOUND, -1); + } +cleanup: + cleanup_env_and_db(env, db); +} + +int test_main(int argc, char * const argv[]) +{ + default_parse_args(argc, argv); + + for (int i = 0; i <= 2; i++) { + do_test(1024*1024, i); + } + for (int i = 4; i <= 1024; i*=2) { + do_test(1024*1024, i); + } + + return 0; +} + diff --git a/storage/tokudb/ft-index/src/tests/hot-optimize-table-tests.cc b/storage/tokudb/ft-index/src/tests/hot-optimize-table-tests.cc index ce245ce69f9..6a00afa4a51 100644 --- a/storage/tokudb/ft-index/src/tests/hot-optimize-table-tests.cc +++ b/storage/tokudb/ft-index/src/tests/hot-optimize-table-tests.cc @@ -247,7 +247,8 @@ hot_test(DB* db, unsigned int size) // Flatten the tree. verbose ? printf("Calling hot optimize...\n") : 0; - r = db->hot_optimize(db, NULL, NULL, NULL, NULL); + uint64_t loops_run; + r = db->hot_optimize(db, NULL, NULL, NULL, NULL, &loops_run); assert(r == 0); verbose ? printf("HOT Finished!\n") : 0; for (unsigned int i = 0; i < size; ++i) { diff --git a/storage/tokudb/ft-index/src/tests/locktree_escalation_stalls.cc b/storage/tokudb/ft-index/src/tests/locktree_escalation_stalls.cc new file mode 100644 index 00000000000..037d2fc46a9 --- /dev/null +++ b/storage/tokudb/ft-index/src/tests/locktree_escalation_stalls.cc @@ -0,0 +1,312 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4: + +/* +COPYING CONDITIONS NOTICE: + + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation, and provided that the + following conditions are met: + + * Redistributions of source code must retain this COPYING + CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the + DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the + PATENT MARKING NOTICE (below), and the PATENT RIGHTS + GRANT (below). + + * Redistributions in binary form must reproduce this COPYING + CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the + DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the + PATENT MARKING NOTICE (below), and the PATENT RIGHTS + GRANT (below) in the documentation and/or other materials + provided with the distribution. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + 02110-1301, USA. + +COPYRIGHT NOTICE: + + TokuDB, Tokutek Fractal Tree Indexing Library. + Copyright (C) 2007-2013 Tokutek, Inc. + +DISCLAIMER: + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + +UNIVERSITY PATENT NOTICE: + + The technology is licensed by the Massachusetts Institute of + Technology, Rutgers State University of New Jersey, and the Research + Foundation of State University of New York at Stony Brook under + United States of America Serial No. 11/760379 and to the patents + and/or patent applications resulting from it. + +PATENT MARKING NOTICE: + + This software is covered by US Patent No. 8,185,551. + This software is covered by US Patent No. 8,489,638. + +PATENT RIGHTS GRANT: + + "THIS IMPLEMENTATION" means the copyrightable works distributed by + Tokutek as part of the Fractal Tree project. + + "PATENT CLAIMS" means the claims of patents that are owned or + licensable by Tokutek, both currently or in the future; and that in + the absence of this license would be infringed by THIS + IMPLEMENTATION or by using or running THIS IMPLEMENTATION. + + "PATENT CHALLENGE" shall mean a challenge to the validity, + patentability, enforceability and/or non-infringement of any of the + PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS. + + Tokutek hereby grants to you, for the term and geographical scope of + the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free, + irrevocable (except as stated in this section) patent license to + make, have made, use, offer to sell, sell, import, transfer, and + otherwise run, modify, and propagate the contents of THIS + IMPLEMENTATION, where such license applies only to the PATENT + CLAIMS. This grant does not include claims that would be infringed + only as a consequence of further modifications of THIS + IMPLEMENTATION. If you or your agent or licensee institute or order + or agree to the institution of patent litigation against any entity + (including a cross-claim or counterclaim in a lawsuit) alleging that + THIS IMPLEMENTATION constitutes direct or contributory patent + infringement, or inducement of patent infringement, then any rights + granted to you under this License shall terminate as of the date + such litigation is filed. If you or your agent or exclusive + licensee institute or order or agree to the institution of a PATENT + CHALLENGE, then Tokutek may terminate any rights granted to you + under this License. +*/ + +#ident "Copyright (c) 2009-2013 Tokutek Inc. All rights reserved." +#ident "$Id$" + +// This test ensures that lock escalation occurs on the big transaction thread. +// locktree_escalation_stalls --max_i 1000000000 --n_small 16 --verbose + +#include "test.h" +#include <db.h> +#include "toku_time.h" +#include "toku_pthread.h" + +// from #include "threaded_stress_test_helpers.h" +// For each line of engine status output, look for lines that contain substrings +// that match any of the strings in the pattern string. The pattern string contains +// 0 or more strings separated by the '|' character, kind of like a regex. +static void print_matching_engine_status_rows(DB_ENV *env, const char *pattern) { + uint64_t num_rows; + env->get_engine_status_num_rows(env, &num_rows); + uint64_t buf_size = num_rows * 128; + const char *row; + char *row_r; + + char *pattern_copy = toku_xstrdup(pattern); + int num_patterns = 1; + for (char *p = pattern_copy; *p != '\0'; p++) { + if (*p == '|') { + *p = '\0'; + num_patterns++; + } + } + + char *XMALLOC_N(buf_size, buf); + int r = env->get_engine_status_text(env, buf, buf_size); + invariant_zero(r); + + for (row = strtok_r(buf, "\n", &row_r); row != nullptr; row = strtok_r(nullptr, "\n", &row_r)) { + const char *p = pattern_copy; + for (int i = 0; i < num_patterns; i++, p += strlen(p) + 1) { + if (strstr(row, p) != nullptr) { + fprintf(stderr, "%s\n", row); + } + } + } + + toku_free(pattern_copy); + toku_free(buf); + fflush(stderr); +} + +static volatile int killed = 0; + +// in a big transaction, insert a bunch of rows. +static void big_test(DB_ENV *env, DB *db, uint64_t max_i) { + if (verbose) + fprintf(stderr, "%u %s\n", toku_os_gettid(), __FUNCTION__); + int r; + + DB_TXN *txn = NULL; + r = env->txn_begin(env, NULL, &txn, 0); + assert(r == 0); + + for (uint64_t i = 0; !killed && i < max_i; i++) { + uint64_t k = htonl(i); + DBT key = { .data = &k, .size = sizeof k }; + DBT val = { .data = &i, .size = sizeof i }; + uint64_t t_start = toku_current_time_microsec(); + r = db->put(db, txn, &key, &val, 0); + assert(r == 0); + uint64_t t_end = toku_current_time_microsec(); + uint64_t t_delta = t_end - t_start; + if (t_delta >= 1000000) { + fprintf(stderr, "%u %s i=%" PRIu64 " %" PRIu64 "\n", toku_os_gettid(), __FUNCTION__, i, t_delta); + if (verbose) + print_matching_engine_status_rows(env, "locktree"); + } + + toku_pthread_yield(); + } + + r = txn->commit(txn, 0); + assert(r == 0); +} + +// insert a row in a single transaction. +static void small_test(DB_ENV *env, DB *db, uint64_t max_i) { + if (verbose) + fprintf(stderr, "%u %s\n", toku_os_gettid(), __FUNCTION__); + int r; + uint64_t k = toku_os_gettid(); // get a unique number + for (uint64_t i = 0; !killed && i < max_i; i++) { + DB_TXN *txn = NULL; + r = env->txn_begin(env, NULL, &txn, 0); + assert(r == 0); + + DBT key = { .data = &k, .size = sizeof k }; + DBT val = { .data = &i, .size = sizeof i }; + uint64_t t_start = toku_current_time_microsec(); + r = db->put(db, txn, &key, &val, 0); + assert(r == 0); + uint64_t t_end = toku_current_time_microsec(); + uint64_t t_delta = t_end - t_start; + if (t_delta >= 1000000) { + fprintf(stderr, "%u %s %" PRIu64 "\n", toku_os_gettid(), __FUNCTION__, t_delta); + assert(0); + } + + r = txn->commit(txn, 0); + assert(r == 0); + + toku_pthread_yield(); + } +} + +struct test_args { + DB_ENV *env; + DB *db; + uint64_t max_i; + void (*test_f)(DB_ENV *env, DB *db, uint64_t max_i); +}; + +static void *test_f(void *args) { + struct test_args *test_args = (struct test_args *) args; + test_args->test_f(test_args->env, test_args->db, test_args->max_i); + return args; +} + +static void run_test(uint64_t max_i, int n_small) { + int r; + + DB_ENV *env = NULL; + r = db_env_create(&env, 0); + assert(r == 0); + env->set_errfile(env, stderr); + r = env->set_cachesize(env, 8, 0, 1); + assert(r == 0); + r = env->set_lk_max_memory(env, 1000000000); + assert(r == 0); + r = env->open(env, TOKU_TEST_FILENAME, DB_INIT_LOCK+DB_INIT_MPOOL+DB_INIT_TXN+DB_INIT_LOG + DB_CREATE + DB_PRIVATE, S_IRWXU+S_IRWXG+S_IRWXO); + assert(r == 0); + + DB *big_db = NULL; + r = db_create(&big_db, env, 0); + assert(r == 0); + + r = big_db->open(big_db, NULL, "big", NULL, DB_BTREE, DB_AUTO_COMMIT+DB_CREATE, S_IRWXU+S_IRWXG+S_IRWXO); + assert(r == 0); + + DB *small_db = NULL; + r = db_create(&small_db, env, 0); + assert(r == 0); + + r = small_db->open(small_db, NULL, "small", NULL, DB_BTREE, DB_AUTO_COMMIT+DB_CREATE, S_IRWXU+S_IRWXG+S_IRWXO); + assert(r == 0); + + struct test_args big_test_args = { + env, big_db, max_i, big_test, + }; + toku_pthread_t big_id; + r = toku_pthread_create(&big_id, NULL, test_f, &big_test_args); + assert(r == 0); + + struct test_args small_test_args[n_small]; + toku_pthread_t small_id[n_small]; + for (int i = 0; i < n_small; i++) { + small_test_args[i] = { env, small_db, max_i, small_test }; + r = toku_pthread_create(&small_id[i], NULL, test_f, &small_test_args[i]); + assert(r == 0); + } + + void *big_ret; + r = toku_pthread_join(big_id, &big_ret); + assert(r == 0); + + killed = 1; + + for (int i = 0; i < n_small; i++) { + void *small_ret; + r = toku_pthread_join(small_id[i], &small_ret); + assert(r == 0); + } + + r = small_db->close(small_db, 0); + assert(r == 0); + + r = big_db->close(big_db, 0); + assert(r == 0); + + r = env->close(env, 0); + assert(r == 0); +} + +int test_main (int argc, char * const argv[]) { + int r; + uint64_t max_i = 10000; + int n_small = 1; + + // parse_args(argc, argv); + for (int i = 1; i < argc; i++) { + if (strcmp(argv[i], "-v") == 0 || strcmp(argv[i], "--verbose") == 0) { + verbose++; + continue; + } + if (strcmp(argv[i], "-q") == 0) { + if (verbose > 0) verbose--; + continue; + } + if (strcmp(argv[i], "--max_i") == 0 && i+1 < argc) { + max_i = atoll(argv[++i]); + continue; + } + if (strcmp(argv[i], "--n_small") == 0 && i+1 < argc) { + n_small = atoi(argv[++i]); + continue; + } + } + + toku_os_recursive_delete(TOKU_TEST_FILENAME); + r = toku_os_mkdir(TOKU_TEST_FILENAME, S_IRWXU+S_IRWXG+S_IRWXO); + assert(r == 0); + + run_test(max_i, n_small); + + return 0; +} diff --git a/storage/tokudb/ft-index/src/tests/recover-child-rollback.cc b/storage/tokudb/ft-index/src/tests/recover-child-rollback.cc new file mode 100644 index 00000000000..00f036cc174 --- /dev/null +++ b/storage/tokudb/ft-index/src/tests/recover-child-rollback.cc @@ -0,0 +1,168 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4: +/* +COPYING CONDITIONS NOTICE: + + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation, and provided that the + following conditions are met: + + * Redistributions of source code must retain this COPYING + CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the + DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the + PATENT MARKING NOTICE (below), and the PATENT RIGHTS + GRANT (below). + + * Redistributions in binary form must reproduce this COPYING + CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the + DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the + PATENT MARKING NOTICE (below), and the PATENT RIGHTS + GRANT (below) in the documentation and/or other materials + provided with the distribution. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + 02110-1301, USA. + +COPYRIGHT NOTICE: + + TokuDB, Tokutek Fractal Tree Indexing Library. + Copyright (C) 2007-2013 Tokutek, Inc. + +DISCLAIMER: + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + +UNIVERSITY PATENT NOTICE: + + The technology is licensed by the Massachusetts Institute of + Technology, Rutgers State University of New Jersey, and the Research + Foundation of State University of New York at Stony Brook under + United States of America Serial No. 11/760379 and to the patents + and/or patent applications resulting from it. + +PATENT MARKING NOTICE: + + This software is covered by US Patent No. 8,185,551. + This software is covered by US Patent No. 8,489,638. + +PATENT RIGHTS GRANT: + + "THIS IMPLEMENTATION" means the copyrightable works distributed by + Tokutek as part of the Fractal Tree project. + + "PATENT CLAIMS" means the claims of patents that are owned or + licensable by Tokutek, both currently or in the future; and that in + the absence of this license would be infringed by THIS + IMPLEMENTATION or by using or running THIS IMPLEMENTATION. + + "PATENT CHALLENGE" shall mean a challenge to the validity, + patentability, enforceability and/or non-infringement of any of the + PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS. + + Tokutek hereby grants to you, for the term and geographical scope of + the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free, + irrevocable (except as stated in this section) patent license to + make, have made, use, offer to sell, sell, import, transfer, and + otherwise run, modify, and propagate the contents of THIS + IMPLEMENTATION, where such license applies only to the PATENT + CLAIMS. This grant does not include claims that would be infringed + only as a consequence of further modifications of THIS + IMPLEMENTATION. If you or your agent or licensee institute or order + or agree to the institution of patent litigation against any entity + (including a cross-claim or counterclaim in a lawsuit) alleging that + THIS IMPLEMENTATION constitutes direct or contributory patent + infringement, or inducement of patent infringement, then any rights + granted to you under this License shall terminate as of the date + such litigation is filed. If you or your agent or exclusive + licensee institute or order or agree to the institution of a PATENT + CHALLENGE, then Tokutek may terminate any rights granted to you + under this License. +*/ + +#ident "Copyright (c) 2007-2013 Tokutek Inc. All rights reserved." +#ident "$Id$" +#include "test.h" + +#include <stdio.h> +#include <stdlib.h> + +#include <toku_pthread.h> +#include <unistd.h> +#include <memory.h> +#include <sys/stat.h> +#include <db.h> + +#include "threaded_stress_test_helpers.h" + +// +// This test is a form of stress that does operations on a single dictionary: +// We create a dictionary bigger than the cachetable (around 4x greater). +// Then, we spawn a bunch of pthreads that do the following: +// - scan dictionary forward with bulk fetch +// - scan dictionary forward slowly +// - scan dictionary backward with bulk fetch +// - scan dictionary backward slowly +// - Grow the dictionary with insertions +// - do random point queries into the dictionary +// With the small cachetable, this should produce quite a bit of churn in reading in and evicting nodes. +// If the test runs to completion without crashing, we consider it a success. It also tests that snapshots +// work correctly by verifying that table scans sum their vals to 0. +// +// This does NOT test: +// - splits and merges +// - multiple DBs +// +// Variables that are interesting to tweak and run: +// - small cachetable +// - number of elements +// + +static void +stress_table(DB_ENV *env, DB **dbp, struct cli_args *cli_args) { + // + // the threads that we want: + // - one (or more) thread(s) constantly updating random values, wrapped in a persistent parent transaction. + + if (verbose) printf("starting creation of pthreads\n"); + const int num_threads = cli_args->num_update_threads; + struct arg myargs[num_threads]; + for (int i = 0; i < num_threads; i++) { + arg_init(&myargs[i], dbp, env, cli_args); + } + + struct update_op_args uoe = get_update_op_args(cli_args, NULL); + // make the guy that updates the db + for (int i = 0; i < cli_args->num_update_threads; ++i) { + myargs[i].operation_extra = &uoe; + myargs[i].operation = update_op; + myargs[i].do_prepare = true; + myargs[i].wrap_in_parent = true; + } + + run_workers(myargs, num_threads, cli_args->num_seconds, true, cli_args); +} + +int +test_main(int argc, char *const argv[]) { + struct cli_args args = get_default_args(); + args.num_seconds = 5; + //args.txn_size = 64; // 100 * 256 is more than enough to spill (4096) byte rollback nodes for parent and child. + //args.val_size = 512; // Large values to overflow a rollback log node fast. + //args.env_args.node_size = 4*1024*1024; // Large nodes to prevent spending much time + //args.env_args.basement_node_size = 128*1024; // Large nodes to prevent spending much time + args.env_args.checkpointing_period = 1; + parse_stress_test_args(argc, argv, &args); + if (args.do_test_and_crash) { + stress_test_main(&args); + } + if (args.do_recover) { + stress_recover(&args); + } + return 0; +} diff --git a/storage/tokudb/ft-index/src/tests/test_4368.cc b/storage/tokudb/ft-index/src/tests/test_4368.cc index 2f578d03bcb..f000efa7813 100644 --- a/storage/tokudb/ft-index/src/tests/test_4368.cc +++ b/storage/tokudb/ft-index/src/tests/test_4368.cc @@ -113,7 +113,8 @@ test_main (int UU(argc), char UU(*const argv[])) { r = db->open(db, NULL, "test.db", 0, DB_BTREE, DB_CREATE, S_IRWXU+S_IRWXG+S_IRWXO); assert(r == 0); // call hot_optimize on an empty db. The empty db should have only a root node, which should invoke the bug - r = db->hot_optimize(db, NULL, NULL, NULL, NULL); assert_zero(r); + uint64_t loops_run; + r = db->hot_optimize(db, NULL, NULL, NULL, NULL, &loops_run); assert_zero(r); r=db->close(db, 0); assert(r==0); r=env->close(env, 0); assert(r==0); diff --git a/storage/tokudb/ft-index/src/tests/test_bad_implicit_promotion.cc b/storage/tokudb/ft-index/src/tests/test_bad_implicit_promotion.cc new file mode 100644 index 00000000000..8fcff7c6132 --- /dev/null +++ b/storage/tokudb/ft-index/src/tests/test_bad_implicit_promotion.cc @@ -0,0 +1,190 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4: +#ident "$Id$" +/* +COPYING CONDITIONS NOTICE: + + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation, and provided that the + following conditions are met: + + * Redistributions of source code must retain this COPYING + CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the + DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the + PATENT MARKING NOTICE (below), and the PATENT RIGHTS + GRANT (below). + + * Redistributions in binary form must reproduce this COPYING + CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the + DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the + PATENT MARKING NOTICE (below), and the PATENT RIGHTS + GRANT (below) in the documentation and/or other materials + provided with the distribution. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + 02110-1301, USA. + +COPYRIGHT NOTICE: + + TokuDB, Tokutek Fractal Tree Indexing Library. + Copyright (C) 2014 Tokutek, Inc. + +DISCLAIMER: + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + +UNIVERSITY PATENT NOTICE: + + The technology is licensed by the Massachusetts Institute of + Technology, Rutgers State University of New Jersey, and the Research + Foundation of State University of New York at Stony Brook under + United States of America Serial No. 11/760379 and to the patents + and/or patent applications resulting from it. + +PATENT MARKING NOTICE: + + This software is covered by US Patent No. 8,185,551. + This software is covered by US Patent No. 8,489,638. + +PATENT RIGHTS GRANT: + + "THIS IMPLEMENTATION" means the copyrightable works distributed by + Tokutek as part of the Fractal Tree project. + + "PATENT CLAIMS" means the claims of patents that are owned or + licensable by Tokutek, both currently or in the future; and that in + the absence of this license would be infringed by THIS + IMPLEMENTATION or by using or running THIS IMPLEMENTATION. + + "PATENT CHALLENGE" shall mean a challenge to the validity, + patentability, enforceability and/or non-infringement of any of the + PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS. + + Tokutek hereby grants to you, for the term and geographical scope of + the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free, + irrevocable (except as stated in this section) patent license to + make, have made, use, offer to sell, sell, import, transfer, and + otherwise run, modify, and propagate the contents of THIS + IMPLEMENTATION, where such license applies only to the PATENT + CLAIMS. This grant does not include claims that would be infringed + only as a consequence of further modifications of THIS + IMPLEMENTATION. If you or your agent or licensee institute or order + or agree to the institution of patent litigation against any entity + (including a cross-claim or counterclaim in a lawsuit) alleging that + THIS IMPLEMENTATION constitutes direct or contributory patent + infringement, or inducement of patent infringement, then any rights + granted to you under this License shall terminate as of the date + such litigation is filed. If you or your agent or exclusive + licensee institute or order or agree to the institution of a PATENT + CHALLENGE, then Tokutek may terminate any rights granted to you + under this License. +*/ + +#ident "Copyright (c) 2014 Tokutek Inc. All rights reserved." + +#include "test.h" + +// Test the following scenario: +// Begin A +// A deletes key K +// A aborts +// Begin B +// B deletes key K-1 +// B deletes key K +// B deletes key K+1 +// B commits +// Begin C +// C queries K, should read K (not the delete!). +// +// An incorrect mvcc implementation would 'implicitly' promote +// A's delete to committed, based on the fact that the oldest +// referenced xid at the time of injection for key k-1 and k+1 +// is greater than A's xid. + +static void test_insert_bad_implicit_promotion(void) { + int r; + + DB_ENV *env; + r = db_env_create(&env, 0); CKERR(r); + r = env->set_cachesize(env, 1, 0, 1); CKERR(r); // 1gb cache so this test fits in memory + r = env->open(env, TOKU_TEST_FILENAME, DB_CREATE+DB_PRIVATE+DB_INIT_MPOOL+DB_INIT_TXN, 0); CKERR(r); + + DB *db; + r = db_create(&db, env, 0); CKERR(r); + r = db->set_pagesize(db, 4096); CKERR(r); + r = db->open(db, NULL, "db", NULL, DB_BTREE, DB_CREATE, 0666); CKERR(r); + + const int val_size = 512; + + DBT key; + DBT val; + char *XMALLOC_N(val_size, val_buf); + memset(val_buf, 'x', val_size); + dbt_init(&val, val_buf, val_size); + + // Insert rows [0, N] + const int N = 1000; + for (int i = 0; i < N; i++) { + int k = toku_htonl(i); + dbt_init(&key, &k, sizeof(k)); + r = db->put(db, NULL, &key, &val, 0); CKERR(r); + } + + int key_500 = toku_htonl(500); + int key_499 = toku_htonl(499); + int key_501 = toku_htonl(501); + // sanity check our keys + r = db->get(db, NULL, dbt_init(&key, &key_500, sizeof(key_500)), &val, 0); CKERR(r); + r = db->get(db, NULL, dbt_init(&key, &key_500, sizeof(key_499)), &val, 0); CKERR(r); + r = db->get(db, NULL, dbt_init(&key, &key_500, sizeof(key_501)), &val, 0); CKERR(r); + + // Abort a delete for key 500 + DB_TXN *txn_A; + r = env->txn_begin(env, NULL, &txn_A, DB_SERIALIZABLE); CKERR(r); + dbt_init(&key, &key_500, sizeof(key_500)); + r = db->del(db, txn_A, &key, DB_DELETE_ANY); CKERR(r); + r = txn_A->abort(txn_A); CKERR(r); + + // Commit two deletes on keys 499 and 501. This should inject + // at least one message in the same buffer that has the delete/abort + // messages for key 500. + DB_TXN *txn_B; + r = env->txn_begin(env, NULL, &txn_B, DB_SERIALIZABLE); CKERR(r); + dbt_init(&key, &key_499, sizeof(key_499)); + r = db->del(db, txn_B, &key, DB_DELETE_ANY); CKERR(r); + dbt_init(&key, &key_501, sizeof(key_501)); + r = db->del(db, txn_B, &key, DB_DELETE_ANY); CKERR(r); + r = txn_B->commit(txn_B, 0); CKERR(r); + + // No transactions are live - so when we create txn C, the oldest + // referenced xid will be txn C. If our implicit promotion logic is + // wrong, we will use txn C's xid to promote the delete on key 500 + // before the abort message hits it, and C's query will return nothing. + DB_TXN *txn_C; + dbt_init(&key, &key_500, sizeof(key_500)); + r = env->txn_begin(env, NULL, &txn_C, DB_TXN_SNAPSHOT); CKERR(r); + r = db->get(db, txn_C, &key, &val, 0); CKERR(r); + r = txn_C->commit(txn_C, 0); CKERR(r); + + toku_free(val_buf); + r = db->close(db, 0); CKERR(r); + r = env->close(env, 0); CKERR(r); +} + +int +test_main(int argc, char *const argv[]) { + parse_args(argc, argv); + + toku_os_recursive_delete(TOKU_TEST_FILENAME); + int r = toku_os_mkdir(TOKU_TEST_FILENAME, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r); + + test_insert_bad_implicit_promotion(); + + return 0; +} diff --git a/storage/tokudb/ft-index/src/tests/test_cursor_interrupt.cc b/storage/tokudb/ft-index/src/tests/test_cursor_interrupt.cc new file mode 100644 index 00000000000..d82fc5131da --- /dev/null +++ b/storage/tokudb/ft-index/src/tests/test_cursor_interrupt.cc @@ -0,0 +1,202 @@ +/* -*- mode: C; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +// vim: expandtab:ts=8:sw=4:softtabstop=4: +/* +COPYING CONDITIONS NOTICE: + + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation, and provided that the + following conditions are met: + + * Redistributions of source code must retain this COPYING + CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the + DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the + PATENT MARKING NOTICE (below), and the PATENT RIGHTS + GRANT (below). + + * Redistributions in binary form must reproduce this COPYING + CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the + DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the + PATENT MARKING NOTICE (below), and the PATENT RIGHTS + GRANT (below) in the documentation and/or other materials + provided with the distribution. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + 02110-1301, USA. + +COPYRIGHT NOTICE: + + TokuDB, Tokutek Fractal Tree Indexing Library. + Copyright (C) 2007-2013 Tokutek, Inc. + +DISCLAIMER: + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + +UNIVERSITY PATENT NOTICE: + + The technology is licensed by the Massachusetts Institute of + Technology, Rutgers State University of New Jersey, and the Research + Foundation of State University of New York at Stony Brook under + United States of America Serial No. 11/760379 and to the patents + and/or patent applications resulting from it. + +PATENT MARKING NOTICE: + + This software is covered by US Patent No. 8,185,551. + This software is covered by US Patent No. 8,489,638. + +PATENT RIGHTS GRANT: + + "THIS IMPLEMENTATION" means the copyrightable works distributed by + Tokutek as part of the Fractal Tree project. + + "PATENT CLAIMS" means the claims of patents that are owned or + licensable by Tokutek, both currently or in the future; and that in + the absence of this license would be infringed by THIS + IMPLEMENTATION or by using or running THIS IMPLEMENTATION. + + "PATENT CHALLENGE" shall mean a challenge to the validity, + patentability, enforceability and/or non-infringement of any of the + PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS. + + Tokutek hereby grants to you, for the term and geographical scope of + the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free, + irrevocable (except as stated in this section) patent license to + make, have made, use, offer to sell, sell, import, transfer, and + otherwise run, modify, and propagate the contents of THIS + IMPLEMENTATION, where such license applies only to the PATENT + CLAIMS. This grant does not include claims that would be infringed + only as a consequence of further modifications of THIS + IMPLEMENTATION. If you or your agent or licensee institute or order + or agree to the institution of patent litigation against any entity + (including a cross-claim or counterclaim in a lawsuit) alleging that + THIS IMPLEMENTATION constitutes direct or contributory patent + infringement, or inducement of patent infringement, then any rights + granted to you under this License shall terminate as of the date + such litigation is filed. If you or your agent or exclusive + licensee institute or order or agree to the institution of a PATENT + CHALLENGE, then Tokutek may terminate any rights granted to you + under this License. +*/ + +#ident "Copyright (c) 2007-2013 Tokutek Inc. All rights reserved." +#include "test.h" + + +#include <stdio.h> + +#include <db.h> + + +int num_interrupts_called; +static bool interrupt(void* extra UU()) { + num_interrupts_called++; + return false; +} + +static bool interrupt_true(void* extra UU()) { + num_interrupts_called++; + return true; +} + + +int +test_main(int argc, char *const argv[]) { + parse_args(argc, argv); + DB_ENV *env; + DB *db; + int r; + + toku_os_recursive_delete(TOKU_TEST_FILENAME); + r=toku_os_mkdir(TOKU_TEST_FILENAME, S_IRWXU+S_IRWXG+S_IRWXO); assert(r==0); + r=db_env_create(&env, 0); assert(r==0); + r=env->open(env, TOKU_TEST_FILENAME, DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN | DB_CREATE | DB_PRIVATE | DB_INIT_LOG, S_IRWXU+S_IRWXG+S_IRWXO); assert(r==0); + + r = db_create(&db, env, 0); + CKERR(r); + r = db->set_readpagesize(db, 1024); + CKERR(r); + r = db->set_pagesize(db, 1024*10); + CKERR(r); + + const char * const fname = "test.change_pagesize"; + r = db->open(db, NULL, fname, "main", DB_BTREE, DB_CREATE, 0666); + CKERR(r); + DB_TXN* txn; + r = env->txn_begin(env, 0, &txn, 0); + CKERR(r); + for (uint64_t i = 0; i < 10000; i++) { + DBT key, val; + uint64_t k = i; + uint64_t v = i; + dbt_init(&key, &k, sizeof k); + dbt_init(&val, &v, sizeof v); + db->put(db, txn, &key, &val, DB_PRELOCKED_WRITE); // adding DB_PRELOCKED_WRITE just to make the test go faster + } + r = txn->commit(txn, 0); + CKERR(r); + + // create a snapshot txn so that when we delete the elements + // we just inserted, that they do not get garbage collected away + DB_TXN* snapshot_txn; + r = env->txn_begin(env, 0, &snapshot_txn, DB_TXN_SNAPSHOT); + CKERR(r); + + DB_TXN* delete_txn; + r = env->txn_begin(env, 0, &delete_txn, DB_TXN_SNAPSHOT); + CKERR(r); + + for (uint64_t i = 0; i < 10000; i++) { + DBT key; + uint64_t k = i; + dbt_init(&key, &k, sizeof k); + db->del(db, delete_txn, &key, DB_PRELOCKED_WRITE | DB_DELETE_ANY); // adding DB_PRELOCKED_WRITE just to make the test go faster + } + r = delete_txn->commit(delete_txn, 0); + CKERR(r); + + // to make more than one basement node in the dictionary's leaf nodes + r = env->txn_checkpoint(env, 0, 0, 0); + CKERR(r); + + // create a txn that should see an empty dictionary + DB_TXN* test_txn; + r = env->txn_begin(env, 0, &test_txn, DB_TXN_SNAPSHOT); + CKERR(r); + DBC* cursor = NULL; + r = db->cursor(db, test_txn, &cursor, 0); + cursor->c_set_check_interrupt_callback(cursor, interrupt, NULL); + DBT key, val; + r = cursor->c_get(cursor, &key, &val, DB_NEXT); + CKERR2(r, DB_NOTFOUND); + assert(num_interrupts_called > 1); + num_interrupts_called = 0; + cursor->c_set_check_interrupt_callback(cursor, interrupt_true, NULL); + r = cursor->c_get(cursor, &key, &val, DB_NEXT); + CKERR2(r, TOKUDB_INTERRUPTED); + assert(num_interrupts_called == 1); + + r = cursor->c_close(cursor); + CKERR(r); + r = test_txn->commit(test_txn, 0); + CKERR(r); + + + r = snapshot_txn->commit(snapshot_txn, 0); + CKERR(r); + + + r = db->close(db, 0); + CKERR(r); + + r = env->close(env, 0); + assert(r == 0); + + return 0; +} diff --git a/storage/tokudb/ft-index/src/tests/test_insert_many_gc.cc b/storage/tokudb/ft-index/src/tests/test_insert_many_gc.cc new file mode 100644 index 00000000000..a1884d22c0c --- /dev/null +++ b/storage/tokudb/ft-index/src/tests/test_insert_many_gc.cc @@ -0,0 +1,157 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4: +#ident "$Id$" +/* +COPYING CONDITIONS NOTICE: + + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation, and provided that the + following conditions are met: + + * Redistributions of source code must retain this COPYING + CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the + DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the + PATENT MARKING NOTICE (below), and the PATENT RIGHTS + GRANT (below). + + * Redistributions in binary form must reproduce this COPYING + CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the + DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the + PATENT MARKING NOTICE (below), and the PATENT RIGHTS + GRANT (below) in the documentation and/or other materials + provided with the distribution. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + 02110-1301, USA. + +COPYRIGHT NOTICE: + + TokuDB, Tokutek Fractal Tree Indexing Library. + Copyright (C) 2014 Tokutek, Inc. + +DISCLAIMER: + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + +UNIVERSITY PATENT NOTICE: + + The technology is licensed by the Massachusetts Institute of + Technology, Rutgers State University of New Jersey, and the Research + Foundation of State University of New York at Stony Brook under + United States of America Serial No. 11/760379 and to the patents + and/or patent applications resulting from it. + +PATENT MARKING NOTICE: + + This software is covered by US Patent No. 8,185,551. + This software is covered by US Patent No. 8,489,638. + +PATENT RIGHTS GRANT: + + "THIS IMPLEMENTATION" means the copyrightable works distributed by + Tokutek as part of the Fractal Tree project. + + "PATENT CLAIMS" means the claims of patents that are owned or + licensable by Tokutek, both currently or in the future; and that in + the absence of this license would be infringed by THIS + IMPLEMENTATION or by using or running THIS IMPLEMENTATION. + + "PATENT CHALLENGE" shall mean a challenge to the validity, + patentability, enforceability and/or non-infringement of any of the + PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS. + + Tokutek hereby grants to you, for the term and geographical scope of + the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free, + irrevocable (except as stated in this section) patent license to + make, have made, use, offer to sell, sell, import, transfer, and + otherwise run, modify, and propagate the contents of THIS + IMPLEMENTATION, where such license applies only to the PATENT + CLAIMS. This grant does not include claims that would be infringed + only as a consequence of further modifications of THIS + IMPLEMENTATION. If you or your agent or licensee institute or order + or agree to the institution of patent litigation against any entity + (including a cross-claim or counterclaim in a lawsuit) alleging that + THIS IMPLEMENTATION constitutes direct or contributory patent + infringement, or inducement of patent infringement, then any rights + granted to you under this License shall terminate as of the date + such litigation is filed. If you or your agent or exclusive + licensee institute or order or agree to the institution of a PATENT + CHALLENGE, then Tokutek may terminate any rights granted to you + under this License. +*/ + +#ident "Copyright (c) 2014 Tokutek Inc. All rights reserved." + +#include "test.h" + + +static void test_insert_many_gc(void) { + int r; + + DB_ENV *env; + r = db_env_create(&env, 0); CKERR(r); + r = env->set_cachesize(env, 1, 0, 1); CKERR(r); // 1gb cache so this test fits in memory + r = env->open(env, TOKU_TEST_FILENAME, DB_CREATE+DB_PRIVATE+DB_INIT_MPOOL+DB_INIT_TXN, 0); CKERR(r); + + DB *db; + r = db_create(&db, env, 0); CKERR(r); + r = db->open(db, NULL, "db", NULL, DB_BTREE, DB_CREATE, 0666); CKERR(r); + + const int val_size = 1 * 1024 * 1024; + + // Begin a snapshot transaction, which should prevent simple garbage collection + // from being effective. Only full garbage collection can prevent many inserts + // into a single leaf node from growing out of control. + DB_TXN *snapshot_txn; + r = env->txn_begin(env, NULL, &snapshot_txn, DB_TXN_SNAPSHOT); CKERR(r); + + DBT key; + int k = 0; + dbt_init(&key, &k, sizeof(k)); + + DBT val; + char *XMALLOC_N(val_size, val_buf); + memset(val_buf, 0, val_size); + dbt_init(&val, val_buf, val_size); + + // Keep overwriting the same row over and over. + const int N = 75; + for (int i = 0; i < N; i++) { + r = db->put(db, NULL, &key, &val, 0); CKERR(r); + } + + // Full garbage collection should have prevented the leaf node + // from having an MVCC stack of size 'N'. At the time of this + // writing, we run full GC on leaf-inject when the leaf is + // 32mb or larger. A good invariant is that the max LE size + // never grew larger than 35mb and that the max commited xr stack + // length never exceeded 35 + const uint64_t le_max_memsize = get_engine_status_val(env, "LE_MAX_MEMSIZE"); + const uint64_t le_max_committed_xr = get_engine_status_val(env, "LE_MAX_COMMITTED_XR"); + invariant(le_max_memsize <= 35 * 1024 * 1024); + invariant(le_max_committed_xr <= 35); + + r = snapshot_txn->commit(snapshot_txn, 0); CKERR(r); + + toku_free(val_buf); + r = db->close(db, 0); CKERR(r); + r = env->close(env, 0); CKERR(r); +} + +int +test_main(int argc, char *const argv[]) { + parse_args(argc, argv); + + toku_os_recursive_delete(TOKU_TEST_FILENAME); + int r = toku_os_mkdir(TOKU_TEST_FILENAME, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r); + + test_insert_many_gc(); + + return 0; +} diff --git a/storage/tokudb/ft-index/src/tests/test_iterate_pending_lock_requests.cc b/storage/tokudb/ft-index/src/tests/test_iterate_pending_lock_requests.cc index fe0667d2b0a..248c346c305 100644 --- a/storage/tokudb/ft-index/src/tests/test_iterate_pending_lock_requests.cc +++ b/storage/tokudb/ft-index/src/tests/test_iterate_pending_lock_requests.cc @@ -148,7 +148,7 @@ int test_main(int UU(argc), char *const UU(argv[])) { r = db_env_create(&env, 0); CKERR(r); r = env->open(env, TOKU_TEST_FILENAME, env_flags, 0755); CKERR(r); - r = env->set_lock_timeout(env, 4000); + r = env->set_lock_timeout(env, 4000, nullptr); r = db_create(&db, env, 0); CKERR(r); r = db->open(db, NULL, dname, NULL, DB_BTREE, DB_CREATE, 0777); CKERR(r); diff --git a/storage/tokudb/ft-index/src/tests/test_kv_gen.h b/storage/tokudb/ft-index/src/tests/test_kv_gen.h index e22bb165e72..f17b6c18641 100644 --- a/storage/tokudb/ft-index/src/tests/test_kv_gen.h +++ b/storage/tokudb/ft-index/src/tests/test_kv_gen.h @@ -180,7 +180,8 @@ static void __attribute__((unused)) do_hot_optimize_on_dbs(DB_ENV *UU(env), DB **dbs, int num_dbs) { for (int i = 0; i < num_dbs; ++i) { - int r = dbs[i]->hot_optimize(dbs[i], NULL, NULL, dummy_progress, NULL); + uint64_t loops_run; + int r = dbs[i]->hot_optimize(dbs[i], NULL, NULL, dummy_progress, NULL, &loops_run); CKERR(r); } } diff --git a/storage/tokudb/ft-index/src/tests/test_lock_timeout_callback.cc b/storage/tokudb/ft-index/src/tests/test_lock_timeout_callback.cc index c56d7968ec5..d4aae0f95d4 100644 --- a/storage/tokudb/ft-index/src/tests/test_lock_timeout_callback.cc +++ b/storage/tokudb/ft-index/src/tests/test_lock_timeout_callback.cc @@ -149,7 +149,7 @@ int test_main(int UU(argc), char *const UU(argv[])) { r = db_env_create(&env, 0); CKERR(r); r = env->open(env, TOKU_TEST_FILENAME, env_flags, 0755); CKERR(r); - r = env->set_lock_timeout(env, 1000); + r = env->set_lock_timeout(env, 1000, nullptr); r = env->set_lock_timeout_callback(env, lock_not_granted); r = db_create(&db, env, 0); CKERR(r); diff --git a/storage/tokudb/ft-index/src/tests/test_stress5.cc b/storage/tokudb/ft-index/src/tests/test_stress5.cc index dc2a26940e1..053da44d66c 100644 --- a/storage/tokudb/ft-index/src/tests/test_stress5.cc +++ b/storage/tokudb/ft-index/src/tests/test_stress5.cc @@ -108,7 +108,7 @@ stress_table(DB_ENV *env, DB **dbp, struct cli_args *cli_args) { // if (verbose) printf("starting creation of pthreads\n"); - const int num_threads = 4 + cli_args->num_update_threads + cli_args->num_ptquery_threads; + const int num_threads = 5 + cli_args->num_update_threads + cli_args->num_ptquery_threads; struct arg myargs[num_threads]; for (int i = 0; i < num_threads; i++) { arg_init(&myargs[i], dbp, env, cli_args); diff --git a/storage/tokudb/ft-index/src/tests/threaded_stress_test_helpers.h b/storage/tokudb/ft-index/src/tests/threaded_stress_test_helpers.h index 46427485129..c173d2d2d63 100644 --- a/storage/tokudb/ft-index/src/tests/threaded_stress_test_helpers.h +++ b/storage/tokudb/ft-index/src/tests/threaded_stress_test_helpers.h @@ -232,6 +232,7 @@ struct arg { bool do_prepare; bool prelock_updates; bool track_thread_performance; + bool wrap_in_parent; }; static void arg_init(struct arg *arg, DB **dbp, DB_ENV *env, struct cli_args *cli_args) { @@ -246,6 +247,7 @@ static void arg_init(struct arg *arg, DB **dbp, DB_ENV *env, struct cli_args *cl arg->do_prepare = false; arg->prelock_updates = false; arg->track_thread_performance = true; + arg->wrap_in_parent = false; } enum operation_type { @@ -568,6 +570,7 @@ static void *worker(void *arg_v) { arg->random_data = &random_data; DB_ENV *env = arg->env; DB_TXN *txn = nullptr; + DB_TXN *ptxn = nullptr; if (verbose) { toku_pthread_t self = toku_pthread_self(); uintptr_t intself = (uintptr_t) self; @@ -575,11 +578,13 @@ static void *worker(void *arg_v) { } if (arg->cli->single_txn) { r = env->txn_begin(env, 0, &txn, arg->txn_flags); CKERR(r); + } else if (arg->wrap_in_parent) { + r = env->txn_begin(env, 0, &ptxn, arg->txn_flags); CKERR(r); } while (run_test) { lock_worker_op(we); if (!arg->cli->single_txn) { - r = env->txn_begin(env, 0, &txn, arg->txn_flags); CKERR(r); + r = env->txn_begin(env, ptxn, &txn, arg->txn_flags); CKERR(r); } r = arg->operation(txn, arg, arg->operation_extra, we->counters); if (r==0 && !arg->cli->single_txn && arg->do_prepare) { @@ -616,6 +621,9 @@ static void *worker(void *arg_v) { if (arg->cli->single_txn) { int flags = get_commit_flags(arg->cli); int chk_r = txn->commit(txn, flags); CKERR(chk_r); + } else if (arg->wrap_in_parent) { + int flags = get_commit_flags(arg->cli); + int chk_r = ptxn->commit(ptxn, flags); CKERR(chk_r); } if (verbose) { toku_pthread_t self = toku_pthread_self(); @@ -1150,7 +1158,9 @@ static void scan_op_worker(void *arg) { static int UU() scan_op_no_check_parallel(DB_TXN *txn, ARG arg, void* operation_extra, void *UU(stats_extra)) { const int num_cores = toku_os_get_number_processors(); const int num_workers = arg->cli->num_DBs < num_cores ? arg->cli->num_DBs : num_cores; - KIBBUTZ kibbutz = toku_kibbutz_create(num_workers); + KIBBUTZ kibbutz = NULL; + int r = toku_kibbutz_create(num_workers, &kibbutz); + assert(r == 0); for (int i = 0; run_test && i < arg->cli->num_DBs; i++) { struct scan_op_worker_info *XCALLOC(info); info->db = arg->dbp[i]; @@ -1621,7 +1631,8 @@ static int UU() hot_op(DB_TXN *UU(txn), ARG UU(arg), void* UU(operation_extra), int r; for (int i = 0; run_test && i < arg->cli->num_DBs; i++) { DB* db = arg->dbp[i]; - r = db->hot_optimize(db, NULL, NULL, hot_progress_callback, nullptr); + uint64_t loops_run; + r = db->hot_optimize(db, NULL, NULL, hot_progress_callback, nullptr, &loops_run); if (run_test) { CKERR(r); } @@ -2102,7 +2113,9 @@ static int fill_tables_default(DB_ENV *env, DB **dbs, struct cli_args *args, boo // be used for internal engine work (ie: flushes, loader threads, etc). const int max_num_workers = (num_cores + 1) / 2; const int num_workers = args->num_DBs < max_num_workers ? args->num_DBs : max_num_workers; - KIBBUTZ kibbutz = toku_kibbutz_create(num_workers); + KIBBUTZ kibbutz = NULL; + int r = toku_kibbutz_create(num_workers, &kibbutz); + assert(r == 0); for (int i = 0; i < args->num_DBs; i++) { struct fill_table_worker_info *XCALLOC(info); info->env = env; diff --git a/storage/tokudb/ft-index/src/tests/update-multiple-with-indexer-array.cc b/storage/tokudb/ft-index/src/tests/update-multiple-with-indexer-array.cc index d4cd4838ce0..684925f9872 100644 --- a/storage/tokudb/ft-index/src/tests/update-multiple-with-indexer-array.cc +++ b/storage/tokudb/ft-index/src/tests/update-multiple-with-indexer-array.cc @@ -92,102 +92,102 @@ PATENT RIGHTS GRANT: // verify that update_multiple where we change the data in row[i] col[j] from x to x+1 +static const int MAX_KEYS = 3; + +static int +array_size(int ndbs) { + return + + 1 + // 0 for old 1 for new + 1 + // ndbs + 2 * MAX_KEYS * (ndbs-1); +} static int get_num_new_keys(int i, int dbnum) { if (dbnum == 0) return 1; if (i & (1<<4)) { dbnum++; // Shift every once in a while. } - return (i + dbnum) % 3; // 0, 1, or 2 + return (i + dbnum) % MAX_KEYS; // 0, 1, or 2 } static int -get_num_keys(int i, int dbnum) { +get_old_num_keys(int i, int dbnum) { if (dbnum == 0) return 1; - return (i + dbnum) % 3; // 0, 1, or 2 + return (i + dbnum) % MAX_KEYS; // 0, 1, or 2 } static int get_total_secondary_rows(int num_primary) { - assert(num_primary % 3 == 0); - return num_primary / 3 * (0 + 1 + 2); -} - -static int -get_total_num_keys(int i, int num_dbs) { - int sum = 0; - for (int db = 1; db < num_dbs; ++db) { - sum += get_num_keys(i, db); - } - return sum; -} - -static int -get_total_num_new_keys(int i, int num_dbs) { - int sum = 0; - for (int db = 1; db < num_dbs; ++db) { - sum += get_num_new_keys(i, db); - } - return sum; + assert(num_primary % MAX_KEYS == 0); + return num_primary / MAX_KEYS * (0 + 1 + 2); } static int -get_key(int i, int dbnum, int which) { +get_old_key(int i, int dbnum, int which) { assert(i < INT16_MAX / 2); assert(which >= 0); - assert(which < get_num_keys(i, dbnum)); assert(which < 4); assert(dbnum < 16); if (dbnum == 0) { assert(which == 0); - return htonl((2*i) << 16); - } else { - return htonl(((2*i+0) << 16) + (dbnum<<8) + (which<<1)); + return htonl(2*i); + } + if (which >= get_old_num_keys(i, dbnum)) { + return htonl(-1); } + return htonl(((2*i+0) << 16) + (dbnum<<8) + (which<<1)); } static int get_new_key(int i, int dbnum, int which) { assert(which >= 0); - assert(which < get_num_new_keys(i, dbnum)); assert(which < 4); assert(dbnum < 16); if (dbnum == 0) { assert(which == 0); - return htonl((2*i+1) << 16); - } else if ((i+dbnum+which) & (1<<5)) { + return htonl(2*i); + } + if (which >= get_num_new_keys(i, dbnum)) { + return htonl(-1); + } + if ((i+dbnum+which) & (1<<5)) { return htonl(((2*i+0) << 16) + (dbnum<<8) + (which<<1)); // no change from original - } else { - return htonl(((2*i+0) << 16) + (dbnum<<8) + (which<<1) + 1); } + return htonl(((2*i+0) << 16) + (dbnum<<8) + (which<<1) + 1); } static void -get_data(int *v, int i, int ndbs) { - int index = 0; +fill_data_2_and_later(int *v, int i, int ndbs) { + int index = 2; for (int dbnum = 1; dbnum < ndbs; dbnum++) { - for (int which = 0; which < get_num_keys(i, dbnum); ++which) { - v[index++] = get_key(i, dbnum, which); + for (int which = 0; which < MAX_KEYS; ++which) { + v[index++] = get_old_key(i, dbnum, which); } } -} - -static void -get_new_data(int *v, int i, int ndbs) { - int index = 0; for (int dbnum = 1; dbnum < ndbs; dbnum++) { - for (int which = 0; which < get_num_new_keys(i, dbnum); ++which) { + for (int which = 0; which < MAX_KEYS; ++which) { v[index++] = get_new_key(i, dbnum, which); - if (which > 0) { - assert(index >= 2); - assert(memcmp(&v[index-2], &v[index-1], sizeof(v[0])) < 0); - } } } } +static void +fill_old_data(int *v, int i, int ndbs) { + v[0] = 0; + v[1] = ndbs; + fill_data_2_and_later(v, i, ndbs); +} + +static void +fill_new_data(int *v, int i, int ndbs) { + v[0] = 1; + v[1] = ndbs; + fill_data_2_and_later(v, i, ndbs); +} + + static int put_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_key_arrays, DBT_ARRAY *dest_val_arrays, const DBT *src_key, const DBT *src_val) { (void)src_val; @@ -201,10 +201,10 @@ put_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_key_arrays, DBT_ARRAY *des int pri_key = *(int *) src_key->data; int* pri_val = (int*) src_val->data; - bool is_new = (ntohl(pri_key) >> 16) % 2 == 1; - int i = (ntohl(pri_key) >> 16) / 2; + bool is_new = pri_val[0] == 1; + int i = (ntohl(pri_key)) / 2; - int num_keys = is_new ? get_num_new_keys(i, dbnum) : get_num_keys(i, dbnum); + int num_keys = is_new ? get_num_new_keys(i, dbnum) : get_old_num_keys(i, dbnum); toku_dbt_array_resize(dest_key_arrays, num_keys); @@ -212,14 +212,16 @@ put_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_key_arrays, DBT_ARRAY *des toku_dbt_array_resize(dest_val_arrays, num_keys); } - int index = 0; - - for (int idb = 1; idb < dbnum; idb++) { - index += is_new ? get_num_new_keys(i, idb) : get_num_keys(i, idb); + int ndbs = pri_val[1]; + int index = 2 + (dbnum-1)*MAX_KEYS; + if (is_new) { + index += MAX_KEYS*(ndbs-1); } + assert(src_val->size % sizeof(int) == 0); assert((int)src_val->size / 4 >= index + num_keys); + for (int which = 0; which < num_keys; which++) { DBT *dest_key = &dest_key_arrays->dbts[which]; DBT *dest_val = NULL; @@ -235,7 +237,7 @@ put_callback(DB *dest_db, DB *src_db, DBT_ARRAY *dest_key_arrays, DBT_ARRAY *des assert(dest_val->flags == DB_DBT_REALLOC); dest_val->size = 0; } - int new_key = is_new ? get_new_key(i, dbnum, which) : get_key(i, dbnum, which); + int new_key = is_new ? get_new_key(i, dbnum, which) : get_old_key(i, dbnum, which); assert(new_key == pri_val[index + which]); *(int*)dest_key->data = new_key; } @@ -265,19 +267,17 @@ do_updates(DB_ENV *env, DB *db[], int ndbs, int nrows) { // update the data i % ndbs col from x to x+1 - int old_k = get_key(i, 0, 0); + int old_k = get_old_key(i, 0, 0); DBT old_key; dbt_init(&old_key, &old_k, sizeof old_k); int new_k = get_new_key(i, 0, 0); DBT new_key; dbt_init(&new_key, &new_k, sizeof new_k); - int num_old_keys = get_total_num_keys(i, ndbs); - int v[num_old_keys]; get_data(v, i, ndbs); + int v[array_size(ndbs)]; fill_old_data(v, i, ndbs); DBT old_data; dbt_init(&old_data, &v[0], sizeof v); - - int num_new_keys = get_total_num_new_keys(i, ndbs); - int newv[num_new_keys]; get_new_data(newv, i, ndbs); + + int newv[array_size(ndbs)]; fill_new_data(newv, i, ndbs); DBT new_data; dbt_init(&new_data, &newv[0], sizeof newv); - + uint32_t flags_array[ndbs]; memset(flags_array, 0, sizeof(flags_array)); r = env->update_multiple(env, db[0], txn, &old_key, &old_data, &new_key, &new_data, ndbs, db, flags_array, narrays, keys, narrays, vals); @@ -298,9 +298,9 @@ populate_primary(DB_ENV *env, DB *db, int ndbs, int nrows) { // populate for (int i = 0; i < nrows; i++) { - int k = get_key(i, 0, 0); - int secondary_keys = get_total_num_keys(i, ndbs); - int v[secondary_keys]; get_data(v, i, ndbs); + int k = get_old_key(i, 0, 0); + int v[array_size(ndbs)]; + fill_old_data(v, i, ndbs); DBT key; dbt_init(&key, &k, sizeof k); DBT val; dbt_init(&val, &v[0], sizeof v); r = db->put(db, txn, &key, &val, 0); assert_zero(r); @@ -317,11 +317,13 @@ populate_secondary(DB_ENV *env, DB *db, int dbnum, int nrows) { // populate for (int i = 0; i < nrows; i++) { - for (int which = 0; which < get_num_keys(i, dbnum); which++) { - int k = get_key(i, dbnum, which); - DBT key; dbt_init(&key, &k, sizeof k); - DBT val; dbt_init(&val, NULL, 0); - r = db->put(db, txn, &key, &val, 0); assert_zero(r); + for (int which = 0; which < MAX_KEYS; which++) { + int k = get_old_key(i, dbnum, which); + if (k >= 0) { + DBT key; dbt_init(&key, &k, sizeof k); + DBT val; dbt_init(&val, NULL, 0); + r = db->put(db, txn, &key, &val, 0); assert_zero(r); + } } } @@ -346,14 +348,14 @@ verify_pri_seq(DB_ENV *env, DB *db, int ndbs, int nrows) { break; int k; int expectk = get_new_key(i, dbnum, 0); - + assert(key.size == sizeof k); memcpy(&k, key.data, key.size); assert(k == expectk); - int num_keys = get_total_num_new_keys(i, ndbs); + int num_keys = array_size(ndbs); assert(val.size == num_keys*sizeof(int)); - int v[num_keys]; get_new_data(v, i, ndbs); + int v[num_keys]; fill_new_data(v, i, ndbs); assert(memcmp(val.data, v, val.size) == 0); } assert(i == nrows); // if (i != nrows) printf("%s:%d %d %d\n", __FUNCTION__, __LINE__, i, nrows); // assert(i == nrows); @@ -398,7 +400,7 @@ verify_sec_seq(DB_ENV *env, DB *db, int dbnum, int nrows) { assert(k == expectk); } - if (k != expectk && which < get_num_keys(i, dbnum) && k == get_key(i, dbnum, which)) { + if (k != expectk && which < get_old_num_keys(i, dbnum) && k == get_old_key(i, dbnum, which)) { // Will fail, never got updated. assert(k == expectk); } @@ -457,9 +459,9 @@ run_test(int ndbs, int nrows) { r = indexer_txn->commit(indexer_txn, 0); assert_zero(r); verify_pri_seq(env, db[0], ndbs, nrows); - for (int dbnum = 1; dbnum < ndbs; dbnum++) + for (int dbnum = 1; dbnum < ndbs; dbnum++) verify_sec_seq(env, db[dbnum], dbnum, nrows); - for (int dbnum = 0; dbnum < ndbs; dbnum++) + for (int dbnum = 0; dbnum < ndbs; dbnum++) r = db[dbnum]->close(db[dbnum], 0); assert_zero(r); r = env->close(env, 0); assert_zero(r); @@ -469,7 +471,7 @@ int test_main(int argc, char * const argv[]) { int r; int ndbs = 10; - int nrows = 3*(1<<5)*4; + int nrows = MAX_KEYS*(1<<5)*4; // parse_args(argc, argv); for (int i = 1; i < argc; i++) { @@ -491,7 +493,7 @@ test_main(int argc, char * const argv[]) { continue; } } - while (nrows % (3*(1<<5)) != 0) { + while (nrows % (MAX_KEYS*(1<<5)) != 0) { nrows++; } //Need at least one to update, and one to index diff --git a/storage/tokudb/ft-index/src/ydb-internal.h b/storage/tokudb/ft-index/src/ydb-internal.h index fe1113d2c0d..f8c10975d5a 100644 --- a/storage/tokudb/ft-index/src/ydb-internal.h +++ b/storage/tokudb/ft-index/src/ydb-internal.h @@ -182,7 +182,12 @@ struct __toku_db_env_internal { int datadir_lockfd; int logdir_lockfd; int tmpdir_lockfd; - uint64_t loader_memory_size; + uint64_t (*get_loader_memory_size_callback)(void); + uint64_t default_lock_timeout_msec; + uint64_t (*get_lock_timeout_callback)(uint64_t default_lock_timeout_msec); + uint64_t default_killed_time_msec; + uint64_t (*get_killed_time_callback)(uint64_t default_killed_time_msec); + int (*killed_callback)(void); }; // test-only environment function for running lock escalation diff --git a/storage/tokudb/ft-index/src/ydb.cc b/storage/tokudb/ft-index/src/ydb.cc index 6a08b89c81a..d1c09f8cc10 100644 --- a/storage/tokudb/ft-index/src/ydb.cc +++ b/storage/tokudb/ft-index/src/ydb.cc @@ -111,6 +111,7 @@ const char *toku_copyright_string = "Copyright (c) 2007-2013 Tokutek Inc. All r #include <sys/types.h> #include <util/status.h> +#include <util/context.h> #include <ft/ft-flusher.h> #include <ft/cachetable.h> @@ -365,8 +366,8 @@ env_fs_init(DB_ENV *env) { static int env_fs_init_minicron(DB_ENV *env) { int r = toku_minicron_setup(&env->i->fs_poller, env->i->fs_poll_time*1000, env_fs_poller, env); - assert(r == 0); - env->i->fs_poller_is_init = true; + if (r == 0) + env->i->fs_poller_is_init = true; return r; } @@ -402,11 +403,12 @@ env_change_fsync_log_period(DB_ENV* env, uint32_t period_ms) { } } -static void +static int env_fsync_log_cron_init(DB_ENV *env) { int r = toku_minicron_setup(&env->i->fsync_log_cron, env->i->fsync_log_period_ms, env_fsync_log_on_minicron, env); - assert(r == 0); - env->i->fsync_log_cron_is_init = true; + if (r == 0) + env->i->fsync_log_cron_is_init = true; + return r; } static void @@ -994,7 +996,11 @@ env_open(DB_ENV * env, const char *home, uint32_t flags, int mode) { if (env->i->cachetable==NULL) { // If we ran recovery then the cachetable should be set here. - toku_cachetable_create(&env->i->cachetable, env->i->cachetable_size, ZERO_LSN, env->i->logger); + r = toku_cachetable_create(&env->i->cachetable, env->i->cachetable_size, ZERO_LSN, env->i->logger); + if (r != 0) { + r = toku_ydb_do_error(env, r, "Cant create a cachetable\n"); + goto cleanup; + } } toku_cachetable_set_env_dir(env->i->cachetable, env->i->dir); @@ -1009,7 +1015,7 @@ env_open(DB_ENV * env, const char *home, uint32_t flags, int mode) { bool create_new_rollback_file = newenv | upgrade_in_progress; r = toku_logger_open_rollback(env->i->logger, env->i->cachetable, create_new_rollback_file); if (r != 0) { - r = toku_ydb_do_error(env, r, "cant open rollback"); + r = toku_ydb_do_error(env, r, "Cant open rollback\n"); goto cleanup; } } @@ -1027,7 +1033,7 @@ env_open(DB_ENV * env, const char *home, uint32_t flags, int mode) { assert_zero(r); r = toku_db_open_iname(env->i->persistent_environment, txn, toku_product_name_strings.environmentdictionary, DB_CREATE, mode); if (r != 0) { - r = toku_ydb_do_error(env, r, "cant open persistent env"); + r = toku_ydb_do_error(env, r, "Cant open persistent env\n"); goto cleanup; } if (newenv) { @@ -1065,20 +1071,29 @@ env_open(DB_ENV * env, const char *home, uint32_t flags, int mode) { assert_zero(r); r = toku_db_open_iname(env->i->directory, txn, toku_product_name_strings.fileopsdirectory, DB_CREATE, mode); if (r != 0) { - r = toku_ydb_do_error(env, r, "cant open %s", toku_product_name_strings.fileopsdirectory); + r = toku_ydb_do_error(env, r, "Cant open %s\n", toku_product_name_strings.fileopsdirectory); goto cleanup; } } if (using_txns) { r = locked_txn_commit(txn, 0); assert_zero(r); + txn = NULL; } cp = toku_cachetable_get_checkpointer(env->i->cachetable); r = toku_checkpoint(cp, env->i->logger, NULL, NULL, NULL, NULL, STARTUP_CHECKPOINT); assert_zero(r); env_fs_poller(env); // get the file system state at startup - env_fs_init_minicron(env); - env_fsync_log_cron_init(env); + r = env_fs_init_minicron(env); + if (r != 0) { + r = toku_ydb_do_error(env, r, "Cant create fs minicron\n"); + goto cleanup; + } + r = env_fsync_log_cron_init(env); + if (r != 0) { + r = toku_ydb_do_error(env, r, "Cant create fsync log minicron\n"); + goto cleanup; + } cleanup: if (r!=0) { if (txn) { @@ -1708,15 +1723,17 @@ env_set_redzone(DB_ENV *env, int redzone) { return r; } -static int -env_get_lock_timeout(DB_ENV *env, uint64_t *lock_timeout_msec) { - *lock_timeout_msec = env->i->ltm.get_lock_wait_time(); +static int env_get_lock_timeout(DB_ENV *env, uint64_t *lock_timeout_msec) { + uint64_t t = env->i->default_lock_timeout_msec; + if (env->i->get_lock_timeout_callback) + t = env->i->get_lock_timeout_callback(t); + *lock_timeout_msec = t; return 0; } -static int -env_set_lock_timeout(DB_ENV *env, uint64_t lock_timeout_msec) { - env->i->ltm.set_lock_wait_time(lock_timeout_msec); +static int env_set_lock_timeout(DB_ENV *env, uint64_t default_lock_timeout_msec, uint64_t (*get_lock_timeout_callback)(uint64_t default_lock_timeout_msec)) { + env->i->default_lock_timeout_msec = default_lock_timeout_msec; + env->i->get_lock_timeout_callback = get_lock_timeout_callback; return 0; } @@ -1900,6 +1917,7 @@ env_get_engine_status_num_rows (DB_ENV * UU(env), uint64_t * num_rowsp) { num_rows += FS_STATUS_NUM_ROWS; num_rows += INDEXER_STATUS_NUM_ROWS; num_rows += LOADER_STATUS_NUM_ROWS; + num_rows += CTX_STATUS_NUM_ROWS; #if 0 // enable when upgrade is supported num_rows += FT_UPGRADE_STATUS_NUM_ROWS; @@ -2086,6 +2104,15 @@ env_get_engine_status (DB_ENV * env, TOKU_ENGINE_STATUS_ROW engstat, uint64_t ma } } } + { + struct context_status ctxstatus; + toku_context_get_status(&ctxstatus); + for (int i = 0; i < CTX_STATUS_NUM_ROWS && row < maxrows; i++) { + if (ctxstatus.status[i].include & include_flags) { + engstat[row++] = ctxstatus.status[i]; + } + } + } #if 0 // enable when upgrade is supported { @@ -2416,12 +2443,21 @@ env_iterate_live_transactions(DB_ENV *env, return toku_txn_manager_iter_over_live_root_txns(txn_manager, iter_txns_callback, &e); } -static void env_set_loader_memory_size(DB_ENV *env, uint64_t loader_memory_size) { - env->i->loader_memory_size = loader_memory_size; +static void env_set_loader_memory_size(DB_ENV *env, uint64_t (*get_loader_memory_size_callback)(void)) { + env->i->get_loader_memory_size_callback = get_loader_memory_size_callback; } static uint64_t env_get_loader_memory_size(DB_ENV *env) { - return env->i->loader_memory_size; + uint64_t memory_size = 0; + if (env->i->get_loader_memory_size_callback) + memory_size = env->i->get_loader_memory_size_callback(); + return memory_size; +} + +static void env_set_killed_callback(DB_ENV *env, uint64_t default_killed_time_msec, uint64_t (*get_killed_time_callback)(uint64_t default_killed_time_msec), int (*killed_callback)(void)) { + env->i->default_killed_time_msec = default_killed_time_msec; + env->i->get_killed_time_callback = get_killed_time_callback; + env->i->killed_callback = killed_callback; } static int @@ -2499,6 +2535,7 @@ toku_env_create(DB_ENV ** envp, uint32_t flags) { USENV(change_fsync_log_period); USENV(set_loader_memory_size); USENV(get_loader_memory_size); + USENV(set_killed_callback); #undef USENV // unlocked methods diff --git a/storage/tokudb/ft-index/src/ydb_cursor.cc b/storage/tokudb/ft-index/src/ydb_cursor.cc index cf7fb1d958b..ae3b6e08820 100644 --- a/storage/tokudb/ft-index/src/ydb_cursor.cc +++ b/storage/tokudb/ft-index/src/ydb_cursor.cc @@ -212,8 +212,7 @@ query_context_base_init(QUERY_CONTEXT_BASE context, DBC *c, uint32_t flag, bool } context->do_locking = (context->db->i->lt != nullptr && !(lock_flags & (DB_PRELOCKED | DB_PRELOCKED_WRITE))); context->r_user_callback = 0; - uint64_t lock_wait_time = context->txn ? context->txn->mgrp->i->ltm.get_lock_wait_time() : 0; - context->request.create(lock_wait_time); + context->request.create(); } static toku::lock_request::type @@ -740,6 +739,11 @@ c_remove_restriction(DBC *dbc) { toku_ft_cursor_remove_restriction(dbc_struct_i(dbc)->c); } +static void +c_set_check_interrupt_callback(DBC* dbc, bool (*interrupt_callback)(void*), void *extra) { + toku_ft_cursor_set_check_interrupt_cb(dbc_struct_i(dbc)->c, interrupt_callback, extra); +} + int toku_c_get(DBC* c, DBT* key, DBT* val, uint32_t flag) { //This function exists for legacy (test compatibility) purposes/parity with bdb. @@ -833,6 +837,7 @@ toku_db_cursor_internal(DB * db, DB_TXN * txn, DBC ** c, uint32_t flags, int is_ SCRS(c_getf_set_range_reverse); SCRS(c_set_bounds); SCRS(c_remove_restriction); + SCRS(c_set_check_interrupt_callback); #undef SCRS result->c_get = toku_c_get; diff --git a/storage/tokudb/ft-index/src/ydb_db.cc b/storage/tokudb/ft-index/src/ydb_db.cc index f0159c44d1f..c9fa99d54de 100644 --- a/storage/tokudb/ft-index/src/ydb_db.cc +++ b/storage/tokudb/ft-index/src/ydb_db.cc @@ -107,6 +107,7 @@ PATENT RIGHTS GRANT: #include "indexer.h" #include <portability/toku_atomic.h> #include <util/status.h> +#include <ft/le-cursor.h> static YDB_DB_LAYER_STATUS_S ydb_db_layer_status; #ifdef STATUS_VALUE @@ -688,6 +689,29 @@ toku_db_get_compression_method(DB *db, enum toku_compression_method *compression return 0; } +static int +toku_db_change_fanout(DB *db, unsigned int fanout) { + HANDLE_PANICKED_DB(db); + if (!db_opened(db)) return EINVAL; + toku_ft_handle_set_fanout(db->i->ft_handle, fanout); + return 0; +} + +static int +toku_db_set_fanout(DB *db, unsigned int fanout) { + HANDLE_PANICKED_DB(db); + if (db_opened(db)) return EINVAL; + toku_ft_handle_set_fanout(db->i->ft_handle, fanout); + return 0; +} + +static int +toku_db_get_fanout(DB *db, unsigned int *fanout) { + HANDLE_PANICKED_DB(db); + toku_ft_handle_get_fanout(db->i->ft_handle, fanout); + return 0; +} + static int toku_db_get_fractal_tree_info64(DB *db, uint64_t *num_blocks_allocated, uint64_t *num_blocks_in_use, uint64_t *size_allocated, uint64_t *size_in_use) { HANDLE_PANICKED_DB(db); @@ -898,13 +922,13 @@ toku_db_optimize(DB *db) { static int toku_db_hot_optimize(DB *db, DBT* left, DBT* right, int (*progress_callback)(void *extra, float progress), - void *progress_extra) + void *progress_extra, uint64_t* loops_run) { HANDLE_PANICKED_DB(db); int r = 0; r = toku_ft_hot_optimize(db->i->ft_handle, left, right, progress_callback, - progress_extra); + progress_extra, loops_run); return r; } @@ -919,6 +943,55 @@ locked_db_optimize(DB *db) { return r; } + +struct last_key_extra { + YDB_CALLBACK_FUNCTION func; + void* extra; +}; + +static int +db_get_last_key_callback(ITEMLEN keylen, bytevec key, ITEMLEN vallen UU(), bytevec val UU(), void *extra, bool lock_only) { + if (!lock_only) { + DBT keydbt; + toku_fill_dbt(&keydbt, key, keylen); + struct last_key_extra * CAST_FROM_VOIDP(info, extra); + info->func(&keydbt, NULL, info->extra); + } + return 0; +} + +static int +toku_db_get_last_key(DB * db, DB_TXN *txn, YDB_CALLBACK_FUNCTION func, void* extra) { + int r; + LE_CURSOR cursor = nullptr; + struct last_key_extra last_extra = { .func = func, .extra = extra }; + + r = toku_le_cursor_create(&cursor, db->i->ft_handle, db_txn_struct_i(txn)->tokutxn); + if (r != 0) { goto cleanup; } + + // Goes in reverse order. First key returned is last in dictionary. + r = toku_le_cursor_next(cursor, db_get_last_key_callback, &last_extra); + if (r != 0) { goto cleanup; } + +cleanup: + if (cursor) { + toku_le_cursor_close(cursor); + } + return r; +} + +static int +autotxn_db_get_last_key(DB* db, YDB_CALLBACK_FUNCTION func, void* extra) { + bool changed; int r; + DB_TXN *txn = nullptr; + // Cursors inside require transactions, but this is _not_ a transactional function. + // Create transaction in a wrapper and then later close it. + r = toku_db_construct_autotxn(db, &txn, &changed, false); + if (r!=0) return r; + r = toku_db_get_last_key(db, txn, func, extra); + return toku_db_destruct_autotxn(txn, r, changed); +} + static int toku_db_get_fragmentation(DB * db, TOKU_DB_FRAGMENTATION report) { HANDLE_PANICKED_DB(db); @@ -1034,6 +1107,9 @@ toku_db_create(DB ** db, DB_ENV * env, uint32_t flags) { USDB(set_compression_method); USDB(get_compression_method); USDB(change_compression_method); + USDB(set_fanout); + USDB(get_fanout); + USDB(change_fanout); USDB(set_flags); USDB(get_flags); USDB(fd); @@ -1061,6 +1137,7 @@ toku_db_create(DB ** db, DB_ENV * env, uint32_t flags) { result->update = autotxn_db_update; result->update_broadcast = autotxn_db_update_broadcast; result->change_descriptor = autotxn_db_change_descriptor; + result->get_last_key = autotxn_db_get_last_key; // unlocked methods result->get = autotxn_db_get; diff --git a/storage/tokudb/ft-index/src/ydb_env_func.cc b/storage/tokudb/ft-index/src/ydb_env_func.cc index febdb5eb309..5247e699a23 100644 --- a/storage/tokudb/ft-index/src/ydb_env_func.cc +++ b/storage/tokudb/ft-index/src/ydb_env_func.cc @@ -117,6 +117,10 @@ void db_env_set_direct_io (bool direct_io_on) { toku_ft_set_direct_io(direct_io_on); } +void db_env_set_compress_buffers_before_eviction (bool compress_buffers) { + toku_ft_set_compress_buffers_before_eviction(compress_buffers); +} + void db_env_set_func_fsync (int (*fsync_function)(int)) { toku_set_func_fsync(fsync_function); } diff --git a/storage/tokudb/ft-index/src/ydb_row_lock.cc b/storage/tokudb/ft-index/src/ydb_row_lock.cc index c355cf1d39e..f7cdcbb563c 100644 --- a/storage/tokudb/ft-index/src/ydb_row_lock.cc +++ b/storage/tokudb/ft-index/src/ydb_row_lock.cc @@ -231,10 +231,8 @@ void toku_db_txn_escalate_callback(TXNID txnid, const toku::locktree *lt, const // Return when the range lock is acquired or the default lock tree timeout has expired. int toku_db_get_range_lock(DB *db, DB_TXN *txn, const DBT *left_key, const DBT *right_key, toku::lock_request::type lock_type) { - uint64_t wait_time = txn->mgrp->i->ltm.get_lock_wait_time(); toku::lock_request request; - request.create(wait_time); - + request.create(); int r = toku_db_start_range_lock(db, txn, left_key, right_key, lock_type, &request); if (r == DB_LOCK_NOTGRANTED) { r = toku_db_wait_range_lock(db, txn, &request); @@ -249,7 +247,7 @@ int toku_db_start_range_lock(DB *db, DB_TXN *txn, const DBT *left_key, const DBT toku::lock_request::type lock_type, toku::lock_request *request) { DB_TXN *txn_anc = txn_oldest_ancester(txn); TXNID txn_anc_id = txn_anc->id64(txn_anc); - request->set(db->i->lt, txn_anc_id, left_key, right_key, lock_type); + request->set(db->i->lt, txn_anc_id, left_key, right_key, lock_type, toku_is_big_txn(txn_anc)); const int r = request->start(); if (r == 0) { @@ -270,8 +268,14 @@ int toku_db_wait_range_lock(DB *db, DB_TXN *txn, toku::lock_request *request) { DB_TXN *txn_anc = txn_oldest_ancester(txn); const DBT *left_key = request->get_left_key(); const DBT *right_key = request->get_right_key(); - - const int r = request->wait(); + DB_ENV *env = db->dbenv; + uint64_t wait_time_msec = env->i->default_lock_timeout_msec; + if (env->i->get_lock_timeout_callback) + wait_time_msec = env->i->get_lock_timeout_callback(wait_time_msec); + uint64_t killed_time_msec = env->i->default_killed_time_msec; + if (env->i->get_killed_time_callback) + killed_time_msec = env->i->get_killed_time_callback(killed_time_msec); + const int r = request->wait(wait_time_msec, killed_time_msec, env->i->killed_callback); if (r == 0) { db_txn_note_row_lock(db, txn_anc, left_key, right_key); } else if (r == DB_LOCK_NOTGRANTED) { @@ -296,11 +300,9 @@ void toku_db_grab_write_lock (DB *db, DBT *key, TOKUTXN tokutxn) { TXNID txn_anc_id = txn_anc->id64(txn_anc); // This lock request must succeed, so we do not want to wait - const uint64_t lock_wait_time = 0; toku::lock_request request; - - request.create(lock_wait_time); - request.set(db->i->lt, txn_anc_id, key, key, toku::lock_request::type::WRITE); + request.create(); + request.set(db->i->lt, txn_anc_id, key, key, toku::lock_request::type::WRITE, toku_is_big_txn(txn_anc)); int r = request.start(); invariant_zero(r); db_txn_note_row_lock(db, txn_anc, key, key); diff --git a/storage/tokudb/ft-index/src/ydb_txn.cc b/storage/tokudb/ft-index/src/ydb_txn.cc index f8453753717..6f1105412d7 100644 --- a/storage/tokudb/ft-index/src/ydb_txn.cc +++ b/storage/tokudb/ft-index/src/ydb_txn.cc @@ -323,7 +323,7 @@ static int locked_txn_commit_with_progress(DB_TXN *txn, uint32_t flags, if (!toku_txn_is_read_only(tokutxn)) { // A readonly transaction does no logging, and therefore does not need the MO lock. holds_mo_lock = true; - if (toku_txn_has_spilled_rollback(tokutxn)) { + if (toku_is_big_tokutxn(tokutxn)) { low_priority = true; toku_low_priority_multi_operation_client_lock(); } else { @@ -351,7 +351,7 @@ static int locked_txn_abort_with_progress(DB_TXN *txn, if (!toku_txn_is_read_only(tokutxn)) { // A readonly transaction does no logging, and therefore does not need the MO lock. holds_mo_lock = true; - if (toku_txn_has_spilled_rollback(tokutxn)) { + if (toku_is_big_tokutxn(tokutxn)) { low_priority = true; toku_low_priority_multi_operation_client_lock(); } else { @@ -602,3 +602,11 @@ void toku_keep_prepared_txn_callback (DB_ENV *env, TOKUTXN tokutxn) { void toku_increase_last_xid(DB_ENV *env, uint64_t increment) { toku_txn_manager_increase_last_xid(toku_logger_get_txn_manager(env->i->logger), increment); } + +bool toku_is_big_txn(DB_TXN *txn) { + return toku_is_big_tokutxn(db_txn_struct_i(txn)->tokutxn); +} + +bool toku_is_big_tokutxn(TOKUTXN tokutxn) { + return toku_txn_has_spilled_rollback(tokutxn); +} diff --git a/storage/tokudb/ft-index/src/ydb_txn.h b/storage/tokudb/ft-index/src/ydb_txn.h index 9992d267737..454b6578e9f 100644 --- a/storage/tokudb/ft-index/src/ydb_txn.h +++ b/storage/tokudb/ft-index/src/ydb_txn.h @@ -107,6 +107,9 @@ int locked_txn_abort(DB_TXN *txn); void toku_keep_prepared_txn_callback(DB_ENV *env, TOKUTXN tokutxn); +bool toku_is_big_txn(DB_TXN *txn); +bool toku_is_big_tokutxn(TOKUTXN tokutxn); + // Test-only function extern "C" void toku_increase_last_xid(DB_ENV *env, uint64_t increment) __attribute__((__visibility__("default"))); |