Merge tag 'tokudb-ft-index/tokudb-7.5.7' into 5.5

14 files changed, 121 insertions, 53 deletions

diff --git a/storage/tokudb/ft-index/buildheader/make_tdb.cc b/storage/tokudb/ft-index/buildheader/make_tdb.cc
index 3f9a721d9aa..53706649231 100644
--- a/storage/tokudb/ft-index/buildheader/make_tdb.cc
+++ b/storage/tokudb/ft-index/buildheader/make_tdb.cc
@@ -587,6 +587,7 @@ static void print_db_txn_struct (void) {
         "uint64_t (*get_client_id)(DB_TXN *)",
         "bool (*is_prepared)(DB_TXN *)",
         "DB_TXN *(*get_child)(DB_TXN *)",
+        "uint64_t (*get_start_time)(DB_TXN *)",
 	NULL};
     sort_and_dump_fields("db_txn", false, extra);
 }
@@ -786,7 +787,7 @@ int main (int argc, char *const argv[] __attribute__((__unused__))) {
 
     printf("typedef void (*lock_timeout_callback)(DB *db, uint64_t requesting_txnid, const DBT *left_key, const DBT *right_key, uint64_t blocking_txnid);\n");
     printf("typedef int (*iterate_row_locks_callback)(DB **db, DBT *left_key, DBT *right_key, void *extra);\n");
-    printf("typedef int (*iterate_transactions_callback)(uint64_t txnid, uint64_t client_id, iterate_row_locks_callback cb, void *locks_extra, void *extra);\n");
+    printf("typedef int (*iterate_transactions_callback)(DB_TXN *dbtxn, iterate_row_locks_callback cb, void *locks_extra, void *extra);\n");
     printf("typedef int (*iterate_requests_callback)(DB *db, uint64_t requesting_txnid, const DBT *left_key, const DBT *right_key, uint64_t blocking_txnid, uint64_t start_time, void *extra);\n");
     print_db_env_struct();
     print_db_key_range_struct();
diff --git a/storage/tokudb/ft-index/ft/ft-verify.cc b/storage/tokudb/ft-index/ft/ft-verify.cc
index d9606f37604..0a85136816f 100644
--- a/storage/tokudb/ft-index/ft/ft-verify.cc
+++ b/storage/tokudb/ft-index/ft/ft-verify.cc
@@ -160,10 +160,14 @@ get_ith_key_dbt (BASEMENTNODE bn, int i) {
 
 #define VERIFY_ASSERTION(predicate, i, string) ({                                                                              \
     if(!(predicate)) {                                                                                                         \
-        (void) verbose; \
-        if (true) {                                                                                                         \
-            fprintf(stderr, "%s:%d: Looking at child %d of block %" PRId64 ": %s\n", __FILE__, __LINE__, i, blocknum.b, string); \
-        }                                                                                                                      \
+        fprintf(stderr, "%s:%d: Looking at child %d of block %" PRId64 ": %s\n", __FILE__, __LINE__, i, blocknum.b, string);   \
+        result = TOKUDB_NEEDS_REPAIR;                                                                                          \
+        if (!keep_going_on_failure) goto done;                                                                                 \
+    }})
+
+#define VERIFY_ASSERTION_BASEMENT(predicate, bn, entry, string) ({           \
+    if(!(predicate)) {                                                                                                         \
+        fprintf(stderr, "%s:%d: Looking at block %" PRId64 " bn %d entry %d: %s\n", __FILE__, __LINE__, blocknum.b, bn, entry, string); \
         result = TOKUDB_NEEDS_REPAIR;                                                                                          \
         if (!keep_going_on_failure) goto done;                                                                                 \
     }})
@@ -201,7 +205,6 @@ struct verify_message_tree_extra {
 int verify_message_tree(const int32_t &offset, const uint32_t UU(idx), struct verify_message_tree_extra *const e) __attribute__((nonnull(3)));
 int verify_message_tree(const int32_t &offset, const uint32_t UU(idx), struct verify_message_tree_extra *const e)
 {
-    int verbose = e->verbose;
     BLOCKNUM blocknum = e->blocknum;
     int keep_going_on_failure = e->keep_going_on_failure;
     int result = 0;
@@ -236,7 +239,6 @@ int error_on_iter(const int32_t &UU(offset), const uint32_t UU(idx), void *UU(e)
 int verify_marked_messages(const int32_t &offset, const uint32_t UU(idx), struct verify_message_tree_extra *const e) __attribute__((nonnull(3)));
 int verify_marked_messages(const int32_t &offset, const uint32_t UU(idx), struct verify_message_tree_extra *const e)
 {
-    int verbose = e->verbose;
     BLOCKNUM blocknum = e->blocknum;
     int keep_going_on_failure = e->keep_going_on_failure;
     int result = 0;
@@ -462,16 +464,16 @@ toku_verify_ftnode_internal(FT_HANDLE ft_handle,
                 DBT kdbt = get_ith_key_dbt(bn, j);
                 if (curr_less_pivot) {
                     int compare = compare_pairs(ft_handle, curr_less_pivot, &kdbt);
-                    VERIFY_ASSERTION(compare < 0, j, "The leafentry is >= the lower-bound pivot");
+                    VERIFY_ASSERTION_BASEMENT(compare < 0, i, j, "The leafentry is >= the lower-bound pivot");
                 }
                 if (curr_geq_pivot) {
                     int compare = compare_pairs(ft_handle, curr_geq_pivot, &kdbt);
-                    VERIFY_ASSERTION(compare >= 0, j, "The leafentry is < the upper-bound pivot");
+                    VERIFY_ASSERTION_BASEMENT(compare >= 0, i, j, "The leafentry is < the upper-bound pivot");
                 }
                 if (0 < j) {
                     DBT prev_key_dbt = get_ith_key_dbt(bn, j-1);
                     int compare = compare_pairs(ft_handle, &prev_key_dbt, &kdbt);
-                    VERIFY_ASSERTION(compare < 0, j, "Adjacent leafentries are out of order");
+                    VERIFY_ASSERTION_BASEMENT(compare < 0, i, j, "Adjacent leafentries are out of order");
                 }
             }
         }
diff --git a/storage/tokudb/ft-index/ft/logger/recover.cc b/storage/tokudb/ft-index/ft/logger/recover.cc
index cae7397651d..680485201da 100644
--- a/storage/tokudb/ft-index/ft/logger/recover.cc
+++ b/storage/tokudb/ft-index/ft/logger/recover.cc
@@ -111,7 +111,8 @@ int tokuft_recovery_trace = 0;                    // turn on recovery tracing, d
 #endif
 
 // time in seconds between recovery progress reports
-#define TOKUDB_RECOVERY_PROGRESS_TIME 15
+#define TOKUFT_RECOVERY_PROGRESS_TIME 15
+time_t tokuft_recovery_progress_time = TOKUFT_RECOVERY_PROGRESS_TIME;
 
 enum ss {
     BACKWARD_NEWER_CHECKPOINT_END = 1,
@@ -325,14 +326,12 @@ static int recover_env_init (RECOVER_ENV renv,
 }
 
 static void recover_env_cleanup (RECOVER_ENV renv) {
-    int r;
-
     invariant_zero(renv->fmap.filenums->size());
     file_map_destroy(&renv->fmap);
 
     if (renv->destroy_logger_at_end) {
         toku_logger_close_rollback(renv->logger);
-        r = toku_logger_close(&renv->logger);
+        int r = toku_logger_close(&renv->logger);
         assert(r == 0);
     } else {
         toku_logger_write_log_files(renv->logger, true);
@@ -749,6 +748,36 @@ static int toku_recover_backward_xbegin (struct logtype_xbegin *UU(l), RECOVER_E
     return 0;
 }
 
+struct toku_txn_progress_extra {
+    time_t tlast;
+    LSN lsn;
+    const char *type;
+    TXNID_PAIR xid;
+    uint64_t last_total;
+};
+
+static void toku_recover_txn_progress(TOKU_TXN_PROGRESS txn_progress, void *extra) {
+    toku_txn_progress_extra *txn_progress_extra = static_cast<toku_txn_progress_extra *>(extra);
+    if (txn_progress_extra->last_total == 0)
+        txn_progress_extra->last_total = txn_progress->entries_total;
+    else
+        assert(txn_progress_extra->last_total == txn_progress->entries_total);
+    time_t tnow = time(NULL);
+    if (tnow - txn_progress_extra->tlast >= tokuft_recovery_progress_time) {
+        txn_progress_extra->tlast = tnow;
+        fprintf(stderr, "%.24s TokuFT ", ctime(&tnow));
+        if (txn_progress_extra->lsn.lsn != 0)
+            fprintf(stderr, "lsn %" PRIu64 " ", txn_progress_extra->lsn.lsn);
+        fprintf(stderr, "%s xid %" PRIu64 ":%" PRIu64 " ",
+                txn_progress_extra->type, txn_progress_extra->xid.parent_id64, txn_progress_extra->xid.child_id64);
+        fprintf(stderr, "%" PRIu64 "/%" PRIu64 " ",
+                txn_progress->entries_processed, txn_progress->entries_total);
+        if (txn_progress->entries_total > 0)
+            fprintf(stderr, "%.0f%% ", ((double) txn_progress->entries_processed / (double) txn_progress->entries_total) * 100.0);
+        fprintf(stderr, "\n");
+    }
+}
+
 static int toku_recover_xcommit (struct logtype_xcommit *l, RECOVER_ENV renv) {
     // find the transaction by transaction id
     TOKUTXN txn = NULL;
@@ -756,8 +785,8 @@ static int toku_recover_xcommit (struct logtype_xcommit *l, RECOVER_ENV renv) {
     assert(txn!=NULL);
 
     // commit the transaction
-    int r = toku_txn_commit_with_lsn(txn, true, l->lsn,
-                                 NULL, NULL);
+    toku_txn_progress_extra extra = { time(NULL), l->lsn, "commit", l->xid };
+    int r = toku_txn_commit_with_lsn(txn, true, l->lsn, toku_recover_txn_progress, &extra);
     assert(r == 0);
 
     // close the transaction
@@ -799,7 +828,8 @@ static int toku_recover_xabort (struct logtype_xabort *l, RECOVER_ENV renv) {
     assert(txn!=NULL);
 
     // abort the transaction
-    r = toku_txn_abort_with_lsn(txn, l->lsn, NULL, NULL);
+    toku_txn_progress_extra extra = { time(NULL), l->lsn, "abort", l->xid };
+    r = toku_txn_abort_with_lsn(txn, l->lsn, toku_recover_txn_progress, &extra);
     assert(r == 0);
 
     // close the transaction
@@ -1301,7 +1331,6 @@ static int is_txn_unprepared(TOKUTXN txn, void* extra) {
     return 0;
 }
 
-
 static int find_an_unprepared_txn (RECOVER_ENV renv, TOKUTXN *txnp) {
     TOKUTXN txn = nullptr;
     int r = toku_txn_manager_iter_over_live_root_txns(
@@ -1326,6 +1355,7 @@ static int call_prepare_txn_callback_iter(TOKUTXN txn, void* extra) {
 }
 
 static void recover_abort_live_txn(TOKUTXN txn) {
+    fprintf(stderr, "%s %" PRIu64 "\n", __FUNCTION__, txn->txnid.parent_id64);
     // recursively abort all children first
     if (txn->child != NULL) {
         recover_abort_live_txn(txn->child);
@@ -1333,7 +1363,8 @@ static void recover_abort_live_txn(TOKUTXN txn) {
     // sanity check that the recursive call successfully NULLs out txn->child
     invariant(txn->child == NULL);
     // abort the transaction
-    int r = toku_txn_abort_txn(txn, NULL, NULL);
+    toku_txn_progress_extra extra = { time(NULL), ZERO_LSN, "abort live", txn->txnid };
+    int r = toku_txn_abort_txn(txn, toku_recover_txn_progress, &extra);
     assert(r == 0);
     
     // close the transaction
@@ -1451,9 +1482,10 @@ static int do_recovery(RECOVER_ENV renv, const char *env_dir, const char *log_di
         // trace progress
         if ((i % 1000) == 0) {
             tnow = time(NULL);
-            if (tnow - tlast >= TOKUDB_RECOVERY_PROGRESS_TIME) {
+            if (tnow - tlast >= tokuft_recovery_progress_time) {
                 thislsn = toku_log_entry_get_lsn(le);
-                fprintf(stderr, "%.24s TokuFT recovery scanning backward from %" PRIu64 " at %" PRIu64 " (%s)\n", ctime(&tnow), lastlsn.lsn, thislsn.lsn, recover_state(renv));
+                fprintf(stderr, "%.24s TokuFT recovery scanning backward from %" PRIu64 " at %" PRIu64 " (%s)\n",
+                        ctime(&tnow), lastlsn.lsn, thislsn.lsn, recover_state(renv));
                 tlast = tnow;
             }
         }
@@ -1482,16 +1514,18 @@ static int do_recovery(RECOVER_ENV renv, const char *env_dir, const char *log_di
     assert(le);
     thislsn = toku_log_entry_get_lsn(le);
     tnow = time(NULL);
-    fprintf(stderr, "%.24s TokuFT recovery starts scanning forward to %" PRIu64 " from %" PRIu64 " left %" PRIu64 " (%s)\n", ctime(&tnow), lastlsn.lsn, thislsn.lsn, lastlsn.lsn - thislsn.lsn, recover_state(renv));
+    fprintf(stderr, "%.24s TokuFT recovery starts scanning forward to %" PRIu64 " from %" PRIu64 " left %" PRIu64 " (%s)\n",
+            ctime(&tnow), lastlsn.lsn, thislsn.lsn, lastlsn.lsn - thislsn.lsn, recover_state(renv));
 
     for (unsigned i=0; 1; i++) {
 
         // trace progress
         if ((i % 1000) == 0) {
             tnow = time(NULL);
-            if (tnow - tlast >= TOKUDB_RECOVERY_PROGRESS_TIME) {
+            if (tnow - tlast >= tokuft_recovery_progress_time) {
                 thislsn = toku_log_entry_get_lsn(le);
-                fprintf(stderr, "%.24s TokuFT recovery scanning forward to %" PRIu64 " at %" PRIu64 " left %" PRIu64 " (%s)\n", ctime(&tnow), lastlsn.lsn, thislsn.lsn, lastlsn.lsn - thislsn.lsn, recover_state(renv));
+                fprintf(stderr, "%.24s TokuFT recovery scanning forward to %" PRIu64 " at %" PRIu64 " left %" PRIu64 " (%s)\n",
+                        ctime(&tnow), lastlsn.lsn, thislsn.lsn, lastlsn.lsn - thislsn.lsn, recover_state(renv));
                 tlast = tnow;
             }
         }
diff --git a/storage/tokudb/ft-index/ft/txn/txn.cc b/storage/tokudb/ft-index/ft/txn/txn.cc
index 18d5a6b67dd..922c955a6b5 100644
--- a/storage/tokudb/ft-index/ft/txn/txn.cc
+++ b/storage/tokudb/ft-index/ft/txn/txn.cc
@@ -344,6 +344,7 @@ static txn_child_manager tcm;
         .state = TOKUTXN_LIVE,
         .num_pin = 0,
         .client_id = 0,
+        .start_time = time(NULL),
     };
 
     TOKUTXN result = NULL;
@@ -787,6 +788,10 @@ void toku_txn_set_client_id(TOKUTXN txn, uint64_t client_id) {
     txn->client_id = client_id;
 }
 
+time_t toku_txn_get_start_time(struct tokutxn *txn) {
+    return txn->start_time;
+}
+
 int toku_txn_reads_txnid(TXNID txnid, TOKUTXN txn) {
     int r = 0;
     TXNID oldest_live_in_snapshot = toku_get_oldest_in_live_root_txn_list(txn);
diff --git a/storage/tokudb/ft-index/ft/txn/txn.h b/storage/tokudb/ft-index/ft/txn/txn.h
index 6381b5a7779..4f2778bf858 100644
--- a/storage/tokudb/ft-index/ft/txn/txn.h
+++ b/storage/tokudb/ft-index/ft/txn/txn.h
@@ -253,6 +253,7 @@ struct tokutxn {
     uint32_t num_pin; // number of threads (all hot indexes) that want this
                       // txn to not transition to commit or abort
     uint64_t client_id;
+    time_t start_time;
 };
 typedef struct tokutxn *TOKUTXN;
 
@@ -368,6 +369,8 @@ bool toku_txn_has_spilled_rollback(struct tokutxn *txn);
 uint64_t toku_txn_get_client_id(struct tokutxn *txn);
 void toku_txn_set_client_id(struct tokutxn *txn, uint64_t client_id);
 
+time_t toku_txn_get_start_time(struct tokutxn *txn);
+
 //
 // This function is used by the leafentry iterators.
 // returns TOKUDB_ACCEPT if live transaction context is allowed to read a value
diff --git a/storage/tokudb/ft-index/src/tests/test_iterate_live_transactions.cc b/storage/tokudb/ft-index/src/tests/test_iterate_live_transactions.cc
index dd00ddeeb9a..c104c5c8541 100644
--- a/storage/tokudb/ft-index/src/tests/test_iterate_live_transactions.cc
+++ b/storage/tokudb/ft-index/src/tests/test_iterate_live_transactions.cc
@@ -104,9 +104,11 @@ struct iterate_extra {
     bool visited_txn[3];
 };
 
-static int iterate_callback(uint64_t txnid, uint64_t client_id,
+static int iterate_callback(DB_TXN *txn,
                             iterate_row_locks_callback iterate_locks,
                             void *locks_extra, void *extra) {
+    uint64_t txnid = txn->id64(txn);
+    uint64_t client_id = txn->get_client_id(txn);
     iterate_extra *info = reinterpret_cast<iterate_extra *>(extra);
     DB *db;
     DBT left_key, right_key;
diff --git a/storage/tokudb/ft-index/src/tests/test_stress0.cc b/storage/tokudb/ft-index/src/tests/test_stress0.cc
index 5dbca08db48..26192d851aa 100644
--- a/storage/tokudb/ft-index/src/tests/test_stress0.cc
+++ b/storage/tokudb/ft-index/src/tests/test_stress0.cc
@@ -140,9 +140,11 @@ static int UU() iterate_pending_lock_requests_op(DB_TXN *UU(txn), ARG arg, void
     return r;
 }
 
-static int iterate_txns(uint64_t txnid, uint64_t client_id,
+static int iterate_txns(DB_TXN *txn,
                         iterate_row_locks_callback iterate_locks,
                         void *locks_extra, void *extra) {
+    uint64_t txnid = txn->id64(txn);
+    uint64_t client_id = txn->get_client_id(txn);
     invariant_null(extra);
     invariant(txnid > 0);
     invariant(client_id == 0);
diff --git a/storage/tokudb/ft-index/src/ydb.cc b/storage/tokudb/ft-index/src/ydb.cc
index 85445a67eef..ab15a44489e 100644
--- a/storage/tokudb/ft-index/src/ydb.cc
+++ b/storage/tokudb/ft-index/src/ydb.cc
@@ -2492,24 +2492,21 @@ struct iter_txns_callback_extra {
 };
 
 static int iter_txns_callback(TOKUTXN txn, void *extra) {
+    int r = 0;
     iter_txns_callback_extra *info =
         reinterpret_cast<iter_txns_callback_extra *>(extra);
-
     DB_TXN *dbtxn = toku_txn_get_container_db_txn(txn);
     invariant_notnull(dbtxn);
+    if (db_txn_struct_i(dbtxn)->tokutxn == txn) { // make sure that the dbtxn is fully initialized
+        toku_mutex_lock(&db_txn_struct_i(dbtxn)->txn_mutex);
+        toku_pthread_rwlock_rdlock(&info->env->i->open_dbs_rwlock);
 
-    toku_mutex_lock(&db_txn_struct_i(dbtxn)->txn_mutex);
-    toku_pthread_rwlock_rdlock(&info->env->i->open_dbs_rwlock);
-
-    iter_txn_row_locks_callback_extra e(info->env, &db_txn_struct_i(dbtxn)->lt_map);
-    const int r = info->callback(toku_txn_get_txnid(txn).parent_id64,
-                                 toku_txn_get_client_id(txn),
-                                 iter_txn_row_locks_callback,
-                                 &e,
-                                 info->extra);
+        iter_txn_row_locks_callback_extra e(info->env, &db_txn_struct_i(dbtxn)->lt_map);
+        r = info->callback(dbtxn, iter_txn_row_locks_callback, &e, info->extra);
 
-    toku_pthread_rwlock_rdunlock(&info->env->i->open_dbs_rwlock);
-    toku_mutex_unlock(&db_txn_struct_i(dbtxn)->txn_mutex);
+        toku_pthread_rwlock_rdunlock(&info->env->i->open_dbs_rwlock);
+        toku_mutex_unlock(&db_txn_struct_i(dbtxn)->txn_mutex);
+    }
 
     return r;
 }
diff --git a/storage/tokudb/ft-index/src/ydb_txn.cc b/storage/tokudb/ft-index/src/ydb_txn.cc
index 82903849535..885c6b713b2 100644
--- a/storage/tokudb/ft-index/src/ydb_txn.cc
+++ b/storage/tokudb/ft-index/src/ydb_txn.cc
@@ -433,6 +433,11 @@ static DB_TXN *toku_txn_get_child(DB_TXN *txn) {
     return db_txn_struct_i(txn)->child;
 }
 
+static uint64_t toku_txn_get_start_time(DB_TXN *txn) {
+    TOKUTXN ttxn = db_txn_struct_i(txn)->tokutxn;
+    return toku_txn_get_start_time(ttxn);
+}
+
 static inline void txn_func_init(DB_TXN *txn) {
 #define STXN(name) txn->name = locked_txn_ ## name
     STXN(abort);
@@ -451,6 +456,7 @@ static inline void txn_func_init(DB_TXN *txn) {
     txn->id64 = toku_txn_id64;
     txn->is_prepared = toku_txn_is_prepared;
     txn->get_child = toku_txn_get_child;
+    txn->get_start_time = toku_txn_get_start_time;
 }
 
 //
diff --git a/storage/tokudb/ft-index/tools/CMakeLists.txt b/storage/tokudb/ft-index/tools/CMakeLists.txt
index 71c44df9acd..f745517d84e 100644
--- a/storage/tokudb/ft-index/tools/CMakeLists.txt
+++ b/storage/tokudb/ft-index/tools/CMakeLists.txt
@@ -1,6 +1,6 @@
 set_property(DIRECTORY APPEND PROPERTY COMPILE_DEFINITIONS _GNU_SOURCE DONT_DEPRECATE_ERRNO)
 
-set(tools tokudb_dump tokuftdump tdb_logprint tdb-recover ftverify ba_replay)
+set(tools tokudb_dump tokuftdump tokuft_logprint tdb-recover ftverify ba_replay)
 foreach(tool ${tools})
   add_executable(${tool} ${tool}.cc)
   add_dependencies(${tool} install_tdb_h)
@@ -12,9 +12,6 @@ endforeach(tool)
 # link in math.h library just for this tool.
 target_link_libraries(ftverify m)
 
-install(
-  TARGETS tokuftdump
-  DESTINATION ${INSTALL_BINDIR}
-  COMPONENT Server
-  )
+install(TARGETS tokuftdump      DESTINATION ${INSTALL_BINDIR} COMPONENT Server)
+install(TARGETS tokuft_logprint DESTINATION ${INSTALL_BINDIR} COMPONENT Server)
 
diff --git a/storage/tokudb/ft-index/tools/tdb_logprint.cc b/storage/tokudb/ft-index/tools/tokuft_logprint.cc
index 15a28632cfb..1dd7581b9f5 100644
--- a/storage/tokudb/ft-index/tools/tdb_logprint.cc
+++ b/storage/tokudb/ft-index/tools/tokuft_logprint.cc
@@ -91,8 +91,6 @@ PATENT RIGHTS GRANT:
 
 /* Dump the log from stdin to stdout. */
 
-#include <config.h>
-
 #include "ft/log_header.h"
 #include "ft/logger/logger.h"
 
diff --git a/storage/tokudb/ft-index/tools/tokuftdump.cc b/storage/tokudb/ft-index/tools/tokuftdump.cc
index d680a3dd0d0..14c3c31a061 100644
--- a/storage/tokudb/ft-index/tools/tokuftdump.cc
+++ b/storage/tokudb/ft-index/tools/tokuftdump.cc
@@ -250,6 +250,8 @@ static int64_t getRootNode(FT ft) {
 }
 
 static int print_le(const void* key, const uint32_t keylen, const LEAFENTRY &le, const uint32_t idx UU(), void *const ai UU()) {
+    unsigned int *le_index = (unsigned int *) ai;
+    printf("%u: ", *le_index); *le_index += 1;
     print_klpair(stdout, key, keylen, le);
     printf("\n");
     return 0;
@@ -539,7 +541,8 @@ ok:
             printf(" n_bytes_in_buffer= %" PRIu64 "", BLB_DATA(n, i)->get_disk_size());
             printf(" items_in_buffer=%u\n", BLB_DATA(n, i)->num_klpairs());
             if (do_dump_data) {
-                BLB_DATA(n, i)->iterate<void, print_le>(NULL);
+                unsigned int le_index = 0;
+                BLB_DATA(n, i)->iterate<void, print_le>(&le_index);
             }
         }
     }
@@ -938,6 +941,7 @@ static void run_iteractive_loop(int fd, FT ft, CACHEFILE cf) {
         } else if (strcmp(fields[0], "header") == 0) {
             toku_ft_free(ft);
             open_header(fd, &ft, cf);
+            dump_header(ft);
         } else if (strcmp(fields[0], "rn") == 0||strcmp(fields[0], "rootNode")==0||strcmp(fields[0], "rootnode") == 0) {
             printf("Root node :%d\n",root);
         } else if (strcmp(fields[0], "block") == 0 && nfields == 2) {
diff --git a/storage/tokudb/ft-index/util/scoped_malloc.cc b/storage/tokudb/ft-index/util/scoped_malloc.cc
index 551bd944beb..15d4fb3e52e 100644
--- a/storage/tokudb/ft-index/util/scoped_malloc.cc
+++ b/storage/tokudb/ft-index/util/scoped_malloc.cc
@@ -145,6 +145,9 @@ namespace toku {
         }
 
         void destroy() {
+#if TOKU_SCOPED_MALLOC_DEBUG
+            printf("%s %p %p\n", __FUNCTION__, this, m_stack);
+#endif
             if (m_stack != NULL) {
                 toku_free(m_stack);
                 m_stack = NULL;
@@ -167,13 +170,17 @@ namespace toku {
         static void destroy_and_deregister(void *key) {
             invariant_notnull(key);
             tl_stack *st = reinterpret_cast<tl_stack *>(key);
-            st->destroy();
 
+            size_t n = 0;
             toku_mutex_lock(&global_stack_set_mutex);
-            invariant_notnull(global_stack_set);
-            size_t n = global_stack_set->erase(st);
-            invariant(n == 1);
+            if (global_stack_set) {
+                n = global_stack_set->erase(st);
+            }
             toku_mutex_unlock(&global_stack_set_mutex);
+
+            if (n == 1) {
+                st->destroy(); // destroy the stack if this function erased it from the set.  otherwise, somebody else destroyed it.
+            }
         }
 
         // Allocate 'size' bytes and return a pointer to the first byte
@@ -244,6 +251,11 @@ void toku_scoped_malloc_init(void) {
 }
 
 void toku_scoped_malloc_destroy(void) {
+    toku_scoped_malloc_destroy_key();
+    toku_scoped_malloc_destroy_set();
+}
+
+void toku_scoped_malloc_destroy_set(void) {
     toku_mutex_lock(&toku::global_stack_set_mutex);
     invariant_notnull(toku::global_stack_set);
     // Destroy any tl_stacks that were registered as thread locals but did not
@@ -254,10 +266,11 @@ void toku_scoped_malloc_destroy(void) {
         (*i)->destroy();
     }
     delete toku::global_stack_set;
+    toku::global_stack_set = nullptr;
     toku_mutex_unlock(&toku::global_stack_set_mutex);
+}
 
-    // We're deregistering the destructor key here. When this thread exits,
-    // the tl_stack destructor won't get called, so we need to do that first.
+void toku_scoped_malloc_destroy_key(void) {
     int r = pthread_key_delete(toku::tl_stack_destroy_pthread_key);
     invariant_zero(r);
 }
diff --git a/storage/tokudb/ft-index/util/scoped_malloc.h b/storage/tokudb/ft-index/util/scoped_malloc.h
index dbd919d155e..0233b0f1aa5 100644
--- a/storage/tokudb/ft-index/util/scoped_malloc.h
+++ b/storage/tokudb/ft-index/util/scoped_malloc.h
@@ -151,3 +151,7 @@ void toku_scoped_malloc_init(void);
 
 void toku_scoped_malloc_destroy(void);
 
+void toku_scoped_malloc_destroy_set(void);
+
+void toku_scoped_malloc_destroy_key(void);
+