Import wiredtiger: b4070294cdfd2df5e6a3a7f355f6c9109e094048 from branch mongodb-master

ref: 06926a917a..b4070294cd
for: 6.2.0-rc0

WT-9670 Add early statistics access via a general event handler. (#8299)

15 files changed, 124 insertions, 27 deletions

diff --git a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c
index 3ffeb0bf262..7e0f8b8b03b 100644
--- a/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c
+++ b/src/third_party/wiredtiger/test/checkpoint/test_checkpoint.c
@@ -275,9 +275,7 @@ run_complete:
 static int
 wt_connect(const char *config_open)
 {
-    static WT_EVENT_HANDLER event_handler = {
-      handle_error, handle_message, NULL, NULL /* Close handler. */
-    };
+    static WT_EVENT_HANDLER event_handler = {handle_error, handle_message, NULL, NULL, NULL};
     WT_RAND_STATE rnd;
     int ret;
     char buf[512], config[1024];
diff --git a/src/third_party/wiredtiger/test/csuite/schema_abort/main.c b/src/third_party/wiredtiger/test/csuite/schema_abort/main.c
index 2f92a0a3585..5d3da58863d 100644
--- a/src/third_party/wiredtiger/test/csuite/schema_abort/main.c
+++ b/src/third_party/wiredtiger/test/csuite/schema_abort/main.c
@@ -171,7 +171,8 @@ subtest_error_handler(
 static WT_EVENT_HANDLER event_handler = {
   subtest_error_handler, NULL, /* Message handler */
   NULL,                        /* Progress handler */
-  NULL                         /* Close handler */
+  NULL,                        /* Close handler */
+  NULL                         /* General handler */
 };
 
 /*
diff --git a/src/third_party/wiredtiger/test/csuite/scope/main.c b/src/third_party/wiredtiger/test/csuite/scope/main.c
index 66e88a15c9c..78b9694ac9f 100644
--- a/src/third_party/wiredtiger/test/csuite/scope/main.c
+++ b/src/third_party/wiredtiger/test/csuite/scope/main.c
@@ -53,7 +53,7 @@ handle_error(WT_EVENT_HANDLER *handler, WT_SESSION *session, int error, const ch
     return (0);
 }
 
-static WT_EVENT_HANDLER event_handler = {handle_error, NULL, NULL, NULL};
+static WT_EVENT_HANDLER event_handler = {handle_error, NULL, NULL, NULL, NULL};
 
 #define SET_KEY                                   \
     do {                                          \
diff --git a/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c b/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c
index 96494830d1b..4c76a4ae2e2 100644
--- a/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c
+++ b/src/third_party/wiredtiger/test/csuite/timestamp_abort/main.c
@@ -71,6 +71,7 @@ static char home[1024]; /* Program working dir */
 /* Include worker threads and prepare extra sessions */
 #define SESSION_MAX (MAX_TH + 3 + MAX_TH * PREPARE_PCT)
 #define STAT_WAIT 1
+#define USEC_STAT 10000
 
 static const char *table_pfx = "table";
 static const char *const uri_collection = "collection";
@@ -133,6 +134,105 @@ static wt_timestamp_t *active_timestamps; /* Oldest timestamps still in use. */
 static void handler(int) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn));
 static void usage(void) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn));
 
+static void handle_conn_close(void);
+static void handle_conn_ready(WT_CONNECTION *);
+static int handle_general(WT_EVENT_HANDLER *, WT_CONNECTION *, WT_SESSION *, WT_EVENT_TYPE);
+
+static WT_CONNECTION *stat_conn = NULL;
+static WT_SESSION *stat_session = NULL;
+static volatile bool stat_run = false;
+static wt_thread_t stat_th;
+
+static WT_EVENT_HANDLER my_event = {NULL, NULL, NULL, NULL, handle_general};
+/*
+ * stat_func --
+ *     Function to run with the early connection and gather statistics.
+ */
+static WT_THREAD_RET
+stat_func(void *arg)
+{
+    WT_CURSOR *stat_c;
+    int64_t last, value;
+    const char *desc, *pvalue;
+
+    WT_UNUSED(arg);
+    testutil_assert(stat_conn != NULL);
+    testutil_check(stat_conn->open_session(stat_conn, NULL, NULL, &stat_session));
+    desc = pvalue = NULL;
+    /* Start last and value at different numbers so we print the first value, likely 0. */
+    last = -1;
+    value = 0;
+    while (stat_run) {
+        testutil_check(stat_session->open_cursor(stat_session, "statistics:", NULL, NULL, &stat_c));
+
+        /* Pick some statistic that is likely changed during recovery RTS. */
+        stat_c->set_key(stat_c, WT_STAT_CONN_TXN_RTS_PAGES_VISITED);
+        testutil_check(stat_c->search(stat_c));
+        testutil_check(stat_c->get_value(stat_c, &desc, &pvalue, &value));
+        testutil_check(stat_c->close(stat_c));
+        if (desc != NULL && value != last)
+            printf("%s: %" PRId64 "\n", desc, value);
+        last = value;
+        usleep(USEC_STAT);
+    }
+    testutil_check(stat_session->close(stat_session, NULL));
+    return (WT_THREAD_RET_VALUE);
+}
+
+/*
+ * handle_conn_close --
+ *     Function to handle connection close callbacks from WiredTiger.
+ */
+static void
+handle_conn_close(void)
+{
+    /*
+     * Signal the statistics thread to exit and clear the global connection. This function cannot
+     * return until the user thread stops using the connection.
+     */
+    stat_run = false;
+    testutil_check(__wt_thread_join(NULL, &stat_th));
+    stat_conn = NULL;
+}
+
+/*
+ * handle_conn_ready --
+ *     Function to handle connection ready callbacks from WiredTiger.
+ */
+static void
+handle_conn_ready(WT_CONNECTION *conn)
+{
+    int unused;
+
+    /*
+     * Set the global connection for statistics and then start a statistics thread.
+     */
+    unused = 0;
+    testutil_assert(stat_conn == NULL);
+    memset(&stat_th, 0, sizeof(stat_th));
+    stat_conn = conn;
+    stat_run = true;
+    testutil_check(__wt_thread_create(NULL, &stat_th, stat_func, (void *)&unused));
+}
+
+/*
+ * handle_general --
+ *     Function to handle general event callbacks.
+ */
+static int
+handle_general(
+  WT_EVENT_HANDLER *handler, WT_CONNECTION *conn, WT_SESSION *session, WT_EVENT_TYPE type)
+{
+    WT_UNUSED(handler);
+    WT_UNUSED(session);
+
+    if (type == WT_EVENT_CONN_CLOSE)
+        handle_conn_close();
+    else if (type == WT_EVENT_CONN_READY)
+        handle_conn_ready(conn);
+    return (0);
+}
+
 /*
  * usage --
  *     TODO: Add a comment describing this function.
@@ -170,7 +270,8 @@ thread_ts_run(void *arg)
     __wt_seconds((WT_SESSION_IMPL *)session, &last_reconfig);
     /* Update the oldest/stable timestamps every 1 millisecond. */
     for (last_ts = 0;; __wt_sleep(0, 1000)) {
-        /* Get the last committed timestamp periodically in order to update the oldest timestamp. */
+        /* Get the last committed timestamp periodically in order to update the oldest
+         * timestamp. */
         ts = maximum_stable_ts(active_timestamps, nth);
         if (ts == last_ts)
             continue;
@@ -797,12 +898,15 @@ main(int argc, char *argv[])
     /* Copy the data to a separate folder for debugging purpose. */
     testutil_copy_data(home);
 
-    printf("Open database, run recovery and verify content\n");
+    printf("Open database and run recovery\n");
 
     /*
      * Open the connection which forces recovery to be run.
      */
-    testutil_check(wiredtiger_open(NULL, NULL, ENV_CONFIG_REC, &conn));
+    testutil_check(wiredtiger_open(NULL, &my_event, ENV_CONFIG_REC, &conn));
+    printf("Connection open and recovery complete. Verify content\n");
+    /* Sleep to guarantee the statistics thread has enough time to run. */
+    usleep(USEC_STAT + 10);
     testutil_check(conn->open_session(conn, NULL, NULL, &session));
     /*
      * Open a cursor on all the tables.
diff --git a/src/third_party/wiredtiger/test/csuite/wt2719_reconfig/main.c b/src/third_party/wiredtiger/test/csuite/wt2719_reconfig/main.c
index 195f61b05e9..610913e227d 100644
--- a/src/third_party/wiredtiger/test/csuite/wt2719_reconfig/main.c
+++ b/src/third_party/wiredtiger/test/csuite/wt2719_reconfig/main.c
@@ -111,7 +111,7 @@ handle_message(WT_EVENT_HANDLER *handler, WT_SESSION *session, const char *messa
     return (0);
 }
 
-static WT_EVENT_HANDLER event_handler = {NULL, handle_message, NULL, NULL};
+static WT_EVENT_HANDLER event_handler = {NULL, handle_message, NULL, NULL, NULL};
 
 static const char *current; /* Current test configuration */
 
diff --git a/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c b/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c
index afbb3ca5986..abff8420eed 100644
--- a/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c
+++ b/src/third_party/wiredtiger/test/csuite/wt2909_checkpoint_integrity/main.c
@@ -485,7 +485,8 @@ subtest_error_handler(
 static WT_EVENT_HANDLER event_handler = {
   subtest_error_handler, NULL, /* Message handler */
   NULL,                        /* Progress handler */
-  NULL                         /* Close handler */
+  NULL,                        /* Close handler */
+  NULL                         /* General handler */
 };
 
 /*
diff --git a/src/third_party/wiredtiger/test/csuite/wt3363_checkpoint_op_races/main.c b/src/third_party/wiredtiger/test/csuite/wt3363_checkpoint_op_races/main.c
index 9e6205935d8..24db947628b 100644
--- a/src/third_party/wiredtiger/test/csuite/wt3363_checkpoint_op_races/main.c
+++ b/src/third_party/wiredtiger/test/csuite/wt3363_checkpoint_op_races/main.c
@@ -66,7 +66,7 @@ static WT_THREAD_RET monitor(void *);
  */
 #define RUNTIME 900.0
 
-static WT_EVENT_HANDLER event_handler = {handle_op_error, handle_op_message, NULL, NULL};
+static WT_EVENT_HANDLER event_handler = {handle_op_error, handle_op_message, NULL, NULL, NULL};
 
 /*
  * main --
diff --git a/src/third_party/wiredtiger/test/csuite/wt4105_large_doc_small_upd/main.c b/src/third_party/wiredtiger/test/csuite/wt4105_large_doc_small_upd/main.c
index 5c236c65e5d..cab313f515d 100644
--- a/src/third_party/wiredtiger/test/csuite/wt4105_large_doc_small_upd/main.c
+++ b/src/third_party/wiredtiger/test/csuite/wt4105_large_doc_small_upd/main.c
@@ -74,7 +74,7 @@ handle_error(WT_EVENT_HANDLER *handler, WT_SESSION *session, int error, const ch
     return (0);
 }
 
-static WT_EVENT_HANDLER event_handler = {handle_error, NULL, NULL, NULL};
+static WT_EVENT_HANDLER event_handler = {handle_error, NULL, NULL, NULL, NULL};
 
 /*
  * main --
diff --git a/src/third_party/wiredtiger/test/csuite/wt4156_metadata_salvage/main.c b/src/third_party/wiredtiger/test/csuite/wt4156_metadata_salvage/main.c
index 72362888423..e8879bbcad6 100644
--- a/src/third_party/wiredtiger/test/csuite/wt4156_metadata_salvage/main.c
+++ b/src/third_party/wiredtiger/test/csuite/wt4156_metadata_salvage/main.c
@@ -67,7 +67,7 @@ handle_message(WT_EVENT_HANDLER *handler, WT_SESSION *session, int error, const
     return (0);
 }
 
-static WT_EVENT_HANDLER event_handler = {handle_message, NULL, NULL, NULL};
+static WT_EVENT_HANDLER event_handler = {handle_message, NULL, NULL, NULL, NULL};
 
 typedef struct table_info {
     const char *name;
diff --git a/src/third_party/wiredtiger/test/csuite/wt4803_history_store_abort/main.c b/src/third_party/wiredtiger/test/csuite/wt4803_history_store_abort/main.c
index 6efbd3b51d4..963f6527b9e 100644
--- a/src/third_party/wiredtiger/test/csuite/wt4803_history_store_abort/main.c
+++ b/src/third_party/wiredtiger/test/csuite/wt4803_history_store_abort/main.c
@@ -76,7 +76,7 @@ handle_message(WT_EVENT_HANDLER *handler, WT_SESSION *session, int error, const
     return (0);
 }
 
-static WT_EVENT_HANDLER event_handler = {handle_message, NULL, NULL, NULL};
+static WT_EVENT_HANDLER event_handler = {handle_message, NULL, NULL, NULL, NULL};
 
 /*
  * hs_workload --
diff --git a/src/third_party/wiredtiger/test/csuite/wt8057_compact_stress/main.c b/src/third_party/wiredtiger/test/csuite/wt8057_compact_stress/main.c
index dfcbe95c23b..91e3529cc33 100644
--- a/src/third_party/wiredtiger/test/csuite/wt8057_compact_stress/main.c
+++ b/src/third_party/wiredtiger/test/csuite/wt8057_compact_stress/main.c
@@ -76,7 +76,8 @@ subtest_error_handler(
 static WT_EVENT_HANDLER event_handler = {
   subtest_error_handler, NULL, /* Message handler */
   NULL,                        /* Progress handler */
-  NULL                         /* Close handler */
+  NULL,                        /* Close handler */
+  NULL                         /* Special handler */
 };
 
 static void sig_handler(int) WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn));
diff --git a/src/third_party/wiredtiger/test/cursor_order/cursor_order.c b/src/third_party/wiredtiger/test/cursor_order/cursor_order.c
index 4cf99d20c5a..2b4c1bc20e4 100644
--- a/src/third_party/wiredtiger/test/cursor_order/cursor_order.c
+++ b/src/third_party/wiredtiger/test/cursor_order/cursor_order.c
@@ -168,9 +168,7 @@ main(int argc, char *argv[])
 static void
 wt_connect(SHARED_CONFIG *cfg, char *config_open)
 {
-    static WT_EVENT_HANDLER event_handler = {
-      handle_error, handle_message, NULL, NULL /* Close handler. */
-    };
+    static WT_EVENT_HANDLER event_handler = {handle_error, handle_message, NULL, NULL, NULL};
     char config[512];
 
     testutil_clean_work_dir(home);
diff --git a/src/third_party/wiredtiger/test/fops/t.c b/src/third_party/wiredtiger/test/fops/t.c
index aa96682cd74..92de1f37e01 100644
--- a/src/third_party/wiredtiger/test/fops/t.c
+++ b/src/third_party/wiredtiger/test/fops/t.c
@@ -147,9 +147,7 @@ main(int argc, char *argv[])
 static void
 wt_startup(char *config_open)
 {
-    static WT_EVENT_HANDLER event_handler = {
-      handle_error, handle_message, NULL, NULL /* Close handler. */
-    };
+    static WT_EVENT_HANDLER event_handler = {handle_error, handle_message, NULL, NULL, NULL};
     char config_buf[128];
 
     testutil_make_work_dir(home);
diff --git a/src/third_party/wiredtiger/test/format/wts.c b/src/third_party/wiredtiger/test/format/wts.c
index 14417ff309f..d1096347971 100644
--- a/src/third_party/wiredtiger/test/format/wts.c
+++ b/src/third_party/wiredtiger/test/format/wts.c
@@ -137,9 +137,7 @@ handle_progress(
     return (0);
 }
 
-static WT_EVENT_HANDLER event_handler = {
-  NULL, handle_message, handle_progress, NULL /* Close handler. */
-};
+static WT_EVENT_HANDLER event_handler = {NULL, handle_message, handle_progress, NULL, NULL};
 
 #define CONFIG_APPEND(p, ...)                                               \
     do {                                                                    \
diff --git a/src/third_party/wiredtiger/test/thread/t.c b/src/third_party/wiredtiger/test/thread/t.c
index 9a0dd5fba20..ab6d0b3e92c 100644
--- a/src/third_party/wiredtiger/test/thread/t.c
+++ b/src/third_party/wiredtiger/test/thread/t.c
@@ -174,9 +174,7 @@ main(int argc, char *argv[])
 static void
 wt_connect(char *config_open)
 {
-    static WT_EVENT_HANDLER event_handler = {
-      handle_error, handle_message, NULL, NULL /* Close handler. */
-    };
+    static WT_EVENT_HANDLER event_handler = {handle_error, handle_message, NULL, NULL, NULL};
     char config[512];
 
     testutil_clean_work_dir(home);