Import wiredtiger: ddc72b3da218d624eb4962ee74cfa23512d0df28 from branch mongodb-master

ref: 042a408af2..ddc72b3da2
for: 7.0.0-rc0

WT-10897 Enable compact to be interrupted quickly; add message when compaction is skipped

6 files changed, 253 insertions, 5 deletions

diff --git a/src/third_party/wiredtiger/dist/s_void b/src/third_party/wiredtiger/dist/s_void
index e81ea1f437d..38e35c66d55 100755
--- a/src/third_party/wiredtiger/dist/s_void
+++ b/src/third_party/wiredtiger/dist/s_void
@@ -103,6 +103,7 @@ func_ok()
         -e '/int demo_file_sync$/d' \
         -e '/int demo_fs_directory_list_free$/d' \
         -e '/int demo_fs_exist$/d' \
+        -e '/int error_handler$/d' \
         -e '/int fail_file_lock$/d' \
         -e '/int fail_file_sync$/d' \
         -e '/int fail_fs_directory_list_free$/d' \
@@ -126,6 +127,7 @@ func_ok()
         -e '/int lz4_pre_size$/d' \
         -e '/int lz4_terminate$/d' \
         -e '/int main$/d' \
+        -e '/int message_handler$/d' \
         -e '/int nop_decompress$/d' \
         -e '/int nop_decrypt$/d' \
         -e '/int nop_error$/d' \
diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data
index 358f6ac0462..ab154a7f5d2 100644
--- a/src/third_party/wiredtiger/import.data
+++ b/src/third_party/wiredtiger/import.data
@@ -2,5 +2,5 @@
     "vendor": "wiredtiger",
     "github": "wiredtiger/wiredtiger.git",
     "branch": "mongodb-master",
-    "commit": "042a408af26e8d6933a51baf3cf82acdf4810e91"
+    "commit": "ddc72b3da218d624eb4962ee74cfa23512d0df28"
 }
diff --git a/src/third_party/wiredtiger/src/block/block_compact.c b/src/third_party/wiredtiger/src/block/block_compact.c
index edb5505bea9..af66b271f4d 100644
--- a/src/third_party/wiredtiger/src/block/block_compact.c
+++ b/src/third_party/wiredtiger/src/block/block_compact.c
@@ -111,7 +111,7 @@ __wt_block_compact_skip(WT_SESSION_IMPL *session, WT_BLOCK *block, bool *skipp)
      * are unlikely to recover 10% of the file.
      */
     if (block->size <= WT_MEGABYTE) {
-        __wt_verbose_info(session, WT_VERB_COMPACT,
+        __wt_verbose_debug1(session, WT_VERB_COMPACT,
           "%s: skipping because the file size must be greater than 1MB: %" PRIuMAX "B.",
           block->name, (uintmax_t)block->size);
 
diff --git a/src/third_party/wiredtiger/src/btree/bt_compact.c b/src/third_party/wiredtiger/src/btree/bt_compact.c
index 6e4d4c21fac..48dfb45dfa2 100644
--- a/src/third_party/wiredtiger/src/btree/bt_compact.c
+++ b/src/third_party/wiredtiger/src/btree/bt_compact.c
@@ -304,7 +304,7 @@ __wt_compact(WT_SESSION_IMPL *session)
     WT_DECL_RET;
     WT_REF *ref;
     u_int i, msg_count;
-    bool skip;
+    bool first, skip;
 
     uint64_t stats_pages_rewritten; /* Pages rewritten */
     uint64_t stats_pages_reviewed;  /* Pages reviewed */
@@ -323,10 +323,13 @@ __wt_compact(WT_SESSION_IMPL *session)
     if (skip) {
         WT_STAT_CONN_INCR(session, session_table_compact_skipped);
         WT_STAT_DATA_INCR(session, btree_compact_skipped);
+        __wt_verbose_info(session, WT_VERB_COMPACT,
+          "%s: there is no useful work to do - skipping compaction", bm->block->name);
         return (0);
     }
 
     /* Walk the tree reviewing pages to see if they should be re-written. */
+    first = true;
     for (i = 0;;) {
 
         /* Track progress. */
@@ -340,8 +343,9 @@ __wt_compact(WT_SESSION_IMPL *session)
          * Periodically check if we've timed out or eviction is stuck. Quit if eviction is stuck,
          * we're making the problem worse.
          */
-        if (++i > 100) {
-            bm->compact_progress(bm, session, &msg_count);
+        if (first || ++i > 100) {
+            if (!first)
+                bm->compact_progress(bm, session, &msg_count);
             WT_ERR(__wt_session_compact_check_timeout(session));
             if (session->event_handler->handle_general != NULL) {
                 ret = session->event_handler->handle_general(session->event_handler,
@@ -354,6 +358,7 @@ __wt_compact(WT_SESSION_IMPL *session)
             if (__wt_cache_stuck(session))
                 WT_ERR(EBUSY);
 
+            first = false;
             i = 0;
         }
 
diff --git a/src/third_party/wiredtiger/test/csuite/wt10897_compact_quick_interrupt/main.c b/src/third_party/wiredtiger/test/csuite/wt10897_compact_quick_interrupt/main.c
new file mode 100644
index 00000000000..538ecc2e958
--- /dev/null
+++ b/src/third_party/wiredtiger/test/csuite/wt10897_compact_quick_interrupt/main.c
@@ -0,0 +1,231 @@
+/*-
+ * Public Domain 2014-present MongoDB, Inc.
+ * Public Domain 2008-2014 WiredTiger, Inc.
+ *
+ * This is free and unencumbered software released into the public domain.
+ *
+ * Anyone is free to copy, modify, publish, use, compile, sell, or
+ * distribute this software, either in source code form or as a compiled
+ * binary, for any purpose, commercial or non-commercial, and by any
+ * means.
+ *
+ * In jurisdictions that recognize copyright laws, the author or authors
+ * of this software dedicate any and all copyright interest in the
+ * software to the public domain. We make this dedication for the benefit
+ * of the public at large and to the detriment of our heirs and
+ * successors. We intend this dedication to be an overt act of
+ * relinquishment in perpetuity of all present and future rights to this
+ * software under copyright law.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "test_util.h"
+
+#include <sys/wait.h>
+#include <signal.h>
+
+/*
+ * This test verifies that we can quickly interrupt compact, before it does much meaningful work.
+ */
+#define NUM_RECORDS1 (10)
+#define NUM_RECORDS2 (100 * WT_THOUSAND)
+
+/* The table URI. */
+#define TABLE_URI "table:compact"
+
+/* Constants and variables declaration. */
+static const char conn_config[] = "create,cache_size=2GB,statistics=(all),verbose=[compact]";
+static const char table_config[] =
+  "allocation_size=4KB,leaf_page_max=4KB,key_format=Q,value_format=" WT_UNCHECKED_STRING(QS);
+
+static char data_str[1024] = "";
+static bool skipped_compaction = false;
+static bool interrupted_compaction = false;
+
+/*
+ * error_handler --
+ *     Error handler.
+ */
+static int
+error_handler(WT_EVENT_HANDLER *handler, WT_SESSION *session, int error, const char *message)
+{
+    (void)(handler);
+    (void)(session);
+    (void)(error);
+
+    if (strstr(message, "compact interrupted") != NULL)
+        interrupted_compaction = true;
+
+    fprintf(stderr, "%s\n", message);
+    return (0);
+}
+
+/*
+ * message_handler --
+ *     Message handler.
+ */
+static int
+message_handler(WT_EVENT_HANDLER *handler, WT_SESSION *session, const char *message)
+{
+    (void)(handler);
+    (void)(session);
+
+    if (strstr(message, "skipping compaction") != NULL)
+        skipped_compaction = true;
+
+    fprintf(stderr, "%s\n", message);
+    return (0);
+}
+
+/*
+ * handle_general --
+ *     General event handler.
+ */
+static int
+handle_general(WT_EVENT_HANDLER *handler, WT_CONNECTION *conn, WT_SESSION *session,
+  WT_EVENT_TYPE type, void *arg)
+{
+    (void)(handler);
+    (void)(conn);
+    (void)(session);
+    (void)(arg);
+
+    if (type == WT_EVENT_COMPACT_CHECK)
+        return (-1);
+
+    return (0);
+}
+
+static WT_EVENT_HANDLER event_handler = {
+  error_handler, message_handler, /* Message handlers */
+  NULL,                           /* Progress handler */
+  NULL,                           /* Close handler */
+  handle_general                  /* General handler */
+};
+
+/*
+ * populate --
+ *     Populate the table.
+ */
+static void
+populate(WT_SESSION *session, uint64_t start, uint64_t end)
+{
+    WT_CURSOR *cursor;
+    WT_RAND_STATE rnd;
+
+    uint64_t i, str_len, val;
+
+    __wt_random_init_seed((WT_SESSION_IMPL *)session, &rnd);
+
+    str_len = sizeof(data_str) / sizeof(data_str[0]);
+    for (i = 0; i < str_len - 1; i++)
+        data_str[i] = 'a' + (uint32_t)__wt_random(&rnd) % 26;
+    data_str[str_len - 1] = '\0';
+
+    testutil_check(session->open_cursor(session, TABLE_URI, NULL, NULL, &cursor));
+
+    for (i = start; i < end; i++) {
+        val = (uint64_t)__wt_random(&rnd);
+        cursor->set_key(cursor, i + 1);
+        cursor->set_value(cursor, val, data_str);
+        testutil_check(cursor->insert(cursor));
+    }
+
+    testutil_check(cursor->close(cursor));
+}
+
+/*
+ * remove_records --
+ *     Remove a range of records.
+ */
+static void
+remove_records(WT_SESSION *session, uint64_t start, uint64_t end)
+{
+    WT_CURSOR *cursor;
+    uint64_t i;
+
+    testutil_check(session->open_cursor(session, TABLE_URI, NULL, NULL, &cursor));
+
+    for (i = start; i < end; i++) {
+        cursor->set_key(cursor, i + 1);
+        testutil_check(cursor->remove(cursor));
+    }
+
+    testutil_check(cursor->close(cursor));
+}
+
+/*
+ * main --
+ *     The main method.
+ */
+int
+main(int argc, char *argv[])
+{
+    TEST_OPTS *opts, _opts;
+    WT_CONNECTION *conn;
+    WT_CURSOR *stat;
+    WT_SESSION *session;
+    int64_t pages_reviewed;
+    int ret;
+    char home[1024];
+    const char *desc, *pvalue;
+
+    opts = &_opts;
+    memset(opts, 0, sizeof(*opts));
+    testutil_check(testutil_parse_opts(argc, argv, opts));
+
+    /* Initialize the database with just a few records. */
+    testutil_work_dir_from_path(home, sizeof(home), "WT_TEST.compact-quick-interrupt");
+    testutil_make_work_dir(home);
+
+    testutil_check(wiredtiger_open(home, &event_handler, conn_config, &conn));
+    testutil_check(conn->open_session(conn, NULL, NULL, &session));
+    testutil_check(session->create(session, TABLE_URI, table_config));
+
+    populate(session, 0, NUM_RECORDS1);
+    testutil_check(session->checkpoint(session, NULL));
+
+    /* At this point, check that compact does not have any meaningful work to do. */
+    skipped_compaction = false;
+    testutil_check(session->compact(session, TABLE_URI, NULL));
+    testutil_assert(skipped_compaction);
+
+    /*
+     * Now populate the table a lot more, make some space, and then see if we can interrupt the
+     * compaction quickly - even before the compaction can get any work done.
+     */
+    populate(session, NUM_RECORDS1, NUM_RECORDS1 + NUM_RECORDS2);
+    testutil_check(session->checkpoint(session, NULL));
+    remove_records(session, 0, NUM_RECORDS1 + NUM_RECORDS2 / 2);
+
+    interrupted_compaction = false;
+    skipped_compaction = false;
+    ret = session->compact(session, TABLE_URI, NULL);
+
+    testutil_assert(ret == WT_ERROR);
+    testutil_assert(interrupted_compaction);
+    testutil_assert(!skipped_compaction);
+
+    /* Check that we didn't get any work done. */
+    testutil_check(session->open_cursor(session, "statistics:" TABLE_URI, NULL, NULL, &stat));
+    stat->set_key(stat, WT_STAT_DSRC_BTREE_COMPACT_PAGES_REVIEWED);
+    testutil_check(stat->search(stat));
+    testutil_check(stat->get_value(stat, &desc, &pvalue, &pages_reviewed));
+    testutil_check(stat->close(stat));
+    testutil_assert(pages_reviewed == 0);
+
+    /* Finish the test and clean up. */
+    testutil_check(session->close(session, NULL));
+    testutil_check(conn->close(conn, NULL));
+
+    if (!opts->preserve)
+        testutil_clean_work_dir(home);
+    testutil_cleanup(opts);
+    return (EXIT_SUCCESS);
+}
diff --git a/src/third_party/wiredtiger/test/evergreen.yml b/src/third_party/wiredtiger/test/evergreen.yml
index 2ed5c44c16b..86137e980b6 100755
--- a/src/third_party/wiredtiger/test/evergreen.yml
+++ b/src/third_party/wiredtiger/test/evergreen.yml
@@ -2226,6 +2226,16 @@ tasks:
         vars:
           test_name: wt10461_skip_list_stress
 
+  - name: csuite-wt10897-compact-quick-interrupt-test
+    tags: ["pull_request"]
+    depends_on:
+      - name: compile
+    commands:
+      - func: "fetch artifacts"
+      - func: "csuite test"
+        vars:
+          test_name: wt10897_compact_quick_interrupt
+
   # End of csuite test tasks
 
   # Start of Python unit test tasks