Import wiredtiger: e01636eb346665d941626ceecdf74eaa8e351974 from branch mongodb-5.1

ref: 70ab26de2a..e01636eb34
for: 5.1.0-rc2

WT-8188       Use compact progress stats in compact related tests

6 files changed, 128 insertions, 1 deletions

diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data
index 8eb40686dfd..2b9069a447c 100644
--- a/src/third_party/wiredtiger/import.data
+++ b/src/third_party/wiredtiger/import.data
@@ -2,5 +2,5 @@
     "vendor": "wiredtiger",
     "github": "wiredtiger/wiredtiger.git",
     "branch": "mongodb-5.1",
-    "commit": "70ab26de2ab263fabab39114aee583f632f4e088"
+    "commit": "e01636eb346665d941626ceecdf74eaa8e351974"
 }
diff --git a/src/third_party/wiredtiger/test/csuite/wt7989_compact_checkpoint/main.c b/src/third_party/wiredtiger/test/csuite/wt7989_compact_checkpoint/main.c
index 6e60b1ee229..6c0ab82fc0c 100644
--- a/src/third_party/wiredtiger/test/csuite/wt7989_compact_checkpoint/main.c
+++ b/src/third_party/wiredtiger/test/csuite/wt7989_compact_checkpoint/main.c
@@ -69,6 +69,8 @@ static void remove_records(WT_SESSION *, const char *);
 static void get_file_stats(WT_SESSION *, const char *, uint64_t *, uint64_t *);
 static void set_timing_stress_checkpoint(WT_CONNECTION *);
 static bool check_db_size(WT_SESSION *, const char *);
+static void get_compact_progress(
+  WT_SESSION *session, const char *, uint64_t *, uint64_t *, uint64_t *);
 
 /* Methods implementation. */
 int
@@ -132,6 +134,7 @@ run_test(bool stress_test, bool column_store, const char *home, const char *uri)
     WT_CONNECTION *conn;
     WT_SESSION *session;
     pthread_t thread_checkpoint;
+    uint64_t pages_reviewed, pages_rewritten, pages_selected;
     bool size_check_res;
 
     testutil_make_work_dir(home);
@@ -180,6 +183,8 @@ run_test(bool stress_test, bool column_store, const char *home, const char *uri)
     (void)pthread_join(thread_checkpoint, NULL);
     (void)pthread_join(thread_compact, NULL);
 
+    /* Collect compact progress stats. */
+    get_compact_progress(session, uri, &pages_reviewed, &pages_selected, &pages_rewritten);
     size_check_res = check_db_size(session, uri);
 
     /* Cleanup */
@@ -194,6 +199,12 @@ run_test(bool stress_test, bool column_store, const char *home, const char *uri)
     testutil_check(conn->close(conn, NULL));
     conn = NULL;
 
+    printf(" - Pages reviewed: %" PRIu64 "\n", pages_reviewed);
+    printf(" - Pages selected for being rewritten: %" PRIu64 "\n", pages_selected);
+    printf(" - Pages actually rewritten: %" PRIu64 "\n", pages_rewritten);
+    testutil_assert(pages_reviewed > 0);
+    testutil_assert(pages_selected > 0);
+    testutil_assert(pages_rewritten > 0);
     /*
      * Check if there's more than 10% available space in the file. Checking result here to allow
      * connection to close properly.
@@ -369,6 +380,31 @@ set_timing_stress_checkpoint(WT_CONNECTION *conn)
     conn_impl->timing_stress_flags |= WT_TIMING_STRESS_CHECKPOINT_SLOW;
 }
 
+static void
+get_compact_progress(WT_SESSION *session, const char *uri, uint64_t *pages_reviewed,
+  uint64_t *pages_selected, uint64_t *pages_rewritten)
+{
+
+    WT_CURSOR *cur_stat;
+    char *descr, *str_val;
+    char stat_uri[128];
+
+    sprintf(stat_uri, "statistics:%s", uri);
+    testutil_check(session->open_cursor(session, stat_uri, NULL, "statistics=(all)", &cur_stat));
+
+    cur_stat->set_key(cur_stat, WT_STAT_DSRC_BTREE_COMPACT_PAGES_REVIEWED);
+    testutil_check(cur_stat->search(cur_stat));
+    testutil_check(cur_stat->get_value(cur_stat, &descr, &str_val, pages_reviewed));
+    cur_stat->set_key(cur_stat, WT_STAT_DSRC_BTREE_COMPACT_PAGES_WRITE_SELECTED);
+    testutil_check(cur_stat->search(cur_stat));
+    testutil_check(cur_stat->get_value(cur_stat, &descr, &str_val, pages_selected));
+    cur_stat->set_key(cur_stat, WT_STAT_DSRC_BTREE_COMPACT_PAGES_REWRITTEN);
+    testutil_check(cur_stat->search(cur_stat));
+    testutil_check(cur_stat->get_value(cur_stat, &descr, &str_val, pages_rewritten));
+
+    testutil_check(cur_stat->close(cur_stat));
+}
+
 static bool
 check_db_size(WT_SESSION *session, const char *uri)
 {
diff --git a/src/third_party/wiredtiger/test/csuite/wt8057_compact_stress/main.c b/src/third_party/wiredtiger/test/csuite/wt8057_compact_stress/main.c
index 22a96c50f78..74665c7acf1 100644
--- a/src/third_party/wiredtiger/test/csuite/wt8057_compact_stress/main.c
+++ b/src/third_party/wiredtiger/test/csuite/wt8057_compact_stress/main.c
@@ -90,6 +90,8 @@ static void verify_tables(WT_SESSION *);
 static int verify_tables_helper(WT_SESSION *, const char *, const char *);
 static void get_file_stats(WT_SESSION *, const char *, uint64_t *, uint64_t *);
 static void log_db_size(WT_SESSION *, const char *);
+static void get_compact_progress(
+  WT_SESSION *session, const char *, uint64_t *, uint64_t *, uint64_t *);
 
 /*
  * Signal handler to catch if the child died unexpectedly.
@@ -213,6 +215,8 @@ workload_compact(const char *home, const char *table_config)
     uint32_t i;
     uint64_t key_range_start;
 
+    uint64_t pages_reviewed, pages_rewritten, pages_selected;
+
     first_ckpt = false;
 
     testutil_check(wiredtiger_open(home, &event_handler, conn_config, &conn));
@@ -258,6 +262,12 @@ workload_compact(const char *home, const char *table_config)
 
         log_db_size(session, uri1);
 
+        /* If we made progress with compact, verify that compact stats support that. */
+        get_compact_progress(session, uri1, &pages_reviewed, &pages_rewritten, &pages_selected);
+        printf(" - Pages reviewed: %" PRIu64 "\n", pages_reviewed);
+        printf(" - Pages selected for being rewritten: %" PRIu64 "\n", pages_selected);
+        printf(" - Pages actually rewritten: %" PRIu64 "\n", pages_rewritten);
+
         /* Put the deleted records back. */
         populate(session, key_range_start, key_range_start + NUM_RECORDS / 3);
 
@@ -412,3 +422,28 @@ log_db_size(WT_SESSION *session, const char *uri)
            "MB (%" PRIu64 "B)\n - %" PRIu64 "%% space available in the file.\n",
       file_sz / WT_MEGABYTE, file_sz, avail_bytes / WT_MEGABYTE, avail_bytes, available_pct);
 }
+
+static void
+get_compact_progress(WT_SESSION *session, const char *uri, uint64_t *pages_reviewed,
+  uint64_t *pages_selected, uint64_t *pages_rewritten)
+{
+
+    WT_CURSOR *cur_stat;
+    char *descr, *str_val;
+    char stat_uri[128];
+
+    sprintf(stat_uri, "statistics:%s", uri);
+    testutil_check(session->open_cursor(session, stat_uri, NULL, "statistics=(all)", &cur_stat));
+
+    cur_stat->set_key(cur_stat, WT_STAT_DSRC_BTREE_COMPACT_PAGES_REVIEWED);
+    testutil_check(cur_stat->search(cur_stat));
+    testutil_check(cur_stat->get_value(cur_stat, &descr, &str_val, pages_reviewed));
+    cur_stat->set_key(cur_stat, WT_STAT_DSRC_BTREE_COMPACT_PAGES_WRITE_SELECTED);
+    testutil_check(cur_stat->search(cur_stat));
+    testutil_check(cur_stat->get_value(cur_stat, &descr, &str_val, pages_selected));
+    cur_stat->set_key(cur_stat, WT_STAT_DSRC_BTREE_COMPACT_PAGES_REWRITTEN);
+    testutil_check(cur_stat->search(cur_stat));
+    testutil_check(cur_stat->get_value(cur_stat, &descr, &str_val, pages_rewritten));
+
+    testutil_check(cur_stat->close(cur_stat));
+}
diff --git a/src/third_party/wiredtiger/test/suite/test_compact01.py b/src/third_party/wiredtiger/test/suite/test_compact01.py
index b5a21ce96a6..789428a1de4 100755
--- a/src/third_party/wiredtiger/test/suite/test_compact01.py
+++ b/src/third_party/wiredtiger/test/suite/test_compact01.py
@@ -60,6 +60,18 @@ class test_compact(wttest.WiredTigerTestCase, suite_subprocess):
     conn_config = 'cache_size=1GB,eviction_checkpoint_target=80,' +\
         'eviction_dirty_target=80,eviction_dirty_trigger=95,statistics=(all)'
 
+    # Return stats that track the progress of compaction.
+    def getCompactProgressStats(self, uri):
+        cstat = self.session.open_cursor(
+            'statistics:' + uri, None, 'statistics=(all)')
+        statDict = {}
+        statDict["pages_reviewed"] = cstat[stat.dsrc.btree_compact_pages_reviewed][2]
+        statDict["pages_skipped"] = cstat[stat.dsrc.btree_compact_pages_skipped][2]
+        statDict["pages_selected"] = cstat[stat.dsrc.btree_compact_pages_write_selected][2]
+        statDict["pages_rewritten"] = cstat[stat.dsrc.btree_compact_pages_rewritten][2]
+        cstat.close()
+        return statDict
+
     # Test compaction.
     def test_compact(self):
         # FIXME-WT-7187
@@ -103,6 +115,14 @@ class test_compact(wttest.WiredTigerTestCase, suite_subprocess):
 
             self.session.compact(uri, None)
 
+        # Verify compact progress stats. We can't do this with utility method as reopening the
+        # connection would reset the stats.
+        if self.utility == 0 and self.reopen == 0:
+            statDict = self.getCompactProgressStats(uri)
+            self.assertGreater(statDict["pages_reviewed"],0)
+            self.assertGreater(statDict["pages_selected"],0)
+            self.assertGreater(statDict["pages_rewritten"],0)
+
         # Confirm compaction worked: check the number of on-disk pages
         self.reopen_conn()
         stat_cursor = self.session.open_cursor('statistics:' + uri, None, None)
diff --git a/src/third_party/wiredtiger/test/suite/test_compact02.py b/src/third_party/wiredtiger/test/suite/test_compact02.py
index a0b162dbaf2..6366c7d6da2 100755
--- a/src/third_party/wiredtiger/test/suite/test_compact02.py
+++ b/src/third_party/wiredtiger/test/suite/test_compact02.py
@@ -79,6 +79,18 @@ class test_compact02(wttest.WiredTigerTestCase):
 
     fullsize = nrecords // 2 * len(bigvalue) + nrecords // 2 * len(smallvalue)
 
+    # Return stats that track the progress of compaction.
+    def getCompactProgressStats(self):
+        cstat = self.session.open_cursor(
+            'statistics:' + self.uri, None, 'statistics=(all)')
+        statDict = {}
+        statDict["pages_reviewed"] = cstat[stat.dsrc.btree_compact_pages_reviewed][2]
+        statDict["pages_skipped"] = cstat[stat.dsrc.btree_compact_pages_skipped][2]
+        statDict["pages_selected"] = cstat[stat.dsrc.btree_compact_pages_write_selected][2]
+        statDict["pages_rewritten"] = cstat[stat.dsrc.btree_compact_pages_rewritten][2]
+        cstat.close()
+        return statDict
+
     # Return the size of the file
     def getSize(self):
         # To allow this to work on systems without ftruncate,
@@ -171,5 +183,11 @@ class test_compact02(wttest.WiredTigerTestCase):
         # After compact, the file size should be less than half the full size.
         self.assertLess(sz, self.fullsize // 2)
 
+        # Verify compact progress stats.
+        statDict = self.getCompactProgressStats()
+        self.assertGreater(statDict["pages_reviewed"],0)
+        self.assertGreater(statDict["pages_selected"],0)
+        self.assertGreater(statDict["pages_rewritten"],0)
+
 if __name__ == '__main__':
     wttest.run()
diff --git a/src/third_party/wiredtiger/test/suite/test_compact03.py b/src/third_party/wiredtiger/test/suite/test_compact03.py
index 732bd39a597..0c263b794bb 100644
--- a/src/third_party/wiredtiger/test/suite/test_compact03.py
+++ b/src/third_party/wiredtiger/test/suite/test_compact03.py
@@ -79,6 +79,18 @@ class test_compact03(wttest.WiredTigerTestCase):
     expectedTableSize = 20 # Mbytes
     nOverflowRecords = 5000
 
+    # Return stats that track the progress of compaction.
+    def getCompactProgressStats(self):
+        cstat = self.session.open_cursor(
+            'statistics:' + self.uri, None, 'statistics=(all)')
+        statDict = {}
+        statDict["pages_reviewed"] = cstat[stat.dsrc.btree_compact_pages_reviewed][2]
+        statDict["pages_skipped"] = cstat[stat.dsrc.btree_compact_pages_skipped][2]
+        statDict["pages_selected"] = cstat[stat.dsrc.btree_compact_pages_write_selected][2]
+        statDict["pages_rewritten"] = cstat[stat.dsrc.btree_compact_pages_rewritten][2]
+        cstat.close()
+        return statDict
+
     # Return the size of the file
     def getSize(self):
         # To allow this to work on systems without ftruncate,
@@ -147,6 +159,12 @@ class test_compact03(wttest.WiredTigerTestCase):
         self.pr('After deleting values and compactions ' + str(sizeAfterCompact // mb) + 'MB')
         self.assertGreater(sizeAfterCompact, (sizeWithOverflow // 10) * 9)
 
+        # Verify that we did indeed rewrote some pages but that didn't help with the file size.
+        statDict = self.getCompactProgressStats()
+        self.assertGreater(statDict["pages_reviewed"],0)
+        self.assertGreater(statDict["pages_selected"],0)
+        self.assertGreater(statDict["pages_rewritten"],0)
+
         # 9. Insert some normal values and expect that file size won't increase as free extents
         #    in the middle of the file will be used to write new data.