Import wiredtiger: f1ef795c8451ec6a9a93a13832290fa469102cf4 from branch mongodb-master

ref: 2a727360bb..f1ef795c84
for: 5.2.0

WT-8188       Use compact progress stats in compact related tests

6 files changed, 131 insertions, 2 deletions

diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data
index 5ef6b0da0d3..b60d5cc1939 100644
--- a/src/third_party/wiredtiger/import.data
+++ b/src/third_party/wiredtiger/import.data
@@ -2,5 +2,5 @@
     "vendor": "wiredtiger",
     "github": "wiredtiger/wiredtiger.git",
     "branch": "mongodb-master",
-    "commit": "2a727360bb291eb6523845e3a8de9d769c1e28b0"
+    "commit": "f1ef795c8451ec6a9a93a13832290fa469102cf4"
 }
diff --git a/src/third_party/wiredtiger/test/csuite/wt7989_compact_checkpoint/main.c b/src/third_party/wiredtiger/test/csuite/wt7989_compact_checkpoint/main.c
index 8cd8c4b3e77..15f7a2fc2f0 100644
--- a/src/third_party/wiredtiger/test/csuite/wt7989_compact_checkpoint/main.c
+++ b/src/third_party/wiredtiger/test/csuite/wt7989_compact_checkpoint/main.c
@@ -65,6 +65,8 @@ static void populate(WT_SESSION *session, const char *uri);
 static void remove_records(WT_SESSION *session, const char *uri);
 static uint64_t get_file_size(WT_SESSION *session, const char *uri);
 static void set_timing_stress_checkpoint(WT_CONNECTION *conn);
+static void get_compact_progress(WT_SESSION *session, const char *uri, uint64_t *pages_reviewed,
+  uint64_t *pages_selected, uint64_t *pages_rewritten);
 
 /* Methods implementation. */
 int
@@ -108,6 +110,7 @@ run_test(bool stress_test, const char *home, const char *uri)
     WT_SESSION *session;
     pthread_t thread_checkpoint;
     uint64_t file_sz_after, file_sz_before;
+    uint64_t pages_reviewed, pages_rewritten, pages_selected;
 
     testutil_make_work_dir(home);
     testutil_check(wiredtiger_open(home, NULL, conn_config, &conn));
@@ -157,6 +160,8 @@ run_test(bool stress_test, const char *home, const char *uri)
     (void)pthread_join(thread_compact, NULL);
 
     file_sz_after = get_file_size(session, uri);
+    /* Collect compact progress stats. */
+    get_compact_progress(session, uri, &pages_reviewed, &pages_selected, &pages_rewritten);
 
     /* Cleanup */
     if (!stress_test) {
@@ -174,8 +179,15 @@ run_test(bool stress_test, const char *home, const char *uri)
     printf(" - Compressed file size MB: %f\n - Original file size MB: %f\n",
       file_sz_after / (1024.0 * 1024), file_sz_before / (1024.0 * 1024));
 
+    printf(" - Pages reviewed: %lu \n", pages_reviewed);
+    printf(" - Pages selected for being rewritten: %lu \n", pages_selected);
+    printf(" - Pages actually rewritten: %lu \n", pages_rewritten);
+
     /* Make sure the compact operation has reduced the file size by at least 20%. */
     testutil_assert((file_sz_before / 100) * 80 > file_sz_after);
+    testutil_assert(pages_reviewed > 0);
+    testutil_assert(pages_selected > 0);
+    testutil_assert(pages_rewritten > 0);
 }
 
 static void *
@@ -341,3 +353,28 @@ set_timing_stress_checkpoint(WT_CONNECTION *conn)
     conn_impl = (WT_CONNECTION_IMPL *)conn;
     conn_impl->timing_stress_flags |= WT_TIMING_STRESS_CHECKPOINT_SLOW;
 }
+
+static void
+get_compact_progress(WT_SESSION *session, const char *uri, uint64_t *pages_reviewed,
+  uint64_t *pages_selected, uint64_t *pages_rewritten)
+{
+
+    WT_CURSOR *cur_stat;
+    char *descr, *str_val;
+    char stat_uri[128];
+
+    sprintf(stat_uri, "statistics:%s", uri);
+    testutil_check(session->open_cursor(session, stat_uri, NULL, "statistics=(all)", &cur_stat));
+
+    cur_stat->set_key(cur_stat, WT_STAT_DSRC_BTREE_COMPACT_PAGES_REVIEWED);
+    testutil_check(cur_stat->search(cur_stat));
+    testutil_check(cur_stat->get_value(cur_stat, &descr, &str_val, pages_reviewed));
+    cur_stat->set_key(cur_stat, WT_STAT_DSRC_BTREE_COMPACT_PAGES_WRITE_SELECTED);
+    testutil_check(cur_stat->search(cur_stat));
+    testutil_check(cur_stat->get_value(cur_stat, &descr, &str_val, pages_selected));
+    cur_stat->set_key(cur_stat, WT_STAT_DSRC_BTREE_COMPACT_PAGES_REWRITTEN);
+    testutil_check(cur_stat->search(cur_stat));
+    testutil_check(cur_stat->get_value(cur_stat, &descr, &str_val, pages_rewritten));
+
+    testutil_check(cur_stat->close(cur_stat));
+}
diff --git a/src/third_party/wiredtiger/test/csuite/wt8057_compact_stress/main.c b/src/third_party/wiredtiger/test/csuite/wt8057_compact_stress/main.c
index 1a495ccf06f..307310503fd 100644
--- a/src/third_party/wiredtiger/test/csuite/wt8057_compact_stress/main.c
+++ b/src/third_party/wiredtiger/test/csuite/wt8057_compact_stress/main.c
@@ -85,7 +85,8 @@ static void remove_records(WT_SESSION *session, const char *uri, int start, int
 static void verify_tables(WT_SESSION *session);
 static int verify_tables_helper(WT_SESSION *session, const char *table1, const char *table2);
 static uint64_t get_file_size(WT_SESSION *session, const char *uri);
-
+static void get_compact_progress(WT_SESSION *session, const char *uri, uint64_t *pages_reviewed,
+  uint64_t *pages_selected, uint64_t *pages_rewritten);
 /*
  * Signal handler to catch if the child died unexpectedly.
  */
@@ -190,6 +191,8 @@ run_test(const char *home)
     time_t t;
     uint64_t file_sz_after, file_sz_before;
 
+    uint64_t pages_reviewed, pages_rewritten, pages_selected;
+
     first_ckpt = false;
     srand((u_int)time(&t));
 
@@ -239,6 +242,14 @@ run_test(const char *home)
         printf(" - Compressed file size MB: %f\n - Original file size MB: %f\n",
           file_sz_after / (1024.0 * 1024), file_sz_before / (1024.0 * 1024));
 
+        /* If we made progress with compact, verify that compact stats support that. */
+        if (file_sz_after < file_sz_before) {
+            get_compact_progress(session, uri1, &pages_reviewed, &pages_rewritten, &pages_selected);
+            printf(" - Pages reviewed: %lu \n", pages_reviewed);
+            printf(" - Pages selected for being rewritten: %lu \n", pages_selected);
+            printf(" - Pages actually rewritten: %lu \n", pages_rewritten);
+        }
+
         /* Put the deleted records back. */
         populate(session, key_range_start, key_range_start + NUM_RECORDS / 3);
 
@@ -378,3 +389,28 @@ get_file_size(WT_SESSION *session, const char *uri)
 
     return (val);
 }
+
+static void
+get_compact_progress(WT_SESSION *session, const char *uri, uint64_t *pages_reviewed,
+  uint64_t *pages_selected, uint64_t *pages_rewritten)
+{
+
+    WT_CURSOR *cur_stat;
+    char *descr, *str_val;
+    char stat_uri[128];
+
+    sprintf(stat_uri, "statistics:%s", uri);
+    testutil_check(session->open_cursor(session, stat_uri, NULL, "statistics=(all)", &cur_stat));
+
+    cur_stat->set_key(cur_stat, WT_STAT_DSRC_BTREE_COMPACT_PAGES_REVIEWED);
+    testutil_check(cur_stat->search(cur_stat));
+    testutil_check(cur_stat->get_value(cur_stat, &descr, &str_val, pages_reviewed));
+    cur_stat->set_key(cur_stat, WT_STAT_DSRC_BTREE_COMPACT_PAGES_WRITE_SELECTED);
+    testutil_check(cur_stat->search(cur_stat));
+    testutil_check(cur_stat->get_value(cur_stat, &descr, &str_val, pages_selected));
+    cur_stat->set_key(cur_stat, WT_STAT_DSRC_BTREE_COMPACT_PAGES_REWRITTEN);
+    testutil_check(cur_stat->search(cur_stat));
+    testutil_check(cur_stat->get_value(cur_stat, &descr, &str_val, pages_rewritten));
+
+    testutil_check(cur_stat->close(cur_stat));
+}
diff --git a/src/third_party/wiredtiger/test/suite/test_compact01.py b/src/third_party/wiredtiger/test/suite/test_compact01.py
index b5a21ce96a6..3bb5b187dfc 100755
--- a/src/third_party/wiredtiger/test/suite/test_compact01.py
+++ b/src/third_party/wiredtiger/test/suite/test_compact01.py
@@ -60,6 +60,18 @@ class test_compact(wttest.WiredTigerTestCase, suite_subprocess):
     conn_config = 'cache_size=1GB,eviction_checkpoint_target=80,' +\
         'eviction_dirty_target=80,eviction_dirty_trigger=95,statistics=(all)'
 
+    # Return stats that track the progress of compaction.
+    def getCompactProgressStats(self, uri):
+        cstat = self.session.open_cursor(
+            'statistics:' + uri, None, 'statistics=(all)')
+        statDict = {}
+        statDict["pages_reviewed"] = cstat[stat.dsrc.btree_compact_pages_reviewed][2]
+        statDict["pages_skipped"] = cstat[stat.dsrc.btree_compact_pages_skipped][2]
+        statDict["pages_selected"] = cstat[stat.dsrc.btree_compact_pages_write_selected][2]
+        statDict["pages_rewritten"] = cstat[stat.dsrc.btree_compact_pages_rewritten][2]
+        cstat.close()
+        return statDict
+
     # Test compaction.
     def test_compact(self):
         # FIXME-WT-7187
@@ -103,6 +115,14 @@ class test_compact(wttest.WiredTigerTestCase, suite_subprocess):
 
             self.session.compact(uri, None)
 
+        # Verify compact progress stats. We can't do this with utility method as reopening the
+        # connection would reset the stats.
+        if self.utility != 1:
+            statDict = self.getCompactProgressStats(uri)
+            self.assertGreater(statDict["pages_reviewed"],0)
+            self.assertGreater(statDict["pages_selected"],0)
+            self.assertGreater(statDict["pages_rewritten"],0)
+
         # Confirm compaction worked: check the number of on-disk pages
         self.reopen_conn()
         stat_cursor = self.session.open_cursor('statistics:' + uri, None, None)
diff --git a/src/third_party/wiredtiger/test/suite/test_compact02.py b/src/third_party/wiredtiger/test/suite/test_compact02.py
index 81d636eee6b..2ad52b5c1b1 100755
--- a/src/third_party/wiredtiger/test/suite/test_compact02.py
+++ b/src/third_party/wiredtiger/test/suite/test_compact02.py
@@ -79,6 +79,18 @@ class test_compact02(wttest.WiredTigerTestCase):
 
     fullsize = nrecords // 2 * len(bigvalue) + nrecords // 2 * len(smallvalue)
 
+    # Return stats that track the progress of compaction.
+    def getCompactProgressStats(self):
+        cstat = self.session.open_cursor(
+            'statistics:' + self.uri, None, 'statistics=(all)')
+        statDict = {}
+        statDict["pages_reviewed"] = cstat[stat.dsrc.btree_compact_pages_reviewed][2]
+        statDict["pages_skipped"] = cstat[stat.dsrc.btree_compact_pages_skipped][2]
+        statDict["pages_selected"] = cstat[stat.dsrc.btree_compact_pages_write_selected][2]
+        statDict["pages_rewritten"] = cstat[stat.dsrc.btree_compact_pages_rewritten][2]
+        cstat.close()
+        return statDict
+
     # Return the size of the file
     def getSize(self):
         # To allow this to work on systems without ftruncate,
@@ -168,5 +180,11 @@ class test_compact02(wttest.WiredTigerTestCase):
         # After compact, the file size should be less than half the full size.
         self.assertLess(sz, self.fullsize // 2)
 
+        # Verify compact progress stats.
+        statDict = self.getCompactProgressStats()
+        self.assertGreater(statDict["pages_reviewed"],0)
+        self.assertGreater(statDict["pages_selected"],0)
+        self.assertGreater(statDict["pages_rewritten"],0)
+
 if __name__ == '__main__':
     wttest.run()
diff --git a/src/third_party/wiredtiger/test/suite/test_compact03.py b/src/third_party/wiredtiger/test/suite/test_compact03.py
index 732bd39a597..0c263b794bb 100644
--- a/src/third_party/wiredtiger/test/suite/test_compact03.py
+++ b/src/third_party/wiredtiger/test/suite/test_compact03.py
@@ -79,6 +79,18 @@ class test_compact03(wttest.WiredTigerTestCase):
     expectedTableSize = 20 # Mbytes
     nOverflowRecords = 5000
 
+    # Return stats that track the progress of compaction.
+    def getCompactProgressStats(self):
+        cstat = self.session.open_cursor(
+            'statistics:' + self.uri, None, 'statistics=(all)')
+        statDict = {}
+        statDict["pages_reviewed"] = cstat[stat.dsrc.btree_compact_pages_reviewed][2]
+        statDict["pages_skipped"] = cstat[stat.dsrc.btree_compact_pages_skipped][2]
+        statDict["pages_selected"] = cstat[stat.dsrc.btree_compact_pages_write_selected][2]
+        statDict["pages_rewritten"] = cstat[stat.dsrc.btree_compact_pages_rewritten][2]
+        cstat.close()
+        return statDict
+
     # Return the size of the file
     def getSize(self):
         # To allow this to work on systems without ftruncate,
@@ -147,6 +159,12 @@ class test_compact03(wttest.WiredTigerTestCase):
         self.pr('After deleting values and compactions ' + str(sizeAfterCompact // mb) + 'MB')
         self.assertGreater(sizeAfterCompact, (sizeWithOverflow // 10) * 9)
 
+        # Verify that we did indeed rewrote some pages but that didn't help with the file size.
+        statDict = self.getCompactProgressStats()
+        self.assertGreater(statDict["pages_reviewed"],0)
+        self.assertGreater(statDict["pages_selected"],0)
+        self.assertGreater(statDict["pages_rewritten"],0)
+
         # 9. Insert some normal values and expect that file size won't increase as free extents
         #    in the middle of the file will be used to write new data.