WT-2764 Replace write leaves phase of checkpoints. (#2876)

Instead, get multi-threaded writes by dialling down the eviction dirty trigger.

Change eviction_dirty_* defaults (to 20/5).

Make sure all files are available for eviction before starting fsync.

Take more care excluding clean files from checkpoints.

Only use scrubbing mode when there is minimal cache pressure.

Avoid penalizing read-only operations with dirty eviction targets.

Enable scrubbing when eviction is keeping space available in cache.

Add stats for checkpoint scrubbing phase.

Improve cache scrubbing with big caches and slow I/O.

Use the rate of bytes written from cache to decide how long to wait for
the dirty bytes to come down.

3 files changed, 10 insertions, 8 deletions

diff --git a/dist/api_data.py b/dist/api_data.py
index e1b877a1a35..1302247e88e 100644
--- a/dist/api_data.py
+++ b/dist/api_data.py
@@ -401,16 +401,16 @@ connection_runtime_config = [
                 vary depending on the current eviction load''',
                 min=1, max=20),
             ]),
-    Config('eviction_dirty_target', '80', r'''
+    Config('eviction_dirty_target', '5', r'''
         continue evicting until the cache has less dirty memory than the
         value, as a percentage of the total cache size. Dirty pages will
         only be evicted if the cache is full enough to trigger eviction''',
-        min=5, max=99),
-    Config('eviction_dirty_trigger', '95', r'''
+        min=1, max=99),
+    Config('eviction_dirty_trigger', '20', r'''
         trigger eviction when the cache is using this much memory for dirty
         content, as a percentage of the total cache size. This setting only
         alters behavior if it is lower than eviction_trigger''',
-        min=5, max=99),
+        min=1, max=99),
     Config('eviction_target', '80', r'''
         continue evicting until the cache has less total memory than the
         value, as a percentage of the total cache size. Must be less than
diff --git a/dist/s_string.ok b/dist/s_string.ok
index 951faf61f30..8c5f1e99bff 100644
--- a/dist/s_string.ok
+++ b/dist/s_string.ok
@@ -938,6 +938,7 @@ prepend
 prepended
 prepending
 presize
+presync
 primary's
 printf
 printlog
diff --git a/dist/stat_data.py b/dist/stat_data.py
index e074eacefe5..51cc487f04c 100644
--- a/dist/stat_data.py
+++ b/dist/stat_data.py
@@ -195,7 +195,8 @@ connection_stats = [
     CacheStat('cache_eviction_internal', 'internal pages evicted'),
     CacheStat('cache_eviction_maximum_page_size', 'maximum page size at eviction', 'no_clear,no_scale,size'),
     CacheStat('cache_eviction_pages_queued', 'pages queued for eviction'),
-    CacheStat('cache_eviction_pages_queued_oldest', 'pages queued for urgent eviction'),
+    CacheStat('cache_eviction_pages_queued_oldest', 'pages queued for urgent eviction during walk'),
+    CacheStat('cache_eviction_pages_queued_urgent', 'pages queued for urgent eviction'),
     CacheStat('cache_eviction_pages_seen', 'pages seen by eviction walk'),
     CacheStat('cache_eviction_queue_empty', 'eviction server candidate queue empty when topping up'),
     CacheStat('cache_eviction_queue_not_empty', 'eviction server candidate queue not empty when topping up'),
@@ -298,11 +299,11 @@ connection_stats = [
     TxnStat('txn_begin', 'transaction begins'),
     TxnStat('txn_checkpoint', 'transaction checkpoints'),
     TxnStat('txn_checkpoint_fsync_post', 'transaction fsync calls for checkpoint after allocating the transaction ID'),
-    TxnStat('txn_checkpoint_fsync_post_duration', 'transaction fsync duration for checkpoint after allocating the transaction ID (usecs)'),
-    TxnStat('txn_checkpoint_fsync_pre', 'transaction fsync calls for checkpoint before allocating the transaction ID'),
-    TxnStat('txn_checkpoint_fsync_pre_duration', 'transaction fsync duration for checkpoint before allocating the transaction ID (usecs)'),
+    TxnStat('txn_checkpoint_fsync_post_duration', 'transaction fsync duration for checkpoint after allocating the transaction ID (usecs)', 'no_clear,no_scale'),
     TxnStat('txn_checkpoint_generation', 'transaction checkpoint generation', 'no_clear,no_scale'),
     TxnStat('txn_checkpoint_running', 'transaction checkpoint currently running', 'no_clear,no_scale'),
+    TxnStat('txn_checkpoint_scrub_target', 'transaction checkpoint scrub dirty target', 'no_clear,no_scale'),
+    TxnStat('txn_checkpoint_scrub_time', 'transaction checkpoint scrub time (msecs)', 'no_clear,no_scale'),
     TxnStat('txn_checkpoint_time_max', 'transaction checkpoint max time (msecs)', 'no_clear,no_scale'),
     TxnStat('txn_checkpoint_time_min', 'transaction checkpoint min time (msecs)', 'no_clear,no_scale'),
     TxnStat('txn_checkpoint_time_recent', 'transaction checkpoint most recent time (msecs)', 'no_clear,no_scale'),