WT-2665 Limit allocator fragmentation from the WiredTiger cache (#2799)

* Add a statistic for the amount of cache used for page images vs other objects

* Reset the "maximum page size" statistic each checkpoint

* Add a separate queue for urgent eviction, replaces existing WOULD_BLOCK eviction mode

Until now, the eviction server has had to walk the cache to find pages that
would otherwise block application threads.  This change allows threads to put
those pages directly on a queue that is checked before ordinary LRU eviction.

* Don't queue pages for urgent eviction when eviction is disabled in a tree.

Decouple evict_queue_lock and the individual queue's evict_lock in
__evict_get_ref.  Don't block the eviction server if there are many
application threads waiting on a queue.

* Take care with the urgent queue.

Entries in the urgent queue cannot be skipped or they will not be considered again.

3 files changed, 20 insertions, 16 deletions

diff --git a/dist/api_data.py b/dist/api_data.py
index 90b1c8378a2..960f800a44d 100644
--- a/dist/api_data.py
+++ b/dist/api_data.py
@@ -388,6 +388,21 @@ connection_runtime_config = [
         ]),
     Config('error_prefix', '', r'''
         prefix string for error messages'''),
+    Config('eviction', '', r'''
+        eviction configuration options.''',
+        type='category', subconfig=[
+            Config('threads_max', '1', r'''
+                maximum number of threads WiredTiger will start to help evict
+                pages from cache. The number of threads started will vary
+                depending on the current eviction load. Each eviction worker
+                thread uses a session from the configured session_max''',
+                min=1, max=20),
+            Config('threads_min', '1', r'''
+                minimum number of threads WiredTiger will start to help evict
+                pages from cache. The number of threads currently running will
+                vary depending on the current eviction load''',
+                min=1, max=20),
+            ]),
     Config('eviction_dirty_target', '80', r'''
         continue evicting until the cache has less dirty memory than the
         value, as a percentage of the total cache size. Dirty pages will
@@ -472,21 +487,6 @@ connection_runtime_config = [
     Config('lsm_merge', 'true', r'''
         merge LSM chunks where possible (deprecated)''',
         type='boolean', undoc=True),
-    Config('eviction', '', r'''
-        eviction configuration options.''',
-        type='category', subconfig=[
-            Config('threads_max', '1', r'''
-                maximum number of threads WiredTiger will start to help evict
-                pages from cache. The number of threads started will vary
-                depending on the current eviction load. Each eviction worker
-                thread uses a session from the configured session_max''',
-                min=1, max=20),
-            Config('threads_min', '1', r'''
-                minimum number of threads WiredTiger will start to help evict
-                pages from cache. The number of threads currently running will
-                vary depending on the current eviction load''',
-                min=1, max=20),
-            ]),
     Config('shared_cache', '', r'''
         shared cache configuration options. A database should configure
         either a cache_size or a shared_cache not both. Enabling a
diff --git a/dist/flags.py b/dist/flags.py
index b5f36fb707a..b661fa348fa 100644
--- a/dist/flags.py
+++ b/dist/flags.py
@@ -37,10 +37,12 @@ flags = {
         'READ_WONT_NEED',
     ],
     'rec_write' : [
+        'EVICTING',
+        'EVICT_CLEAN',
         'EVICT_IN_MEMORY',
+        'EVICT_INMEM_SPLIT',
         'EVICT_LOOKASIDE',
         'EVICT_UPDATE_RESTORE',
-        'EVICTING',
         'VISIBILITY_ERR',
     ],
     'txn_log_checkpoint' : [
diff --git a/dist/stat_data.py b/dist/stat_data.py
index 2dc828d8afb..4721f3e629f 100644
--- a/dist/stat_data.py
+++ b/dist/stat_data.py
@@ -168,10 +168,12 @@ connection_stats = [
     # Cache and eviction statistics
     ##########################################
     CacheStat('cache_bytes_dirty', 'tracked dirty bytes in the cache', 'no_clear,no_scale,size'),
+    CacheStat('cache_bytes_image', 'bytes belonging to page images in the cache', 'no_clear,no_scale,size'),
     CacheStat('cache_bytes_internal', 'tracked bytes belonging to internal pages in the cache', 'no_clear,no_scale,size'),
     CacheStat('cache_bytes_inuse', 'bytes currently in the cache', 'no_clear,no_scale,size'),
     CacheStat('cache_bytes_leaf', 'tracked bytes belonging to leaf pages in the cache', 'no_clear,no_scale,size'),
     CacheStat('cache_bytes_max', 'maximum bytes configured', 'no_clear,no_scale,size'),
+    CacheStat('cache_bytes_other', 'bytes not belonging to page images in the cache', 'no_clear,no_scale,size'),
     CacheStat('cache_bytes_overflow', 'tracked bytes belonging to overflow pages in the cache', 'no_clear,no_scale,size'),
     CacheStat('cache_bytes_read', 'bytes read into cache', 'size'),
     CacheStat('cache_bytes_write', 'bytes written from cache', 'size'),