WT-2932 Add a configuration option allowing tables to ignore cache limits (#3069)

Only available for in-memory configurations, designed to be used for
applications that maintain a small amount of content that is required
for correctness.

7 files changed, 140 insertions, 36 deletions

diff --git a/dist/api_data.py b/dist/api_data.py
index 4e0cc9bdb06..d02d7e4b985 100644
--- a/dist/api_data.py
+++ b/dist/api_data.py
@@ -183,6 +183,12 @@ file_config = format_meta + [
         configure Huffman encoding for values.  Permitted values are
         \c "none", \c "english", \c "utf8<file>" or \c "utf16<file>".
         See @ref huffman for more information'''),
+    Config('ignore_in_memory_cache_size', 'false', r'''
+        allow update and insert operations to proceed even if the cache is
+        already at capacity. Only valid in conjunction with in-memory
+        databases. Should be used with caution - this configuration allows
+        WiredTiger to consume memory over the configured cache limit''',
+        type='boolean'),
     Config('internal_key_truncate', 'true', r'''
         configure internal key truncation, discarding unnecessary
         trailing bytes on internal keys (ignored for custom
diff --git a/src/btree/bt_handle.c b/src/btree/bt_handle.c
index 7887a8bcf35..337a3ea036f 100644
--- a/src/btree/bt_handle.c
+++ b/src/btree/bt_handle.c
@@ -271,6 +271,17 @@ __btree_conf(WT_SESSION_IMPL *session, WT_CKPT *ckpt)
 	else
 		F_CLR(btree, WT_BTREE_IN_MEMORY | WT_BTREE_NO_EVICTION);
 
+	WT_RET(__wt_config_gets(session,
+	    cfg, "ignore_in_memory_cache_size", &cval));
+	if (cval.val) {
+		if (!F_ISSET(conn, WT_CONN_IN_MEMORY))
+			WT_RET_MSG(session, EINVAL,
+			    "ignore_in_memory_cache_size setting is only valid "
+			    "with databases configured to run in-memory");
+		F_SET(btree, WT_BTREE_IGNORE_CACHE);
+	} else
+		F_CLR(btree, WT_BTREE_IGNORE_CACHE);
+
 	WT_RET(__wt_config_gets(session, cfg, "log.enabled", &cval));
 	if (cval.val)
 		F_CLR(btree, WT_BTREE_NO_LOGGING);
diff --git a/src/config/config_def.c b/src/config/config_def.c
index 7bad5f12a9f..7bce4bc9cef 100644
--- a/src/config/config_def.c
+++ b/src/config/config_def.c
@@ -246,6 +246,9 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_create[] = {
 	{ "format", "string", NULL, "choices=[\"btree\"]", NULL, 0 },
 	{ "huffman_key", "string", NULL, NULL, NULL, 0 },
 	{ "huffman_value", "string", NULL, NULL, NULL, 0 },
+	{ "ignore_in_memory_cache_size", "boolean",
+	    NULL, NULL,
+	    NULL, 0 },
 	{ "immutable", "boolean", NULL, NULL, NULL, 0 },
 	{ "internal_item_max", "int", NULL, "min=0", NULL, 0 },
 	{ "internal_key_max", "int", NULL, "min=0", NULL, 0 },
@@ -413,6 +416,9 @@ static const WT_CONFIG_CHECK confchk_file_config[] = {
 	{ "format", "string", NULL, "choices=[\"btree\"]", NULL, 0 },
 	{ "huffman_key", "string", NULL, NULL, NULL, 0 },
 	{ "huffman_value", "string", NULL, NULL, NULL, 0 },
+	{ "ignore_in_memory_cache_size", "boolean",
+	    NULL, NULL,
+	    NULL, 0 },
 	{ "internal_item_max", "int", NULL, "min=0", NULL, 0 },
 	{ "internal_key_max", "int", NULL, "min=0", NULL, 0 },
 	{ "internal_key_truncate", "boolean", NULL, NULL, NULL, 0 },
@@ -471,6 +477,9 @@ static const WT_CONFIG_CHECK confchk_file_meta[] = {
 	{ "huffman_key", "string", NULL, NULL, NULL, 0 },
 	{ "huffman_value", "string", NULL, NULL, NULL, 0 },
 	{ "id", "string", NULL, NULL, NULL, 0 },
+	{ "ignore_in_memory_cache_size", "boolean",
+	    NULL, NULL,
+	    NULL, 0 },
 	{ "internal_item_max", "int", NULL, "min=0", NULL, 0 },
 	{ "internal_key_max", "int", NULL, "min=0", NULL, 0 },
 	{ "internal_key_truncate", "boolean", NULL, NULL, NULL, 0 },
@@ -544,6 +553,9 @@ static const WT_CONFIG_CHECK confchk_lsm_meta[] = {
 	{ "format", "string", NULL, "choices=[\"btree\"]", NULL, 0 },
 	{ "huffman_key", "string", NULL, NULL, NULL, 0 },
 	{ "huffman_value", "string", NULL, NULL, NULL, 0 },
+	{ "ignore_in_memory_cache_size", "boolean",
+	    NULL, NULL,
+	    NULL, 0 },
 	{ "internal_item_max", "int", NULL, "min=0", NULL, 0 },
 	{ "internal_key_max", "int", NULL, "min=0", NULL, 0 },
 	{ "internal_key_truncate", "boolean", NULL, NULL, NULL, 0 },
@@ -1053,18 +1065,18 @@ static const WT_CONFIG_ENTRY config_entries[] = {
 	  "block_compressor=,cache_resident=false,checksum=uncompressed,"
 	  "colgroups=,collator=,columns=,dictionary=0,encryption=(keyid=,"
 	  "name=),exclusive=false,extractor=,format=btree,huffman_key=,"
-	  "huffman_value=,immutable=false,internal_item_max=0,"
-	  "internal_key_max=0,internal_key_truncate=true,"
-	  "internal_page_max=4KB,key_format=u,key_gap=10,leaf_item_max=0,"
-	  "leaf_key_max=0,leaf_page_max=32KB,leaf_value_max=0,"
-	  "log=(enabled=true),lsm=(auto_throttle=true,bloom=true,"
-	  "bloom_bit_count=16,bloom_config=,bloom_hash_count=8,"
+	  "huffman_value=,ignore_in_memory_cache_size=false,immutable=false"
+	  ",internal_item_max=0,internal_key_max=0,"
+	  "internal_key_truncate=true,internal_page_max=4KB,key_format=u,"
+	  "key_gap=10,leaf_item_max=0,leaf_key_max=0,leaf_page_max=32KB,"
+	  "leaf_value_max=0,log=(enabled=true),lsm=(auto_throttle=true,"
+	  "bloom=true,bloom_bit_count=16,bloom_config=,bloom_hash_count=8,"
 	  "bloom_oldest=false,chunk_count_limit=0,chunk_max=5GB,"
 	  "chunk_size=10MB,merge_max=15,merge_min=0),memory_page_max=5MB,"
 	  "os_cache_dirty_max=0,os_cache_max=0,prefix_compression=false,"
 	  "prefix_compression_min=4,source=,split_deepen_min_child=0,"
 	  "split_deepen_per_child=0,split_pct=75,type=file,value_format=u",
-	  confchk_WT_SESSION_create, 40
+	  confchk_WT_SESSION_create, 41
 	},
 	{ "WT_SESSION.drop",
 	  "checkpoint_wait=true,force=false,lock_wait=true,"
@@ -1148,7 +1160,8 @@ static const WT_CONFIG_ENTRY config_entries[] = {
 	  "allocation_size=4KB,app_metadata=,block_allocation=best,"
 	  "block_compressor=,cache_resident=false,checksum=uncompressed,"
 	  "collator=,columns=,dictionary=0,encryption=(keyid=,name=),"
-	  "format=btree,huffman_key=,huffman_value=,internal_item_max=0,"
+	  "format=btree,huffman_key=,huffman_value=,"
+	  "ignore_in_memory_cache_size=false,internal_item_max=0,"
 	  "internal_key_max=0,internal_key_truncate=true,"
 	  "internal_page_max=4KB,key_format=u,key_gap=10,leaf_item_max=0,"
 	  "leaf_key_max=0,leaf_page_max=32KB,leaf_value_max=0,"
@@ -1156,14 +1169,15 @@ static const WT_CONFIG_ENTRY config_entries[] = {
 	  "os_cache_max=0,prefix_compression=false,prefix_compression_min=4"
 	  ",split_deepen_min_child=0,split_deepen_per_child=0,split_pct=75,"
 	  "value_format=u",
-	  confchk_file_config, 33
+	  confchk_file_config, 34
 	},
 	{ "file.meta",
 	  "allocation_size=4KB,app_metadata=,block_allocation=best,"
 	  "block_compressor=,cache_resident=false,checkpoint=,"
 	  "checkpoint_lsn=,checksum=uncompressed,collator=,columns=,"
 	  "dictionary=0,encryption=(keyid=,name=),format=btree,huffman_key="
-	  ",huffman_value=,id=,internal_item_max=0,internal_key_max=0,"
+	  ",huffman_value=,id=,ignore_in_memory_cache_size=false,"
+	  "internal_item_max=0,internal_key_max=0,"
 	  "internal_key_truncate=true,internal_page_max=4KB,key_format=u,"
 	  "key_gap=10,leaf_item_max=0,leaf_key_max=0,leaf_page_max=32KB,"
 	  "leaf_value_max=0,log=(enabled=true),memory_page_max=5MB,"
@@ -1171,7 +1185,7 @@ static const WT_CONFIG_ENTRY config_entries[] = {
 	  "prefix_compression_min=4,split_deepen_min_child=0,"
 	  "split_deepen_per_child=0,split_pct=75,value_format=u,"
 	  "version=(major=0,minor=0)",
-	  confchk_file_meta, 37
+	  confchk_file_meta, 38
 	},
 	{ "index.meta",
 	  "app_metadata=,collator=,columns=,extractor=,immutable=false,"
@@ -1183,18 +1197,19 @@ static const WT_CONFIG_ENTRY config_entries[] = {
 	  "block_compressor=,cache_resident=false,checksum=uncompressed,"
 	  "chunks=,collator=,columns=,dictionary=0,encryption=(keyid=,"
 	  "name=),format=btree,huffman_key=,huffman_value=,"
-	  "internal_item_max=0,internal_key_max=0,"
-	  "internal_key_truncate=true,internal_page_max=4KB,key_format=u,"
-	  "key_gap=10,last=,leaf_item_max=0,leaf_key_max=0,"
-	  "leaf_page_max=32KB,leaf_value_max=0,log=(enabled=true),"
-	  "lsm=(auto_throttle=true,bloom=true,bloom_bit_count=16,"
-	  "bloom_config=,bloom_hash_count=8,bloom_oldest=false,"
-	  "chunk_count_limit=0,chunk_max=5GB,chunk_size=10MB,merge_max=15,"
-	  "merge_min=0),memory_page_max=5MB,old_chunks=,"
-	  "os_cache_dirty_max=0,os_cache_max=0,prefix_compression=false,"
-	  "prefix_compression_min=4,split_deepen_min_child=0,"
-	  "split_deepen_per_child=0,split_pct=75,value_format=u",
-	  confchk_lsm_meta, 37
+	  "ignore_in_memory_cache_size=false,internal_item_max=0,"
+	  "internal_key_max=0,internal_key_truncate=true,"
+	  "internal_page_max=4KB,key_format=u,key_gap=10,last=,"
+	  "leaf_item_max=0,leaf_key_max=0,leaf_page_max=32KB,"
+	  "leaf_value_max=0,log=(enabled=true),lsm=(auto_throttle=true,"
+	  "bloom=true,bloom_bit_count=16,bloom_config=,bloom_hash_count=8,"
+	  "bloom_oldest=false,chunk_count_limit=0,chunk_max=5GB,"
+	  "chunk_size=10MB,merge_max=15,merge_min=0),memory_page_max=5MB,"
+	  "old_chunks=,os_cache_dirty_max=0,os_cache_max=0,"
+	  "prefix_compression=false,prefix_compression_min=4,"
+	  "split_deepen_min_child=0,split_deepen_per_child=0,split_pct=75,"
+	  "value_format=u",
+	  confchk_lsm_meta, 38
 	},
 	{ "table.meta",
 	  "app_metadata=,colgroups=,collator=,columns=,key_format=u,"
diff --git a/src/include/api.h b/src/include/api.h
index e1b2f8edaf3..2783d17f825 100644
--- a/src/include/api.h
+++ b/src/include/api.h
@@ -139,7 +139,9 @@
 	(s) = (WT_SESSION_IMPL *)(cur)->session;			\
 	TXN_API_CALL_NOCONF(s, WT_CURSOR, n, cur,			\
 	    ((bt) == NULL) ? NULL : ((WT_BTREE *)(bt))->dhandle);	\
-	if (F_ISSET(S2C(s), WT_CONN_IN_MEMORY) && __wt_cache_full(s))	\
+	if (F_ISSET(S2C(s), WT_CONN_IN_MEMORY) &&			\
+	    !F_ISSET((WT_BTREE *)(bt), WT_BTREE_IGNORE_CACHE) &&	\
+	    __wt_cache_full(s))						\
 		WT_ERR(WT_CACHE_FULL);
 
 #define	JOINABLE_CURSOR_UPDATE_API_CALL(cur, s, n, bt)			\
diff --git a/src/include/btree.h b/src/include/btree.h
index 4756b19d338..713d46ae85f 100644
--- a/src/include/btree.h
+++ b/src/include/btree.h
@@ -154,18 +154,19 @@ struct __wt_btree {
 	WT_SPINLOCK	flush_lock;	/* Lock to flush the tree's pages */
 
 	/* Flags values up to 0xff are reserved for WT_DHANDLE_* */
-#define	WT_BTREE_BULK		0x00100	/* Bulk-load handle */
-#define	WT_BTREE_IN_MEMORY	0x00200	/* Cache-resident object */
-#define	WT_BTREE_LOOKASIDE	0x00400	/* Look-aside table */
-#define	WT_BTREE_NO_CHECKPOINT	0x00800	/* Disable checkpoints */
-#define	WT_BTREE_NO_EVICTION	0x01000	/* Disable eviction */
-#define	WT_BTREE_NO_LOGGING	0x02000	/* Disable logging */
-#define	WT_BTREE_NO_RECONCILE	0x04000 /* Allow splits, even with no evict */
-#define	WT_BTREE_REBALANCE	0x08000	/* Handle is for rebalance */
-#define	WT_BTREE_SALVAGE	0x10000	/* Handle is for salvage */
-#define	WT_BTREE_SKIP_CKPT	0x20000	/* Handle skipped checkpoint */
-#define	WT_BTREE_UPGRADE	0x40000	/* Handle is for upgrade */
-#define	WT_BTREE_VERIFY		0x80000	/* Handle is for verify */
+#define	WT_BTREE_BULK		0x000100 /* Bulk-load handle */
+#define	WT_BTREE_IGNORE_CACHE	0x000200 /* Cache-resident object */
+#define	WT_BTREE_IN_MEMORY	0x000400 /* Cache-resident object */
+#define	WT_BTREE_LOOKASIDE	0x000800 /* Look-aside table */
+#define	WT_BTREE_NO_CHECKPOINT	0x001000 /* Disable checkpoints */
+#define	WT_BTREE_NO_EVICTION	0x002000 /* Disable eviction */
+#define	WT_BTREE_NO_LOGGING	0x004000 /* Disable logging */
+#define	WT_BTREE_NO_RECONCILE	0x008000 /* Allow splits, even with no evict */
+#define	WT_BTREE_REBALANCE	0x010000 /* Handle is for rebalance */
+#define	WT_BTREE_SALVAGE	0x020000 /* Handle is for salvage */
+#define	WT_BTREE_SKIP_CKPT	0x040000 /* Handle skipped checkpoint */
+#define	WT_BTREE_UPGRADE	0x080000 /* Handle is for upgrade */
+#define	WT_BTREE_VERIFY		0x100000 /* Handle is for verify */
 	uint32_t flags;
 };
 
diff --git a/src/include/wiredtiger.in b/src/include/wiredtiger.in
index 0236cbdbe2d..2b71a580532 100644
--- a/src/include/wiredtiger.in
+++ b/src/include/wiredtiger.in
@@ -1069,6 +1069,11 @@ struct __wt_session {
 	 * Permitted values are \c "none"\, \c "english"\, \c "utf8<file>" or \c
 	 * "utf16<file>". See @ref huffman for more information., a string;
 	 * default \c none.}
+	 * @config{ignore_in_memory_cache_size, allow update and insert
+	 * operations to proceed even if the cache is already at capacity.  Only
+	 * valid in conjunction with in-memory databases.  Should be used with
+	 * caution - this configuration allows WiredTiger to consume memory over
+	 * the configured cache limit., a boolean flag; default \c false.}
 	 * @config{immutable, configure the index to be immutable - that is an
 	 * index is not changed by any update to a record in the table., a
 	 * boolean flag; default \c false.}
diff --git a/test/suite/test_inmem02.py b/test/suite/test_inmem02.py
new file mode 100644
index 00000000000..9eb8330b2a3
--- /dev/null
+++ b/test/suite/test_inmem02.py
@@ -0,0 +1,64 @@
+#!/usr/bin/env python
+#
+# Public Domain 2014-2016 MongoDB, Inc.
+# Public Domain 2008-2014 WiredTiger, Inc.
+#
+# This is free and unencumbered software released into the public domain.
+#
+# Anyone is free to copy, modify, publish, use, compile, sell, or
+# distribute this software, either in source code form or as a compiled
+# binary, for any purpose, commercial or non-commercial, and by any
+# means.
+#
+# In jurisdictions that recognize copyright laws, the author or authors
+# of this software dedicate any and all copyright interest in the
+# software to the public domain. We make this dedication for the benefit
+# of the public at large and to the detriment of our heirs and
+# successors. We intend this dedication to be an overt act of
+# relinquishment in perpetuity of all present and future rights to this
+# software under copyright law.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+
+import wiredtiger, wttest
+from time import sleep
+from helper import simple_populate, simple_populate_check
+from helper import key_populate, value_populate
+from wtscenario import make_scenarios
+
+# test_inmem02.py
+#    Test in-memory with ignore-cache-size setting.
+class test_inmem02(wttest.WiredTigerTestCase):
+    uri = 'table:inmem02'
+    conn_config = \
+        'cache_size=3MB,file_manager=(close_idle_time=0),in_memory=true'
+    table_config = 'key_format=S,value_format=S,memory_page_max=32k,leaf_page_max=4k'
+
+    # Add more data than fits into the configured cache and verify it fails.
+    def test_insert_over_allowed(self):
+
+        # Create a new table that is allowed to exceed the cache size, do this
+        # before filling the cache so that the create succeeds
+        self.session.create(
+            self.uri + '_over', 'ignore_in_memory_cache_size=true')
+
+        # Populate a table with enough data to fill the cache.
+        msg = '/WT_CACHE_FULL.*/'
+        self.assertRaisesHavingMessage(wiredtiger.WiredTigerError,
+            lambda:simple_populate(
+            self, self.uri, self.table_config, 10000000), msg)
+
+        # Add some content to the new table
+        cursor = self.session.open_cursor(self.uri + '_over', None)
+        for i in range(1, 1000):
+            cursor[str('%015d' % i)] = str(i) + ': abcdefghijklmnopqrstuvwxyz'
+        cursor.close()
+
+if __name__ == '__main__':
+    wttest.run()