Merge pull request #2425 from wiredtiger/WT-2196-30-backport

WT-2196 MongoDB 3.0 backport fixes for size only statistics

8 files changed, 140 insertions, 30 deletions

diff --git a/src/block/block_open.c b/src/block/block_open.c
index 4f4b7c57279..5493d9a2a4c 100644
--- a/src/block/block_open.c
+++ b/src/block/block_open.c
@@ -426,7 +426,7 @@ __wt_block_manager_size(
 {
 	wt_off_t filesize;
 
-	WT_RET(__wt_filesize_name(session, filename, &filesize));
+	WT_RET(__wt_filesize_name(session, filename, false, &filesize));
 	WT_STAT_SET(stats, block_size, filesize);
 
 	return (0);
diff --git a/src/include/extern.h b/src/include/extern.h
index 845102ca428..d30bb916e12 100644
--- a/src/include/extern.h
+++ b/src/include/extern.h
@@ -463,7 +463,7 @@ extern int __wt_exist(WT_SESSION_IMPL *session, const char *filename, bool *exis
 extern void __wt_fallocate_config(WT_SESSION_IMPL *session, WT_FH *fh);
 extern int __wt_fallocate( WT_SESSION_IMPL *session, WT_FH *fh, wt_off_t offset, wt_off_t len);
 extern int __wt_filesize(WT_SESSION_IMPL *session, WT_FH *fh, wt_off_t *sizep);
-extern int __wt_filesize_name( WT_SESSION_IMPL *session, const char *filename, wt_off_t *sizep);
+extern int __wt_filesize_name(WT_SESSION_IMPL *session, const char *filename, bool silent, wt_off_t *sizep);
 extern int __wt_bytelock(WT_FH *fhp, wt_off_t byte, bool lock);
 extern int __wt_directory_sync_fh(WT_SESSION_IMPL *session, WT_FH *fh);
 extern int __wt_directory_sync(WT_SESSION_IMPL *session, char *path);
diff --git a/src/lsm/lsm_tree.c b/src/lsm/lsm_tree.c
index d3979da0da1..4beb5f11f83 100644
--- a/src/lsm/lsm_tree.c
+++ b/src/lsm/lsm_tree.c
@@ -213,7 +213,7 @@ __wt_lsm_tree_set_chunk_size(
 	if (!WT_PREFIX_SKIP(filename, "file:"))
 		WT_RET_MSG(session, EINVAL,
 		    "Expected a 'file:' URI: %s", chunk->uri);
-	WT_RET(__wt_filesize_name(session, filename, &size));
+	WT_RET(__wt_filesize_name(session, filename, false, &size));
 
 	chunk->size = (uint64_t)size;
 
diff --git a/src/os_posix/os_filesize.c b/src/os_posix/os_filesize.c
index b01fc91514b..c58f73b0665 100644
--- a/src/os_posix/os_filesize.c
+++ b/src/os_posix/os_filesize.c
@@ -34,8 +34,8 @@ __wt_filesize(WT_SESSION_IMPL *session, WT_FH *fh, wt_off_t *sizep)
  *	Return the size of a file in bytes, given a file name.
  */
 int
-__wt_filesize_name(
-    WT_SESSION_IMPL *session, const char *filename, wt_off_t *sizep)
+__wt_filesize_name(WT_SESSION_IMPL *session,
+    const char *filename, bool silent, wt_off_t *sizep)
 {
 	struct stat sb;
 	WT_DECL_RET;
@@ -52,5 +52,11 @@ __wt_filesize_name(
 		return (0);
 	}
 
-	WT_RET_MSG(session, ret, "%s: fstat", filename);
+	/*
+	 * Some callers of this function expect failure if the file doesn't
+	 * exist, and don't want an error message logged.
+	 */
+	if (!silent)
+		WT_RET_MSG(session, ret, "%s: fstat", filename);
+	return (ret);
 }
diff --git a/src/os_win/os_filesize.c b/src/os_win/os_filesize.c
index dfeadc31fc4..7f231b5ba9a 100644
--- a/src/os_win/os_filesize.c
+++ b/src/os_win/os_filesize.c
@@ -15,8 +15,8 @@
 int
 __wt_filesize(WT_SESSION_IMPL *session, WT_FH *fh, wt_off_t *sizep)
 {
-	WT_DECL_RET;
 	LARGE_INTEGER size;
+	WT_DECL_RET;
 
 	WT_RET(__wt_verbose(
 	    session, WT_VERB_FILEOPS, "%s: GetFileSizeEx", fh->name));
@@ -34,11 +34,11 @@ __wt_filesize(WT_SESSION_IMPL *session, WT_FH *fh, wt_off_t *sizep)
  *	Return the size of a file in bytes, given a file name.
  */
 int
-__wt_filesize_name(
-    WT_SESSION_IMPL *session, const char *filename, wt_off_t *sizep)
+__wt_filesize_name(WT_SESSION_IMPL *session,
+    const char *filename, bool silent, wt_off_t *sizep)
 {
-	WT_DECL_RET;
 	WIN32_FILE_ATTRIBUTE_DATA data;
+	WT_DECL_RET;
 	char *path;
 
 	WT_RET(__wt_filename(session, filename, &path));
@@ -53,5 +53,12 @@ __wt_filesize_name(
 		return (0);
 	}
 
-	WT_RET_MSG(session, __wt_errno(), "%s: GetFileAttributesEx", filename);
+	/*
+	 * Some callers of this function expect failure if the file doesn't
+	 * exist, and don't want an error message logged.
+	 */
+	ret = __wt_errno();
+	if (!silent)
+		WT_RET_MSG(session, ret, "%s: GetFileAttributesEx", filename);
+	return (ret);
 }
diff --git a/src/schema/schema_stat.c b/src/schema/schema_stat.c
index dba1dfe5f55..0b8c2c2951a 100644
--- a/src/schema/schema_stat.c
+++ b/src/schema/schema_stat.c
@@ -89,19 +89,23 @@ __curstat_size_only(WT_SESSION_IMPL *session,
 	/* Build up the file name from the table URI. */
 	WT_ERR(__wt_buf_fmt(
 	    session, &namebuf, "%s.wt", uri + strlen("table:")));
+
 	/*
-	 * Get the size of the underlying file.  There is nothing stopping a
-	 * race with schema level table operations (for example drop) if there
-	 * is a race there will be an error message generated.
+	 * Get the size of the underlying file. This will fail for anything
+	 * other than simple tables (LSM for example) and will fail if there
+	 * are concurrent schema level operations (for example drop). That is
+	 * fine - failing here results in falling back to the slow path of
+	 * opening the handle.
+	 * !!! Deliberately discard the return code from a failed call - the
+	 * error is flagged by not setting fast to true.
 	 */
-	WT_ERR(__wt_filesize_name(session, namebuf.data, &filesize));
-
-	/* Setup and populate the statistics structure */
-	__wt_stat_init_dsrc_stats(&cst->u.dsrc_stats);
-	WT_STAT_SET(&cst->u.dsrc_stats, block_size, filesize);
-	__wt_curstat_dsrc_final(cst);
-
-	*was_fast = true;
+	if (__wt_filesize_name(session, namebuf.data, true, &filesize) == 0) {
+		/* Setup and populate the statistics structure */
+		__wt_stat_init_dsrc_stats(&cst->u.dsrc_stats);
+		WT_STAT_SET(&cst->u.dsrc_stats, block_size, filesize);
+		__wt_curstat_dsrc_final(cst);
+		*was_fast = true;
+	}
 
 err:	__wt_free(session, tableconf);
 	__wt_buf_free(session, &namebuf);
diff --git a/test/suite/test_stat02.py b/test/suite/test_stat02.py
index c2f2a69b046..8643d700793 100644
--- a/test/suite/test_stat02.py
+++ b/test/suite/test_stat02.py
@@ -37,20 +37,24 @@ from helper import complex_populate, complex_populate_lsm, simple_populate
 class test_stat_cursor_config(wttest.WiredTigerTestCase):
     pfx = 'test_stat_cursor_config'
     uri = [
-        ('1',  dict(uri='file:' + pfx, pop=simple_populate)),
-        ('2', dict(uri='table:' + pfx, pop=simple_populate)),
-        ('3', dict(uri='table:' + pfx, pop=complex_populate)),
-        ('4', dict(uri='table:' + pfx, pop=complex_populate_lsm))
+        ('file',  dict(uri='file:' + pfx, pop=simple_populate, cfg='')),
+        ('table', dict(uri='table:' + pfx, pop=simple_populate, cfg='')),
+        ('table-lsm',
+            dict(uri='table:' + pfx, pop=simple_populate, cfg=',type=lsm')),
+        ('complex', dict(uri='table:' + pfx, pop=complex_populate, cfg='')),
+        ('complex-lsm',
+            dict(uri='table:' + pfx, pop=complex_populate_lsm, cfg=''))
     ]
     data_config = [
         ('none', dict(data_config='none', ok=[])),
-        ( 'all',  dict(data_config='all', ok=['empty', 'fast', 'all'])),
-        ('fast', dict(data_config='fast', ok=['empty', 'fast']))
+        ( 'all',  dict(data_config='all', ok=['empty', 'fast', 'all', 'size'])),
+        ('fast', dict(data_config='fast', ok=['empty', 'fast', 'size']))
     ]
     cursor_config = [
         ('empty', dict(cursor_config='empty')),
         ( 'all', dict(cursor_config='all')),
-        ('fast', dict(cursor_config='fast'))
+        ('fast', dict(cursor_config='fast')),
+        ('size', dict(cursor_config='size'))
     ]
 
     scenarios = number_scenarios(
@@ -67,7 +71,7 @@ class test_stat_cursor_config(wttest.WiredTigerTestCase):
     # For each database/cursor configuration, confirm the right combinations
     # succeed or fail.
     def test_stat_cursor_config(self):
-        self.pop(self, self.uri, 'key_format=S', 100)
+        self.pop(self, self.uri, 'key_format=S' + self.cfg, 100)
         config = 'statistics=('
         if self.cursor_config != 'empty':
             config = config + self.cursor_config
diff --git a/test/suite/test_stat05.py b/test/suite/test_stat05.py
new file mode 100644
index 00000000000..9c8fa48cc79
--- /dev/null
+++ b/test/suite/test_stat05.py
@@ -0,0 +1,89 @@
+#!/usr/bin/env python
+#
+# Public Domain 2014-2015 MongoDB, Inc.
+# Public Domain 2008-2014 WiredTiger, Inc.
+#
+# This is free and unencumbered software released into the public domain.
+#
+# Anyone is free to copy, modify, publish, use, compile, sell, or
+# distribute this software, either in source code form or as a compiled
+# binary, for any purpose, commercial or non-commercial, and by any
+# means.
+#
+# In jurisdictions that recognize copyright laws, the author or authors
+# of this software dedicate any and all copyright interest in the
+# software to the public domain. We make this dedication for the benefit
+# of the public at large and to the detriment of our heirs and
+# successors. We intend this dedication to be an overt act of
+# relinquishment in perpetuity of all present and future rights to this
+# software under copyright law.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+
+import itertools, wiredtiger, wttest
+from suite_subprocess import suite_subprocess
+from wtscenario import multiply_scenarios, number_scenarios
+from wiredtiger import stat
+from helper import complex_populate, complex_populate_lsm, simple_populate
+from helper import complex_value_populate, key_populate, value_populate
+
+# test_stat05.py
+#    Statistics cursor using size only
+class test_stat_cursor_config(wttest.WiredTigerTestCase):
+    pfx = 'test_stat_cursor_size'
+    uri = [
+        ('file',  dict(uri='file:' + pfx, pop=simple_populate, cfg='')),
+        ('table', dict(uri='table:' + pfx, pop=simple_populate, cfg='')),
+        ('table-lsm', dict(uri='table:' + pfx, pop=simple_populate,
+            cfg=',type=lsm,lsm=(chunk_size=1MB,merge_min=2)')),
+        ('complex', dict(uri='table:' + pfx, pop=complex_populate, cfg='')),
+        ('complex-lsm',
+            dict(uri='table:' + pfx, pop=complex_populate_lsm,
+            cfg=',lsm=(chunk_size=1MB,merge_min=2)')),
+    ]
+
+    scenarios = number_scenarios(uri)
+
+    # Override WiredTigerTestCase to enable statistics
+    def setUpConnectionOpen(self, dir):
+        conn = wiredtiger.wiredtiger_open(dir,
+            'create,' +
+            'statistics=(fast),' +
+            'error_prefix="%s: "' % self.shortid())
+        return conn
+
+    def openAndWalkStatCursor(self):
+        c = self.session.open_cursor(
+            'statistics:' + self.uri, None, 'statistics=(size)')
+        count = 0
+        while c.next() == 0:
+            count += 1
+        c.close()
+
+
+    # Open a size-only statistics cursor on various table types. Ensure that
+    # the cursor open succeeds. Insert enough data that LSM tables to need to
+    # switch and merge.
+    def test_stat_cursor_size(self):
+        self.pop(self, self.uri, 'key_format=S' + self.cfg, 100)
+        self.openAndWalkStatCursor()
+        cursor = self.session.open_cursor(self.uri, None)
+        for i in range(100, 40000 + 1):
+            if i % 100 == 0:
+                self.openAndWalkStatCursor()
+            if self.pop == simple_populate:
+                cursor[key_populate(cursor, i)] = value_populate(cursor, i)
+            else:
+                v = complex_value_populate(cursor, i)
+                cursor[key_populate(cursor, i)] = (v[0], v[1], v[2], v[3])
+        cursor.close()
+        self.openAndWalkStatCursor()
+
+if __name__ == '__main__':
+    wttest.run()