Merge pull request #2386 from wiredtiger/wt-2276-checkpoint-decodemongodb-3.3.0mongodb-3.2.1

WT-2276 decode checkpoints via 'wt list -c' and via standalone Python utility.

5 files changed, 216 insertions, 14 deletions

diff --git a/examples/c/ex_data_source.c b/examples/c/ex_data_source.c
index 3cd44257d39..5d1f2622ecb 100644
--- a/examples/c/ex_data_source.c
+++ b/examples/c/ex_data_source.c
@@ -439,7 +439,7 @@ my_open_cursor(WT_DATA_SOURCE *dsrc, WT_SESSION *session,
 	{
 	/*! [WT_EXTENSION metadata search] */
 	/*
-	 * Insert a new WiredTiger metadata record.
+	 * Search for a WiredTiger metadata record.
 	 */
 	const char *key = "datasource_uri";
 	char *value;
diff --git a/src/block/block_addr.c b/src/block/block_addr.c
index 6d50e5f0f4e..9ba4ec4a8b2 100644
--- a/src/block/block_addr.c
+++ b/src/block/block_addr.c
@@ -14,7 +14,7 @@
  * caller's buffer reference so it can be called repeatedly to load a buffer.
  */
 static int
-__block_buffer_to_addr(WT_BLOCK *block,
+__block_buffer_to_addr(uint32_t allocsize,
     const uint8_t **pp, wt_off_t *offsetp, uint32_t *sizep, uint32_t *cksump)
 {
 	uint64_t o, s, c;
@@ -39,8 +39,8 @@ __block_buffer_to_addr(WT_BLOCK *block,
 		*offsetp = 0;
 		*sizep = *cksump = 0;
 	} else {
-		*offsetp = (wt_off_t)(o + 1) * block->allocsize;
-		*sizep = (uint32_t)s * block->allocsize;
+		*offsetp = (wt_off_t)(o + 1) * allocsize;
+		*sizep = (uint32_t)s * allocsize;
 		*cksump = (uint32_t)c;
 	}
 	return (0);
@@ -80,7 +80,8 @@ int
 __wt_block_buffer_to_addr(WT_BLOCK *block,
     const uint8_t *p, wt_off_t *offsetp, uint32_t *sizep, uint32_t *cksump)
 {
-	return (__block_buffer_to_addr(block, &p, offsetp, sizep, cksump));
+	return (__block_buffer_to_addr(
+	    block->allocsize, &p, offsetp, sizep, cksump));
 }
 
 /*
@@ -139,12 +140,12 @@ __wt_block_addr_string(WT_SESSION_IMPL *session,
 }
 
 /*
- * __wt_block_buffer_to_ckpt --
+ * __block_buffer_to_ckpt --
  *	Convert a checkpoint cookie into its components.
  */
-int
-__wt_block_buffer_to_ckpt(WT_SESSION_IMPL *session,
-    WT_BLOCK *block, const uint8_t *p, WT_BLOCK_CKPT *ci)
+static int
+__block_buffer_to_ckpt(WT_SESSION_IMPL *session,
+    uint32_t allocsize, const uint8_t *p, WT_BLOCK_CKPT *ci)
 {
 	uint64_t a;
 	const uint8_t **pp;
@@ -154,13 +155,13 @@ __wt_block_buffer_to_ckpt(WT_SESSION_IMPL *session,
 		WT_RET_MSG(session, WT_ERROR, "unsupported checkpoint version");
 
 	pp = &p;
-	WT_RET(__block_buffer_to_addr(block, pp,
+	WT_RET(__block_buffer_to_addr(allocsize, pp,
 	    &ci->root_offset, &ci->root_size, &ci->root_cksum));
-	WT_RET(__block_buffer_to_addr(block, pp,
+	WT_RET(__block_buffer_to_addr(allocsize, pp,
 	    &ci->alloc.offset, &ci->alloc.size, &ci->alloc.cksum));
-	WT_RET(__block_buffer_to_addr(block, pp,
+	WT_RET(__block_buffer_to_addr(allocsize, pp,
 	    &ci->avail.offset, &ci->avail.size, &ci->avail.cksum));
-	WT_RET(__block_buffer_to_addr(block, pp,
+	WT_RET(__block_buffer_to_addr(allocsize, pp,
 	    &ci->discard.offset, &ci->discard.size, &ci->discard.cksum));
 	WT_RET(__wt_vunpack_uint(pp, 0, &a));
 	ci->file_size = (wt_off_t)a;
@@ -171,6 +172,32 @@ __wt_block_buffer_to_ckpt(WT_SESSION_IMPL *session,
 }
 
 /*
+ * __wt_block_buffer_to_ckpt --
+ *	Convert a checkpoint cookie into its components, block manager version.
+ */
+int
+__wt_block_buffer_to_ckpt(WT_SESSION_IMPL *session,
+    WT_BLOCK *block, const uint8_t *p, WT_BLOCK_CKPT *ci)
+{
+	return (__block_buffer_to_ckpt(session, block->allocsize, p, ci));
+}
+
+/*
+ * __wt_block_ckpt_decode --
+ *	Convert a checkpoint cookie into its components, external utility
+ * version.
+ */
+int
+__wt_block_ckpt_decode(WT_SESSION *wt_session,
+    size_t allocsize, const uint8_t *p, WT_BLOCK_CKPT *ci)
+{
+	WT_SESSION_IMPL *session;
+
+	session = (WT_SESSION_IMPL *)wt_session;
+	return (__block_buffer_to_ckpt(session, (uint32_t)allocsize, p, ci));
+}
+
+/*
  * __wt_block_ckpt_to_buffer --
  *	Convert the components into its checkpoint cookie.
  */
diff --git a/src/include/extern.h b/src/include/extern.h
index 18d1d292c15..7338f8dae3b 100644
--- a/src/include/extern.h
+++ b/src/include/extern.h
@@ -14,6 +14,7 @@ extern int __wt_block_buffer_to_addr(WT_BLOCK *block, const uint8_t *p, wt_off_t
 extern int __wt_block_addr_invalid(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *addr, size_t addr_size, bool live);
 extern int __wt_block_addr_string(WT_SESSION_IMPL *session, WT_BLOCK *block, WT_ITEM *buf, const uint8_t *addr, size_t addr_size);
 extern int __wt_block_buffer_to_ckpt(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *p, WT_BLOCK_CKPT *ci);
+extern int __wt_block_ckpt_decode(WT_SESSION *wt_session, size_t allocsize, const uint8_t *p, WT_BLOCK_CKPT *ci);
 extern int __wt_block_ckpt_to_buffer(WT_SESSION_IMPL *session, WT_BLOCK *block, uint8_t **pp, WT_BLOCK_CKPT *ci);
 extern int __wt_block_ckpt_init( WT_SESSION_IMPL *session, WT_BLOCK_CKPT *ci, const char *name);
 extern int __wt_block_checkpoint_load(WT_SESSION_IMPL *session, WT_BLOCK *block, const uint8_t *addr, size_t addr_size, uint8_t *root_addr, size_t *root_addr_sizep, bool checkpoint);
diff --git a/src/utilities/util_list.c b/src/utilities/util_list.c
index 99a1455a74e..135a8bab225 100644
--- a/src/utilities/util_list.c
+++ b/src/utilities/util_list.c
@@ -8,6 +8,7 @@
 
 #include "util.h"
 
+static int list_get_allocsize(WT_SESSION *, const char *, size_t *);
 static int list_print(WT_SESSION *, const char *, bool, bool);
 static int list_print_checkpoint(WT_SESSION *, const char *);
 static int usage(void);
@@ -56,6 +57,48 @@ util_list(WT_SESSION *session, int argc, char *argv[])
 }
 
 /*
+ * list_get_allocsize --
+ *	Get the allocation size for this file from the metadata.
+ */
+static int
+list_get_allocsize(WT_SESSION *session, const char *key, size_t *allocsize)
+{
+	WT_CONFIG_ITEM szvalue;
+	WT_CONFIG_PARSER *parser;
+	WT_DECL_RET;
+	WT_EXTENSION_API *wt_api;
+	char *config;
+
+	wt_api = session->connection->get_extension_api(session->connection);
+	if ((ret =
+	    wt_api->metadata_search(wt_api, session, key, &config)) != 0) {
+		fprintf(stderr, "%s: %s: extension_api.metadata_search: %s\n",
+		    progname, key, session->strerror(session, ret));
+		return (ret);
+	}
+	if ((ret = wt_api->config_parser_open(wt_api, session, config,
+	    strlen(config), &parser)) != 0) {
+		fprintf(stderr, "%s: extension_api.config_parser_open: %s\n",
+		    progname, session->strerror(session, ret));
+		return (ret);
+	}
+	if ((ret = parser->get(parser, "allocation_size", &szvalue)) != 0) {
+		if (ret != WT_NOTFOUND)
+			fprintf(stderr, "%s: config_parser.get: %s\n",
+			    progname, session->strerror(session, ret));
+		(void)parser->close(parser);
+		return (ret);
+	}
+	if ((ret = parser->close(parser)) != 0) {
+		fprintf(stderr, "%s: config_parser.close: %s\n",
+		    progname, session->strerror(session, ret));
+		return (ret);
+	}
+	*allocsize = (size_t)szvalue.val;
+	return (0);
+}
+
+/*
  * list_print --
  *	List the high-level objects in the database.
  */
@@ -137,9 +180,10 @@ list_print(WT_SESSION *session, const char *name, bool cflag, bool vflag)
 static int
 list_print_checkpoint(WT_SESSION *session, const char *key)
 {
+	WT_BLOCK_CKPT ci;
 	WT_DECL_RET;
 	WT_CKPT *ckpt, *ckptbase;
-	size_t len;
+	size_t allocsize, len;
 	time_t t;
 	uint64_t v;
 
@@ -151,6 +195,14 @@ list_print_checkpoint(WT_SESSION *session, const char *key)
 	if ((ret = __wt_metadata_get_ckptlist(session, key, &ckptbase)) != 0)
 		return (ret == WT_NOTFOUND ? 0 : ret);
 
+	/* We need the allocation size for decoding the checkpoint addr */
+	if ((ret = list_get_allocsize(session, key, &allocsize)) != 0) {
+		if (ret == WT_NOTFOUND)
+			allocsize = 0;
+		else
+			return (ret);
+	}
+
 	/* Find the longest name, so we can pretty-print. */
 	len = 0;
 	WT_CKPT_FOREACH(ckptbase, ckpt)
@@ -158,7 +210,15 @@ list_print_checkpoint(WT_SESSION *session, const char *key)
 			len = strlen(ckpt->name);
 	++len;
 
+	memset(&ci, 0, sizeof(ci));
 	WT_CKPT_FOREACH(ckptbase, ckpt) {
+		if (allocsize != 0 && (ret = __wt_block_ckpt_decode(
+		    session, allocsize, ckpt->raw.data, &ci)) != 0) {
+			fprintf(stderr, "%s: __wt_block_buffer_to_ckpt: %s\n",
+			    progname, session->strerror(session, ret));
+			/* continue if damaged */
+			ci.root_size = 0;
+		}
 		/*
 		 * Call ctime, not ctime_r; ctime_r has portability problems,
 		 * the Solaris version is different from the POSIX standard.
@@ -179,6 +239,17 @@ list_print_checkpoint(WT_SESSION *session, const char *key)
 			printf(" (%" PRIu64 " KB)\n", v / WT_KILOBYTE);
 		else
 			printf(" (%" PRIu64 " B)\n", v);
+		if (ci.root_size != 0) {
+			printf("\t\t" "root offset: %" PRIuMAX
+			    " (0x%" PRIxMAX ")\n",
+			    (intmax_t)ci.root_offset, (intmax_t)ci.root_offset);
+			printf("\t\t" "root size: %" PRIu32
+			    " (0x%" PRIx32 ")\n",
+			    ci.root_size, ci.root_size);
+			printf("\t\t" "root checksum: %" PRIu32
+			    " (0x%" PRIx32 ")\n",
+			    ci.root_cksum, ci.root_cksum);
+		}
 	}
 
 	__wt_metadata_free_ckptlist(session, ckptbase);
diff --git a/tools/wt_ckpt_decode.py b/tools/wt_ckpt_decode.py
new file mode 100644
index 00000000000..b27a0ae5297
--- /dev/null
+++ b/tools/wt_ckpt_decode.py
@@ -0,0 +1,103 @@
+#!/usr/bin/env python
+#
+# Public Domain 2014-2015 MongoDB, Inc.
+# Public Domain 2008-2014 WiredTiger, Inc.
+#
+# This is free and unencumbered software released into the public domain.
+#
+# Anyone is free to copy, modify, publish, use, compile, sell, or
+# distribute this software, either in source code form or as a compiled
+# binary, for any purpose, commercial or non-commercial, and by any
+# means.
+#
+# In jurisdictions that recognize copyright laws, the author or authors
+# of this software dedicate any and all copyright interest in the
+# software to the public domain. We make this dedication for the benefit
+# of the public at large and to the detriment of our heirs and
+# successors. We intend this dedication to be an overt act of
+# relinquishment in perpetuity of all present and future rights to this
+# software under copyright law.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+
+# Decode a checkpoint 'addr'
+
+import os, sys, getopt
+
+def usage():
+    print 'Usage:\n\
+  $ python .../tools/wt_ckpt_decode.py [ -a allocsize ] addr...\n\
+\n\
+addr is a hex string\n\
+'
+
+def err_usage(msg):
+        print 'wt_ckpt_decode.py: ERROR: ' + msg
+        usage()
+        sys.exit(False)
+
+# Set paths
+wt_disttop = sys.path[0]
+while not os.path.isdir(wt_disttop + '/build_posix'):
+    if wt_disttop == '/':
+        err_usage('current dir not in wiredtiger development directory')
+    wt_disttop = os.path.dirname(wt_disttop)
+sys.path.insert(1, os.path.join(wt_disttop, 'lang', 'python', 'wiredtiger'))
+
+from packing import pack, unpack
+
+def show_one(label, value):
+    l = 16 - len(label)
+    l = l if l > 1 else 1
+    print '    {0}{1}{2:10d}  (0x{2:x})'.format(label, (' ' * l), value, value)
+    
+def show_triple(triple, name, allocsize):
+    off = triple[0]
+    size = triple[1]
+    csum = triple[2]
+    if size == 0:
+        off = -1
+        csum = 0
+    show_one(name + ' offset', (off + 1) * allocsize)
+    show_one(name + ' size', (size) * allocsize)
+    show_one(name + ' cksum', csum)
+    print ''
+
+def decode_arg(arg, allocsize):
+    addr = arg.decode("hex")
+    version = ord(addr[0])
+    print arg + ': '
+    if version != 1:
+        print '**** ERROR: unknown version ' + str(version)
+    addr = addr[1:]
+    result = unpack('iiiiiiiiiiiiii',addr)
+    if len(result) != 14:
+        print '**** ERROR: result len unexpected: ' + str(len(result))
+    show_triple(result[0:3], 'root', allocsize)
+    show_triple(result[3:6], 'alloc', allocsize)
+    show_triple(result[6:9], 'avail', allocsize)
+    show_triple(result[9:12], 'discard', allocsize)
+    file_size = result[12]
+    ckpt_size = result[13]
+    show_one('file size', file_size)
+    show_one('checkpoint size', ckpt_size)
+
+#decode_arg('018281e420f2fa4a8381e40c5855ca808080808080e22fc0e20fc0', 4096)
+    
+allocsize = 4096
+try:
+    opts, args = getopt.getopt(sys.argv[1:], "a:", ["allocsize"])
+except getopt.GetoptError as err:
+    err_usage(str(err))
+for o, a in opts:
+    if o == '-a':
+        allocsize = int(a)
+    
+for arg in args:
+    decode_arg(arg, allocsize)