WT-3136 bug fix: WiredTiger doesn't check sprintf calls for error return (#3340)

* WT-3136 bug fix: WiredTiger doesn't check sprintf calls for error return

Make a pass through the source base to check sprintf, snprintf, vsprintf
and vsnprintf calls for errors.

* A WiredTiger key is a uint64_t.
Use sizeof(), don't hard-wire buffer sizes into the code.

* More (u_int) vs. (uint64_t) fixes.

* Use CONFIG_APPEND instead of FORMAT_APPEND, it makes more sense.

* revert part of 4475ae9, there's an explicit allocation of the size of
the buffer.

* MVSC complaints:
test\format\config.c(765): warning C4018: '<': signed/unsigned mismatch
test\format\config.c(765): warning C4018: '>': signed/unsigned mismatch

* Change Windows testing shim to correctly use __wt_snprintf

* Change Windows test shim to use the __wt_XXX functions

* MSDN's _vscprintf API returns the number of characters excluding the
termininating nul byte, return that value.

5 files changed, 97 insertions, 26 deletions

diff --git a/src/include/extern_posix.h b/src/include/extern_posix.h
index fed7835ada1..57d94e392d1 100644
--- a/src/include/extern_posix.h
+++ b/src/include/extern_posix.h
@@ -24,8 +24,9 @@ extern bool __wt_has_priv(void) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden")
 extern void __wt_stream_set_line_buffer(FILE *fp) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default")));
 extern void __wt_stream_set_no_buffer(FILE *fp) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default")));
 extern void __wt_sleep(uint64_t seconds, uint64_t micro_seconds) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default")));
+extern int __wt_vsnprintf_len_incr( char *buf, size_t size, size_t *retsizep, const char *fmt, va_list ap) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
 extern int __wt_thread_create(WT_SESSION_IMPL *session, wt_thread_t *tidret, WT_THREAD_CALLBACK(*func)(void *), void *arg) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden")));
 extern int __wt_thread_join(WT_SESSION_IMPL *session, wt_thread_t tid) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden")));
-extern void __wt_thread_id(char *buf, size_t buflen) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default")));
+extern int __wt_thread_id(char *buf, size_t buflen) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default"))) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
 extern void __wt_epoch(WT_SESSION_IMPL *session, struct timespec *tsp) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default")));
 extern void __wt_yield(void) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("default")));
diff --git a/src/include/extern_win.h b/src/include/extern_win.h
index 0bfc821c7a6..43127a0c79f 100644
--- a/src/include/extern_win.h
+++ b/src/include/extern_win.h
@@ -22,9 +22,10 @@ extern bool __wt_has_priv(void) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden")
 extern void __wt_stream_set_line_buffer(FILE *fp) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden")));
 extern void __wt_stream_set_no_buffer(FILE *fp) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden")));
 extern void __wt_sleep(uint64_t seconds, uint64_t micro_seconds) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden")));
+extern int __wt_vsnprintf_len_incr( char *buf, size_t size, size_t *retsizep, const char *fmt, va_list ap) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden")));
 extern int __wt_thread_create(WT_SESSION_IMPL *session, wt_thread_t *tidret, WT_THREAD_CALLBACK(*func)(void *), void *arg) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden")));
 extern int __wt_thread_join(WT_SESSION_IMPL *session, wt_thread_t tid) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden")));
-extern void __wt_thread_id(char *buf, size_t buflen) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden")));
+extern int __wt_thread_id(char *buf, size_t buflen) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden")));
 extern void __wt_epoch(WT_SESSION_IMPL *session, struct timespec *tsp) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden")));
 extern int __wt_to_utf16_string( WT_SESSION_IMPL *session, const char*utf8, WT_ITEM **outbuf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden")));
 extern int __wt_to_utf8_string( WT_SESSION_IMPL *session, const wchar_t*wide, WT_ITEM **outbuf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)) WT_GCC_FUNC_DECL_ATTRIBUTE((visibility("hidden")));
diff --git a/src/include/misc.i b/src/include/misc.i
index d5692a3f9cf..7040886cf82 100644
--- a/src/include/misc.i
+++ b/src/include/misc.i
@@ -86,3 +86,94 @@ __wt_verbose(WT_SESSION_IMPL *session, int flag, const char *fmt, ...)
 	WT_UNUSED(fmt);
 #endif
 }
+
+/*
+ * __wt_snprintf --
+ *	snprintf convenience function, ignoring the returned size.
+ */
+static inline int
+__wt_snprintf(char *buf, size_t size, const char *fmt, ...)
+    WT_GCC_FUNC_ATTRIBUTE((format (printf, 3, 4)))
+{
+	WT_DECL_RET;
+	size_t len;
+	va_list ap;
+
+	len = 0;
+
+	va_start(ap, fmt);
+	ret = __wt_vsnprintf_len_incr(buf, size, &len, fmt, ap);
+	va_end(ap);
+	WT_RET(ret);
+
+	/* It's an error if the buffer couldn't hold everything. */
+	return (len >= size ? ERANGE : 0);
+}
+
+/*
+ * __wt_vsnprintf --
+ *	vsnprintf convenience function, ignoring the returned size.
+ */
+static inline int
+__wt_vsnprintf(char *buf, size_t size, const char *fmt, va_list ap)
+{
+	size_t len;
+
+	len = 0;
+
+	WT_RET(__wt_vsnprintf_len_incr(buf, size, &len, fmt, ap));
+
+	/* It's an error if the buffer couldn't hold everything. */
+	return (len >= size ? ERANGE : 0);
+}
+
+/*
+ * __wt_snprintf_len_set --
+ *	snprintf convenience function, setting the returned size.
+ */
+static inline int
+__wt_snprintf_len_set(
+    char *buf, size_t size, size_t *retsizep, const char *fmt, ...)
+    WT_GCC_FUNC_ATTRIBUTE((format (printf, 4, 5)))
+{
+	WT_DECL_RET;
+	va_list ap;
+
+	*retsizep = 0;
+
+	va_start(ap, fmt);
+	ret = __wt_vsnprintf_len_incr(buf, size, retsizep, fmt, ap);
+	va_end(ap);
+	return (ret);
+}
+
+/*
+ * __wt_vsnprintf_len_set --
+ *	vsnprintf convenience function, setting the returned size.
+ */
+static inline int
+__wt_vsnprintf_len_set(
+    char *buf, size_t size, size_t *retsizep, const char *fmt, va_list ap)
+{
+	*retsizep = 0;
+
+	return (__wt_vsnprintf_len_incr(buf, size, retsizep, fmt, ap));
+}
+
+/*
+ * __wt_snprintf_len_incr --
+ *	snprintf convenience function, incrementing the returned size.
+ */
+static inline int
+__wt_snprintf_len_incr(
+    char *buf, size_t size, size_t *retsizep, const char *fmt, ...)
+    WT_GCC_FUNC_ATTRIBUTE((format (printf, 4, 5)))
+{
+	WT_DECL_RET;
+	va_list ap;
+
+	va_start(ap, fmt);
+	ret = __wt_vsnprintf_len_incr(buf, size, retsizep, fmt, ap);
+	va_end(ap);
+	return (ret);
+}
diff --git a/src/include/os_windows.h b/src/include/os_windows.h
index 65938ac9f17..c1e5f788dc6 100644
--- a/src/include/os_windows.h
+++ b/src/include/os_windows.h
@@ -43,16 +43,6 @@ typedef uint32_t	u_int;
 typedef unsigned char	u_char;
 typedef uint64_t	u_long;
 
-/* <= VS 2013 is not C99 compat */
-#if _MSC_VER < 1900
-#define	snprintf _wt_snprintf
-
-_Check_return_opt_ int __cdecl _wt_snprintf(
-    _Out_writes_(_MaxCount) char * _DstBuf,
-    _In_ size_t _MaxCount,
-    _In_z_ _Printf_format_string_ const char * _Format, ...);
-#endif
-
 /*
  * Windows does have ssize_t
  * Python headers declare also though so we need to guard it
@@ -61,18 +51,6 @@ _Check_return_opt_ int __cdecl _wt_snprintf(
 typedef int ssize_t;
 #endif
 
-/*
- * Provide a custom version of vsnprintf that returns the
- * needed buffer length instead of -1 on truncation
- */
-#define	vsnprintf _wt_vsnprintf
-
-_Check_return_opt_ int __cdecl _wt_vsnprintf(
-    _Out_writes_(_MaxCount) char * _DstBuf,
-    _In_ size_t _MaxCount,
-    _In_z_ _Printf_format_string_ const char * _Format,
-    va_list _ArgList);
-
 /* Provide a custom version of localtime_r */
 struct tm *localtime_r(const time_t* timer, struct tm* result);
 
diff --git a/src/include/packing.i b/src/include/packing.i
index 6b4bcd49e04..0eadb2f2027 100644
--- a/src/include/packing.i
+++ b/src/include/packing.i
@@ -104,8 +104,8 @@ __pack_name_next(WT_PACK_NAME *pn, WT_CONFIG_ITEM *name)
 	WT_CONFIG_ITEM ignore;
 
 	if (pn->genname) {
-		(void)snprintf(pn->buf, sizeof(pn->buf),
-		    (pn->iskey ? "key%d" : "value%d"), pn->count);
+		WT_RET(__wt_snprintf(pn->buf, sizeof(pn->buf),
+		    (pn->iskey ? "key%d" : "value%d"), pn->count));
 		WT_CLEAR(*name);
 		name->str = pn->buf;
 		name->len = strlen(pn->buf);