WT-2863 Support UTF-8 paths on Windows (#2984)

9 files changed, 185 insertions, 59 deletions

diff --git a/dist/filelist b/dist/filelist
index eabca36fe72..e16ae879a33 100644
--- a/dist/filelist
+++ b/dist/filelist
@@ -147,6 +147,7 @@ src/os_win/os_sleep.c			WINDOWS_HOST
 src/os_win/os_snprintf.c		WINDOWS_HOST
 src/os_win/os_thread.c			WINDOWS_HOST
 src/os_win/os_time.c			WINDOWS_HOST
+src/os_win/os_utf8.c			WINDOWS_HOST
 src/os_win/os_vsnprintf.c		WINDOWS_HOST
 src/os_win/os_winerr.c			WINDOWS_HOST
 src/os_win/os_yield.c			WINDOWS_HOST
diff --git a/dist/s_string.ok b/dist/s_string.ok
index ba782cd757c..30a2f1b77e9 100644
--- a/dist/s_string.ok
+++ b/dist/s_string.ok
@@ -71,9 +71,9 @@ CloseHandle
 Comparator
 Config
 Coverity
-CreateFileA
 CreateFileMapping
-CreateFileMappingA
+CreateFileMappingW
+CreateFileW
 Crummey
 CustomersPhone
 DECL
@@ -89,7 +89,7 @@ DbCursor
 DbEnv
 Decrement
 Decrypt
-DeleteFileA
+DeleteFileW
 EACCES
 EAGAIN
 EB
@@ -139,11 +139,11 @@ GIDs
 Gcc
 Geoff
 GetEnvironmentVariableA
-GetFileAttributesA
 GetFileAttributesEx
+GetFileAttributesW
 GetFileSizeEx
 GetLastError
-GetModuleHandleEx
+GetModuleHandleExW
 GetProcAddress
 Google
 HFS
@@ -228,8 +228,9 @@ Mewhort
 Mitzenmacher
 MongoDB
 MoveFile
-MoveFileA
+MoveFileW
 Multi
+MultiByteToWideChar
 Multithreaded
 Mutex
 MySecret
@@ -376,6 +377,7 @@ Wconditional
 WeakHashLen
 Werror
 Wformat
+WideCharToMultiByte
 WinNT
 WiredTiger
 WiredTiger's
diff --git a/src/include/extern_win.h b/src/include/extern_win.h
index bceb5158c28..8c2b19056e0 100644
--- a/src/include/extern_win.h
+++ b/src/include/extern_win.h
@@ -26,6 +26,8 @@ extern int __wt_thread_create(WT_SESSION_IMPL *session, wt_thread_t *tidret, WT_
 extern int __wt_thread_join(WT_SESSION_IMPL *session, wt_thread_t tid) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
 extern void __wt_thread_id(char *buf, size_t buflen);
 extern int __wt_epoch(WT_SESSION_IMPL *session, struct timespec *tsp) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
+extern int __wt_to_utf16_string( WT_SESSION_IMPL *session, const char*utf8, WT_ITEM **outbuf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
+extern int __wt_to_utf8_string( WT_SESSION_IMPL *session, const wchar_t*wide, WT_ITEM **outbuf) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
 extern DWORD __wt_getlasterror(void);
 extern int __wt_map_windows_error(DWORD windows_error) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
 extern const char *__wt_formatmessage(WT_SESSION_IMPL *session, DWORD windows_error);
diff --git a/src/os_win/os_dir.c b/src/os_win/os_dir.c
index dccacc1e446..f024d131387 100644
--- a/src/os_win/os_dir.c
+++ b/src/os_win/os_dir.c
@@ -19,11 +19,14 @@ __wt_win_directory_list(WT_FILE_SYSTEM *file_system,
 {
 	DWORD windows_error;
 	HANDLE findhandle;
-	WIN32_FIND_DATA finddata;
+	WIN32_FIND_DATAW finddata;
 	WT_DECL_ITEM(pathbuf);
+	WT_DECL_ITEM(file_utf8);
+	WT_DECL_ITEM(pathbuf_wide);
+	WT_DECL_ITEM(prefix_wide);
 	WT_DECL_RET;
 	WT_SESSION_IMPL *session;
-	size_t dirallocsz, pathlen;
+	size_t dirallocsz, pathlen, prefix_widelen;
 	uint32_t count;
 	char *dir_copy, **entries;
 
@@ -45,7 +48,11 @@ __wt_win_directory_list(WT_FILE_SYSTEM *file_system,
 	WT_ERR(__wt_scr_alloc(session, pathlen + 3, &pathbuf));
 	WT_ERR(__wt_buf_fmt(session, pathbuf, "%s\\*", dir_copy));
 
-	findhandle = FindFirstFileA(pathbuf->data, &finddata);
+	WT_ERR(__wt_to_utf16_string(session, pathbuf->data, &pathbuf_wide));
+	WT_ERR(__wt_to_utf16_string(session, prefix, &prefix_wide));
+	prefix_widelen = wcslen(prefix_wide->data);
+
+	findhandle = FindFirstFileW(pathbuf_wide->data, &finddata);
 	if (findhandle == INVALID_HANDLE_VALUE) {
 		windows_error = __wt_getlasterror();
 		__wt_errx(session,
@@ -59,21 +66,25 @@ __wt_win_directory_list(WT_FILE_SYSTEM *file_system,
 		/*
 		 * Skip . and ..
 		 */
-		if (strcmp(finddata.cFileName, ".") == 0 ||
-		    strcmp(finddata.cFileName, "..") == 0)
+		if (wcscmp(finddata.cFileName, L".") == 0 ||
+		    wcscmp(finddata.cFileName, L"..") == 0)
 			continue;
 
 		/* The list of files is optionally filtered by a prefix. */
 		if (prefix != NULL &&
-		    !WT_PREFIX_MATCH(finddata.cFileName, prefix))
+		    wcsncmp(finddata.cFileName, prefix_wide->data,
+			prefix_widelen) != 0)
 			continue;
 
 		WT_ERR(__wt_realloc_def(
 		    session, &dirallocsz, count + 1, &entries));
-		WT_ERR(__wt_strdup(
-		    session, finddata.cFileName, &entries[count]));
+
+		WT_ERR(__wt_to_utf8_string(
+		    session, finddata.cFileName, &file_utf8));
+		WT_ERR(__wt_strdup(session, file_utf8->data, &entries[count]));
 		++count;
-	} while (FindNextFileA(findhandle, &finddata) != 0);
+		__wt_scr_free(session, &file_utf8);
+	} while (FindNextFileW(findhandle, &finddata) != 0);
 
 	*dirlistp = entries;
 	*countp = count;
@@ -91,6 +102,9 @@ err:	if (findhandle != INVALID_HANDLE_VALUE)
 
 	__wt_free(session, dir_copy);
 	__wt_scr_free(session, &pathbuf);
+	__wt_scr_free(session, &file_utf8);
+	__wt_scr_free(session, &pathbuf_wide);
+	__wt_scr_free(session, &prefix_wide);
 
 	if (ret == 0)
 		return (0);
diff --git a/src/os_win/os_dlopen.c b/src/os_win/os_dlopen.c
index 3da47bf23a3..6857be2a05e 100644
--- a/src/os_win/os_dlopen.c
+++ b/src/os_win/os_dlopen.c
@@ -25,11 +25,11 @@ __wt_dlopen(WT_SESSION_IMPL *session, const char *path, WT_DLH **dlhp)
 
 	/* NULL means load from the current binary */
 	if (path == NULL) {
-		if (GetModuleHandleExA(
+		if (GetModuleHandleExW(
 		    0, NULL, (HMODULE *)&dlh->handle) == FALSE) {
 			windows_error = __wt_getlasterror();
 			__wt_errx(session,
-			    "GetModuleHandleEx: %s: %s",
+			    "GetModuleHandleExW: %s: %s",
 			    path, __wt_formatmessage(session, windows_error));
 			WT_ERR(__wt_map_windows_error(windows_error));
 		}
diff --git a/src/os_win/os_fs.c b/src/os_win/os_fs.c
index 2cac2ac550c..7ab7178114b 100644
--- a/src/os_win/os_fs.c
+++ b/src/os_win/os_fs.c
@@ -17,17 +17,20 @@ __win_fs_exist(WT_FILE_SYSTEM *file_system,
     WT_SESSION *wt_session, const char *name, bool *existp)
 {
 	WT_DECL_RET;
+	WT_DECL_ITEM(name_wide);
 	WT_SESSION_IMPL *session;
 
 	WT_UNUSED(file_system);
 
 	session = (WT_SESSION_IMPL *)wt_session;
+	*existp = false;
 
-	if (GetFileAttributesA(name) != INVALID_FILE_ATTRIBUTES)
+	WT_RET(__wt_to_utf16_string(session, name, &name_wide));
+
+	if (GetFileAttributesW(name_wide->data) != INVALID_FILE_ATTRIBUTES)
 		*existp = true;
-	else
-		*existp = false;
 
+	__wt_scr_free(session, &name_wide);
 	return (0);
 }
 
@@ -40,6 +43,8 @@ __win_fs_remove(WT_FILE_SYSTEM *file_system,
     WT_SESSION *wt_session, const char *name, uint32_t flags)
 {
 	DWORD windows_error;
+	WT_DECL_RET;
+	WT_DECL_ITEM(name_wide);
 	WT_SESSION_IMPL *session;
 
 	WT_UNUSED(file_system);
@@ -47,14 +52,18 @@ __win_fs_remove(WT_FILE_SYSTEM *file_system,
 
 	session = (WT_SESSION_IMPL *)wt_session;
 
-	if (DeleteFileA(name) == FALSE) {
+	WT_RET(__wt_to_utf16_string(session, name, &name_wide));
+
+	if (DeleteFileW(name_wide->data) == FALSE) {
 		windows_error = __wt_getlasterror();
 		__wt_errx(session,
-		    "%s: file-remove: DeleteFileA: %s",
+		    "%s: file-remove: DeleteFileW: %s",
 		    name, __wt_formatmessage(session, windows_error));
-		return (__wt_map_windows_error(windows_error));
+		WT_ERR(__wt_map_windows_error(windows_error));
 	}
-	return (0);
+
+err:	__wt_scr_free(session, &name_wide);
+	return (ret);
 }
 
 /*
@@ -66,35 +75,42 @@ __win_fs_rename(WT_FILE_SYSTEM *file_system,
     WT_SESSION *wt_session, const char *from, const char *to, uint32_t flags)
 {
 	DWORD windows_error;
+	WT_DECL_RET;
+	WT_DECL_ITEM(from_wide);
+	WT_DECL_ITEM(to_wide);
 	WT_SESSION_IMPL *session;
 
 	WT_UNUSED(file_system);
 	WT_UNUSED(flags);
-
 	session = (WT_SESSION_IMPL *)wt_session;
 
+	WT_ERR(__wt_to_utf16_string(session, from, &from_wide));
+	WT_ERR(__wt_to_utf16_string(session, to, &to_wide));
+
 	/*
 	 * Check if file exists since Windows does not override the file if
 	 * it exists.
 	 */
-	if (GetFileAttributesA(to) != INVALID_FILE_ATTRIBUTES)
-		if (DeleteFileA(to) == FALSE) {
+	if (GetFileAttributesW(to_wide->data) != INVALID_FILE_ATTRIBUTES)
+		if (DeleteFileW(to_wide->data) == FALSE) {
 			windows_error = __wt_getlasterror();
 			__wt_errx(session,
-			    "%s: file-rename: DeleteFileA: %s",
+			    "%s: file-rename: DeleteFileW: %s",
 			    to, __wt_formatmessage(session, windows_error));
-			return (__wt_map_windows_error(windows_error));
+			WT_ERR(__wt_map_windows_error(windows_error));
 		}
 
-	if (MoveFileA(from, to) == FALSE) {
+	if (MoveFileW(from_wide->data, to_wide->data) == FALSE) {
 		windows_error = __wt_getlasterror();
 		__wt_errx(session,
-		    "%s to %s: file-rename: MoveFileA: %s",
+		    "%s to %s: file-rename: MoveFileW: %s",
 		    from, to, __wt_formatmessage(session, windows_error));
-		return (__wt_map_windows_error(windows_error));
+		WT_ERR(__wt_map_windows_error(windows_error));
 	}
 
-	return (0);
+err:	__wt_scr_free(session, &from_wide);
+	__wt_scr_free(session, &to_wide);
+	return (ret);
 }
 
 /*
@@ -106,24 +122,29 @@ __wt_win_fs_size(WT_FILE_SYSTEM *file_system,
     WT_SESSION *wt_session, const char *name, wt_off_t *sizep)
 {
 	DWORD windows_error;
+	WT_DECL_RET;
 	WIN32_FILE_ATTRIBUTE_DATA data;
+	WT_DECL_ITEM(name_wide);
 	WT_SESSION_IMPL *session;
 
 	WT_UNUSED(file_system);
-
 	session = (WT_SESSION_IMPL *)wt_session;
 
-	if (GetFileAttributesExA(name, GetFileExInfoStandard, &data) != 0) {
-		*sizep =
-		    ((int64_t)data.nFileSizeHigh << 32) | data.nFileSizeLow;
-		return (0);
+	WT_RET(__wt_to_utf16_string(session, name, &name_wide));
+
+	if (GetFileAttributesExW(
+	    name_wide->data, GetFileExInfoStandard, &data) == 0) {
+		windows_error = __wt_getlasterror();
+		__wt_errx(session,
+		    "%s: file-size: GetFileAttributesEx: %s",
+		    name, __wt_formatmessage(session, windows_error));
+		WT_ERR(__wt_map_windows_error(windows_error));
 	}
 
-	windows_error = __wt_getlasterror();
-	__wt_errx(session,
-	    "%s: file-size: GetFileAttributesEx: %s",
-	    name, __wt_formatmessage(session, windows_error));
-	return (__wt_map_windows_error(windows_error));
+	*sizep = ((int64_t)data.nFileSizeHigh << 32) | data.nFileSizeLow;
+
+err:	__wt_scr_free(session, &name_wide);
+	return (ret);
 }
 
 /*
@@ -435,26 +456,26 @@ __win_open_file(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session,
 	DWORD dwCreationDisposition, windows_error;
 	WT_CONNECTION_IMPL *conn;
 	WT_DECL_RET;
+	WT_DECL_ITEM(name_wide);
 	WT_FILE_HANDLE *file_handle;
 	WT_FILE_HANDLE_WIN *win_fh;
 	WT_SESSION_IMPL *session;
 	int desired_access, f;
 
 	WT_UNUSED(file_system);
-
-	*file_handlep = NULL;
-
 	session = (WT_SESSION_IMPL *)wt_session;
 	conn = S2C(session);
+	*file_handlep = NULL;
 
 	WT_RET(__wt_calloc_one(session, &win_fh));
-
 	win_fh->direct_io = false;
 
 	/* Set up error handling. */
 	win_fh->filehandle =
 	    win_fh->filehandle_secondary = INVALID_HANDLE_VALUE;
 
+	WT_ERR(__wt_to_utf16_string(session, name, &name_wide));
+
 	/*
 	 * Opening a file handle on a directory is only to support filesystems
 	 * that require a directory sync for durability, and Windows doesn't
@@ -504,23 +525,23 @@ __win_open_file(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session,
 	if (file_type == WT_FS_OPEN_FILE_TYPE_DATA)
 		f |= FILE_FLAG_RANDOM_ACCESS;
 
-	win_fh->filehandle = CreateFileA(name, desired_access,
+	win_fh->filehandle = CreateFileW(name_wide->data, desired_access,
 	    FILE_SHARE_READ | FILE_SHARE_WRITE,
 	    NULL, dwCreationDisposition, f, NULL);
 	if (win_fh->filehandle == INVALID_HANDLE_VALUE) {
 		if (LF_ISSET(WT_FS_OPEN_CREATE) &&
 		    GetLastError() == ERROR_FILE_EXISTS)
-			win_fh->filehandle = CreateFileA(name, desired_access,
-			    FILE_SHARE_READ | FILE_SHARE_WRITE,
+			win_fh->filehandle = CreateFileW(name_wide->data,
+			    desired_access, FILE_SHARE_READ | FILE_SHARE_WRITE,
 			    NULL, OPEN_EXISTING, f, NULL);
 		if (win_fh->filehandle == INVALID_HANDLE_VALUE) {
 			windows_error = __wt_getlasterror();
 			__wt_errx(session,
 			    win_fh->direct_io ?
-			    "%s: handle-open: CreateFileA: failed with direct "
+			    "%s: handle-open: CreateFileW: failed with direct "
 			    "I/O configured, some filesystem types do not "
 			    "support direct I/O: %s" :
-			    "%s: handle-open: CreateFileA: %s",
+			    "%s: handle-open: CreateFileW: %s",
 			    name, __wt_formatmessage(session, windows_error));
 			WT_ERR(__wt_map_windows_error(windows_error));
 		}
@@ -532,13 +553,13 @@ __win_open_file(WT_FILE_SYSTEM *file_system, WT_SESSION *wt_session,
 	 * file pointer.
 	 */
 	if (!LF_ISSET(WT_FS_OPEN_READONLY)) {
-		win_fh->filehandle_secondary = CreateFileA(name, desired_access,
-		    FILE_SHARE_READ | FILE_SHARE_WRITE,
+		win_fh->filehandle_secondary = CreateFileW(name_wide->data,
+		    desired_access, FILE_SHARE_READ | FILE_SHARE_WRITE,
 		    NULL, OPEN_EXISTING, f, NULL);
 		if (win_fh->filehandle_secondary == INVALID_HANDLE_VALUE) {
 			windows_error = __wt_getlasterror();
 			__wt_errx(session,
-			    "%s: handle-open: CreateFileA: secondary: %s",
+			    "%s: handle-open: Creatively: secondary: %s",
 			    name, __wt_formatmessage(session, windows_error));
 			WT_ERR(__wt_map_windows_error(windows_error));
 		}
@@ -572,9 +593,11 @@ directory_open:
 
 	*file_handlep = file_handle;
 
+	__wt_scr_free(session, &name_wide);
 	return (0);
 
-err:	WT_TRET(__win_file_close((WT_FILE_HANDLE *)win_fh, wt_session));
+err:	__wt_scr_free(session, &name_wide);
+	WT_TRET(__win_file_close((WT_FILE_HANDLE *)win_fh, wt_session));
 	return (ret);
 }
 
diff --git a/src/os_win/os_map.c b/src/os_win/os_map.c
index 93e81007c89..a03e6cc3e52 100644
--- a/src/os_win/os_map.c
+++ b/src/os_win/os_map.c
@@ -38,12 +38,12 @@ __wt_win_map(WT_FILE_HANDLE *file_handle, WT_SESSION *wt_session,
 	__wt_verbose(session, WT_VERB_HANDLEOPS,
 	    "%s: memory-map: %" WT_SIZET_FMT " bytes", file_handle->name, len);
 
-	mapped_cookie = CreateFileMappingA(
+	mapped_cookie = CreateFileMappingW(
 	    win_fh->filehandle, NULL, PAGE_READONLY, 0, 0, NULL);
 	if (mapped_cookie == NULL) {
 		windows_error = __wt_getlasterror();
 		__wt_errx(session,
-		    "%s: memory-map: CreateFileMappingA: %s",
+		    "%s: memory-map: CreateFileMappingW: %s",
 		    file_handle->name,
 		    __wt_formatmessage(session, windows_error));
 		return (__wt_map_windows_error(windows_error));
diff --git a/src/os_win/os_utf8.c b/src/os_win/os_utf8.c
new file mode 100644
index 00000000000..f7d11c24f03
--- /dev/null
+++ b/src/os_win/os_utf8.c
@@ -0,0 +1,84 @@
+/*-
+ * Copyright (c) 2014-2016 MongoDB, Inc.
+ * Copyright (c) 2008-2014 WiredTiger, Inc.
+ *  All rights reserved.
+ *
+ * See the file LICENSE for redistribution information.
+ */
+
+#include "wt_internal.h"
+
+/*
+ * __wt_to_utf16_string --
+ *  Convert UTF-8 encoded string to UTF-16.
+ */
+int
+__wt_to_utf16_string(
+    WT_SESSION_IMPL *session, const char* utf8, WT_ITEM **outbuf)
+{
+	DWORD windows_error;
+	int bufferSize;
+	WT_DECL_RET;
+
+	bufferSize = MultiByteToWideChar(CP_UTF8, 0, utf8, -1, NULL, 0);
+	windows_error = __wt_getlasterror();
+
+	if (bufferSize == 0 && windows_error != ERROR_INSUFFICIENT_BUFFER) {
+		__wt_errx(session, "MultiByteToWideChar: %s",
+		    __wt_formatmessage(session, windows_error));
+		return (__wt_map_windows_error(windows_error));
+	}
+
+	WT_RET(__wt_scr_alloc(session, bufferSize * sizeof(wchar_t), outbuf));
+	bufferSize = MultiByteToWideChar(
+	    CP_UTF8, 0, utf8, -1, (*outbuf)->mem, bufferSize);
+
+	if (bufferSize == 0) {
+		windows_error = __wt_getlasterror();
+		__wt_scr_free(session, outbuf);
+		__wt_errx(session, "MultiByteToWideChar: %s",
+		    __wt_formatmessage(session, windows_error));
+		return (__wt_map_windows_error(windows_error));
+	}
+
+	(*outbuf)->size = bufferSize;
+	return (0);
+}
+
+/*
+ * __wt_to_utf8_string --
+ *  Convert UTF-16 encoded string to UTF-8.
+ */
+int
+__wt_to_utf8_string(
+    WT_SESSION_IMPL *session, const wchar_t* wide, WT_ITEM **outbuf)
+{
+	DWORD windows_error;
+	int bufferSize;
+	WT_DECL_RET;
+
+	bufferSize = WideCharToMultiByte(
+	    CP_UTF8, 0, wide, -1, NULL, 0, NULL, NULL);
+	windows_error = __wt_getlasterror();
+
+	if (bufferSize == 0 && windows_error != ERROR_INSUFFICIENT_BUFFER) {
+		__wt_errx(session, "WideCharToMultiByte: %s",
+		    __wt_formatmessage(session, windows_error));
+		return (__wt_map_windows_error(windows_error));
+	}
+
+	WT_RET(__wt_scr_alloc(session, bufferSize, outbuf));
+
+	bufferSize = WideCharToMultiByte(
+	    CP_UTF8, 0, wide, -1, (*outbuf)->mem, bufferSize, NULL, NULL);
+	if (bufferSize == 0) {
+		windows_error = __wt_getlasterror();
+		__wt_scr_free(session, outbuf);
+		__wt_errx(session, "WideCharToMultiByte: %s",
+		    __wt_formatmessage(session, windows_error));
+		return (__wt_map_windows_error(windows_error));
+	}
+
+	(*outbuf)->size = bufferSize;
+	return (0);
+}
diff --git a/test/mciproject.yml b/test/mciproject.yml
index 8825bb65052..ff58d1839cf 100644
--- a/test/mciproject.yml
+++ b/test/mciproject.yml
@@ -109,7 +109,7 @@ tasks:
             set -o errexit
             set -o verbose
 
-            scons.bat ${smp_command|} "CFLAGS=/Gv /wd4090 /wd4996 /we4047 /we4024 /TC /we4100" wiredtiger.dll libwiredtiger.lib
+            scons.bat ${smp_command|} "CFLAGS=/Gv /wd4090 /wd4996 /we4047 /we4024 /TC /we4100 /w4133" wiredtiger.dll libwiredtiger.lib
 
   - name: fops
     depends_on: