SERVER-2939 Supporting code for UTF-8 in the shell

This commit lets the shell read UTF-8 from the command
line and fixes a display problem with lines that start
with a UTF-8 character.  It does not include the actual
UTF-8 enabling in linenoise, but prepares for it.

6 files changed, 93 insertions, 5 deletions

diff --git a/src/mongo/shell/dbshell.cpp b/src/mongo/shell/dbshell.cpp
index 2dc3517c64e..f85b7006314 100644
--- a/src/mongo/shell/dbshell.cpp
+++ b/src/mongo/shell/dbshell.cpp
@@ -974,6 +974,26 @@ int _main( int argc, char* argv[] ) {
     return 0;
 }
 
+#ifdef _WIN32
+int wmain( int argc, wchar_t* argvW[] ) {
+    static mongo::StaticObserver staticObserver;
+    UINT initialConsoleInputCodePage = GetConsoleCP();
+    UINT initialConsoleOutputCodePage = GetConsoleOutputCP();
+    SetConsoleCP( CP_UTF8 );
+    SetConsoleOutputCP( CP_UTF8 );
+    int returnValue = -1;
+    try {
+        WindowsCommandLine wcl( argc, argvW );
+        returnValue = _main( argc, wcl.argv() );
+    }
+    catch ( mongo::DBException& e ) {
+        cerr << "exception: " << e.what() << endl;
+    }
+    SetConsoleCP( initialConsoleInputCodePage );
+    SetConsoleOutputCP( initialConsoleOutputCodePage );
+    return returnValue;
+}
+#else // #ifdef _WIN32
 int main( int argc, char* argv[] ) {
     static mongo::StaticObserver staticObserver;
     try {
@@ -984,3 +1004,4 @@ int main( int argc, char* argv[] ) {
         return -1;
     }
 }
+#endif // #ifdef _WIN32
diff --git a/src/mongo/shell/msvc/mongo.vcxproj b/src/mongo/shell/msvc/mongo.vcxproj
index b0c14065afc..d391f4a4b2d 100755
--- a/src/mongo/shell/msvc/mongo.vcxproj
+++ b/src/mongo/shell/msvc/mongo.vcxproj
@@ -177,6 +177,12 @@
     </PreBuildEvent>

   </ItemDefinitionGroup>

   <ItemGroup>

+    <ClCompile Include="..\..\..\third_party\linenoise\linenoise_utf8.cpp">

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">NotUsing</PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">NotUsing</PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Win2008PlusDebug|Win32'">NotUsing</PrecompiledHeader>

+      <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Win2008PlusRelease|Win32'">NotUsing</PrecompiledHeader>

+    </ClCompile>

     <ClCompile Include="..\..\..\third_party\boost\libs\filesystem\v2\src\v2_operations.cpp">

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">NotUsing</PrecompiledHeader>

       <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">NotUsing</PrecompiledHeader>

@@ -850,6 +856,7 @@
     </Library>

   </ItemGroup>

   <ItemGroup>

+    <ClInclude Include="..\..\..\third_party\linenoise\linenoise_utf8.h" />

     <ClInclude Include="..\..\..\third_party\pcre-8.30\config.h" />

     <ClInclude Include="..\..\..\third_party\pcre-8.30\pcre.h" />

     <ClInclude Include="..\..\..\third_party\pcre-8.30\pcrecpp.h" />

diff --git a/src/mongo/shell/msvc/mongo.vcxproj.filters b/src/mongo/shell/msvc/mongo.vcxproj.filters
index bdc7b07b1fb..b9c429d9c3a 100644
--- a/src/mongo/shell/msvc/mongo.vcxproj.filters
+++ b/src/mongo/shell/msvc/mongo.vcxproj.filters
@@ -277,7 +277,7 @@
       <Filter>third_party\pcre</Filter>

     </ClCompile>

     <ClCompile Include="..\..\util\md5.cpp">

-      <Filter>shell</Filter>

+      <Filter>util</Filter>

     </ClCompile>

     <ClCompile Include="..\..\util\stringutils.cpp">

       <Filter>util</Filter>

@@ -360,6 +360,9 @@
     <ClCompile Include="..\..\..\third_party\boost\libs\program_options\src\winmain.cpp">

       <Filter>Boost</Filter>

     </ClCompile>

+    <ClCompile Include="..\..\..\third_party\linenoise\linenoise_utf8.cpp">

+      <Filter>third_party\linenoise</Filter>

+    </ClCompile>

     <ClCompile Include="..\..\util\progress_meter.cpp">

       <Filter>util</Filter>

     </ClCompile>

@@ -854,6 +857,9 @@
     <ClInclude Include="..\..\targetver.h">

       <Filter>third_party</Filter>

     </ClInclude>

+    <ClInclude Include="..\..\..\third_party\linenoise\linenoise_utf8.h">

+      <Filter>third_party\linenoise</Filter>

+    </ClInclude>

     <ClInclude Include="..\..\util\progress_meter.h">

       <Filter>util</Filter>

     </ClInclude>

diff --git a/src/mongo/util/log.cpp b/src/mongo/util/log.cpp
index 2bf6f400c68..1f05eaee3b5 100644
--- a/src/mongo/util/log.cpp
+++ b/src/mongo/util/log.cpp
@@ -243,15 +243,32 @@ namespace mongo {
     }
 
     void Logstream::logLockless( const StringData& s ) {
+
         if ( s.size() == 0 )
             return;
 
         if ( doneSetup == 1717 ) {
-#ifndef _WIN32
+
+#if defined(_WIN32)
+            // fwrite() has a behavior problem in Windows when writing to the console
+            //  when the console is in the UTF-8 code page: fwrite() sends a single
+            //  byte and then the rest of the string.  If the first character is
+            //  non-ASCII, the console then displays two UTF-8 replacement characters
+            //  instead of the single UTF-8 character we want.  write() doesn't have
+            //  this problem.
+            int fd = fileno( logfile );
+            if ( _isatty( fd ) ) {
+                fflush( logfile );
+                _write( fd, s.data(), s.size() );
+                return;
+            }
+#else
             if ( isSyslog ) {
                 syslog( LOG_INFO , "%s" , s.data() );
-            } else
+                return;
+            }
 #endif
+
             if (fwrite(s.data(), s.size(), 1, logfile)) {
                 fflush(logfile);
             }
diff --git a/src/mongo/util/text.cpp b/src/mongo/util/text.cpp
index 69ea16158ce..a4091d684bb 100644
--- a/src/mongo/util/text.cpp
+++ b/src/mongo/util/text.cpp
@@ -105,7 +105,30 @@ namespace mongo {
     }
 #endif
 
-#endif
+    WindowsCommandLine::WindowsCommandLine( int argc, wchar_t* argvW[] ) {
+        vector < string >   utf8args;
+        vector < size_t >   utf8argLength;
+        size_t blockSize = argc * sizeof( char * );
+        size_t blockPtr = blockSize;
+        for ( int i = 0; i < argc; ++i ) {
+            utf8args.push_back( toUtf8String( argvW[ i ] ) );
+            size_t argLength = utf8args[ i ].length() + 1;
+            utf8argLength.push_back( argLength );
+            blockSize += argLength;
+        }
+        _argv = reinterpret_cast< char** >( malloc( blockSize ) );
+        for ( int i = 0; i < argc; ++i ) {
+            _argv[ i ] = reinterpret_cast< char * >( _argv ) + blockPtr;
+            strcpy_s( _argv[ i ], utf8argLength[ i ], utf8args[ i ].c_str() );
+            blockPtr += utf8argLength[ i ];
+        }
+    }
+
+    WindowsCommandLine::~WindowsCommandLine() {
+        free( _argv );
+    }
+
+#endif // #if defined(_WIN32)
 
     struct TextUnitTest : public UnitTest {
         void run() {
diff --git a/src/mongo/util/text.h b/src/mongo/util/text.h
index bf25c86fd39..423fb653e08 100644
--- a/src/mongo/util/text.h
+++ b/src/mongo/util/text.h
@@ -145,4 +145,18 @@ namespace mongo {
 #endif // !defined(_WIN32)
         return ret;
     }
-}
+
+#if defined(_WIN32)
+
+    class WindowsCommandLine {
+        char**  _argv;
+
+    public:
+        WindowsCommandLine( int argc, wchar_t* argvW[] );
+        ~WindowsCommandLine();
+        char** argv( void ) const { return _argv; };
+    };
+
+#endif // #if defined(_WIN32)
+
+} // namespace mongo