SERVER-10074 RotatableFileWriter type, introduce logger module.

7 files changed, 499 insertions, 1 deletions

diff --git a/.gitignore b/.gitignore
index d55d0d0f029..dc5b2551e23 100644
--- a/.gitignore
+++ b/.gitignore
@@ -62,7 +62,7 @@ failfile.smoke
 
 #temp dirs
 dump
-log
+/log
 logs
 docs/html
 docs/latex
diff --git a/src/mongo/SConscript b/src/mongo/SConscript
index afd67a06fa2..bdeb82b5a2b 100644
--- a/src/mongo/SConscript
+++ b/src/mongo/SConscript
@@ -20,6 +20,7 @@ env.SConscript(['base/SConscript',
                 'db/fts/SConscript',
                 'db/ops/SConscript',
                 'db/SConscript',
+                'logger/SConscript',
                 'platform/SConscript',
                 's/SConscript',
                 'unittest/SConscript'])
diff --git a/src/mongo/base/error_codes.err b/src/mongo/base/error_codes.err
index 949c042fb63..8ef50b8779a 100644
--- a/src/mongo/base/error_codes.err
+++ b/src/mongo/base/error_codes.err
@@ -37,5 +37,8 @@ error_code("PrivilegeNotFound", 33)
 error_code("CannotBackfillArray", 34)
 error_code("UserModificationFailed", 35)
 error_code("RemoteChangeDetected", 36)
+error_code("FileRenameFailed", 37)
+error_code("FileNotOpen", 38)
+error_code("FileStreamFailed", 39)
 
 error_class("NetworkError", ["HostUnreachable", "HostNotFound"])
diff --git a/src/mongo/logger/SConscript b/src/mongo/logger/SConscript
new file mode 100644
index 00000000000..369486f3b43
--- /dev/null
+++ b/src/mongo/logger/SConscript
@@ -0,0 +1,14 @@
+# -*- mode: python -*-
+
+Import("env")
+
+env.StaticLibrary('logger',
+                  [
+                   'rotatable_file_writer.cpp',
+                   ],
+                  LIBDEPS=['$BUILD_DIR/mongo/base/base'])
+
+env.CppUnitTest('rotatable_file_writer_test',
+                'rotatable_file_writer_test.cpp',
+                LIBDEPS=['logger'])
+
diff --git a/src/mongo/logger/rotatable_file_writer.cpp b/src/mongo/logger/rotatable_file_writer.cpp
new file mode 100644
index 00000000000..74c38247c2d
--- /dev/null
+++ b/src/mongo/logger/rotatable_file_writer.cpp
@@ -0,0 +1,265 @@
+/*    Copyright 2013 10gen Inc.
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/logger/rotatable_file_writer.h"
+
+#include <boost/scoped_array.hpp>
+#include <cstdio>
+#include <fstream>
+
+#include "mongo/base/string_data.h"
+#include "mongo/util/mongoutils/str.h"
+
+namespace mongo {
+namespace logger {
+
+namespace {
+    /**
+     * Renames file "oldName" to "newName".
+     *
+     * Both names are UTF-8 encoded.
+     */
+    int renameFile(const std::string& oldName, const std::string& newName);
+}  // namespace
+
+#ifdef _WIN32
+namespace {
+
+    /**
+     * Converts UTF-8 encoded "utf8Str" to std::wstring.
+     */
+    std::wstring utf8ToWide(const StringData& utf8Str) {
+        if (utf8Str.empty())
+            return std::wstring();
+
+        // A Windows wchar_t encoding of a unicode codepoint never takes more instances of wchar_t
+        // than the UTF-8 encoding takes instances of char.
+        boost::scoped_array<wchar_t> tempBuffer(new wchar_t[utf8Str.size()]);
+        tempBuffer[0] = L'\0';
+        int finalSize = MultiByteToWideChar(
+                CP_UTF8,               // Code page
+                0,                     // Flags
+                utf8Str.rawData(),     // Input string
+                utf8Str.size(),        // Count
+                tempBuffer.get(),      // UTF-16 output buffer
+                utf8Str.size()         // Buffer size in wide characters
+        );
+        // TODO(schwerin): fassert finalSize > 0?
+        return std::wstring(tempBuffer.get(), finalSize);
+    }
+
+
+    /**
+     * Minimal implementation of a std::streambuf for writing to Win32 files via HANDLEs.
+     *
+     * We require this implementation and the std::ostream subclass below to handle the following:
+     * (1) Opening files for shared-delete access, so that open file handles may be renamed.
+     * (2) Opening files with non-ASCII characters in their names.
+     */
+    class Win32FileStreambuf : public std::streambuf {
+        MONGO_DISALLOW_COPYING(Win32FileStreambuf);
+    public:
+        Win32FileStreambuf();
+        virtual ~Win32FileStreambuf();
+
+        bool open(const StringData& fileName, bool append);
+        bool is_open() { return _fileHandle != INVALID_HANDLE_VALUE; }
+
+    private:
+        virtual std::streamsize xsputn(const char* s, std::streamsize count);
+        virtual int_type overflow(int_type ch = traits_type::eof());
+
+        HANDLE _fileHandle;
+    };
+
+    /**
+     * Minimal implementation of a stream to Win32 files.
+     */
+    class Win32FileOStream : public std::ostream {
+        MONGO_DISALLOW_COPYING(Win32FileOStream);
+    public:
+        /**
+         * Constructs an instance, opening "fileName" in append or truncate mode according to
+         * "append".
+         */
+        Win32FileOStream(const std::string& fileName, bool append) : std::ostream(&_buf), _buf() {
+
+            if (!_buf.open(fileName, append)) {
+                setstate(failbit);
+            }
+        }
+
+        virtual ~Win32FileOStream() {}
+
+    private:
+        Win32FileStreambuf _buf;
+    };
+
+    Win32FileStreambuf::Win32FileStreambuf() : _fileHandle(INVALID_HANDLE_VALUE) {}
+    Win32FileStreambuf::~Win32FileStreambuf() {
+        if (is_open()) {
+            CloseHandle(_fileHandle);  // TODO(schwerin): Should we check for failure?
+        }
+    }
+
+    bool Win32FileStreambuf::open(const StringData& fileName, bool append) {
+        _fileHandle = CreateFileW(
+                utf8ToWide(fileName).c_str(),         // lpFileName
+                GENERIC_WRITE,                        // dwDesiredAccess
+                FILE_SHARE_DELETE | FILE_SHARE_READ,  // dwShareMode
+                NULL,                                 // lpSecurityAttributes
+                OPEN_ALWAYS,                          // dwCreationDisposition
+                FILE_ATTRIBUTE_NORMAL,                // dwFlagsAndAttributes
+                NULL                                  // hTemplateFile
+                );
+
+
+        if (INVALID_HANDLE_VALUE == _fileHandle)
+            return false;
+
+        LARGE_INTEGER zero;
+        zero.QuadPart = 0LL;
+
+        if (append) {
+            if (SetFilePointerEx(_fileHandle, zero, NULL, FILE_END)) {
+                return true;
+            }
+        }
+        else {
+            if (SetFilePointerEx(_fileHandle, zero, NULL, FILE_BEGIN) &&
+                SetEndOfFile(_fileHandle)) {
+
+                return true;
+            }
+
+        }
+        // TODO(schwerin): Record error info?
+        CloseHandle(_fileHandle);
+        return false;
+    }
+
+    std::streamsize Win32FileStreambuf::xsputn(const char* s, std::streamsize count) {
+        DWORD totalBytesWritten = 0;
+        while (count > totalBytesWritten) {
+            DWORD bytesWritten;
+            if (!WriteFile(
+                        _fileHandle,
+                        s,
+                        count - totalBytesWritten,
+                        &bytesWritten,
+                        NULL)) {
+                break;
+            }
+            totalBytesWritten += bytesWritten;
+        }
+        return totalBytesWritten;
+    }
+
+    Win32FileStreambuf::int_type Win32FileStreambuf::overflow(int_type ch) {
+        if (ch == traits_type::eof())
+            return ~ch;  // Returning traits_type::eof() => failure, anything else => success.
+        char toPut = static_cast<char>(ch);
+        if (1 == xsputn(&toPut, 1))
+            return ch;
+        return traits_type::eof();
+    }
+
+    // Win32 implementation of renameFile that handles non-ascii file names.
+    int renameFile(const std::string& oldName, const std::string& newName) {
+        return _wrename(utf8ToWide(oldName).c_str(), utf8ToWide(newName).c_str());
+    }
+
+}  // namespace
+#else
+
+namespace {
+
+    // *nix implementation of renameFile that assumes the OS is encoding file names in UTF-8.
+    int renameFile(const std::string& oldName, const std::string& newName) {
+        return rename(oldName.c_str(), newName.c_str());
+    }
+
+}  // namespace
+#endif
+
+    RotatableFileWriter::RotatableFileWriter() : _stream(NULL) {}
+
+    RotatableFileWriter::Use::Use(RotatableFileWriter* writer) :
+        _writer(writer),
+        _lock(writer->_mutex) {
+    }
+
+    Status RotatableFileWriter::Use::setFileName(const std::string& name, bool append) {
+        _writer->_fileName = name;
+        return _openFileStream(append);
+    }
+
+    Status RotatableFileWriter::Use::rotate(const std::string& renameTarget) {
+        if (_writer->_stream) {
+            _writer->_stream->flush();
+            if (0 != renameFile(_writer->_fileName, renameTarget)) {
+                return Status(ErrorCodes::FileRenameFailed, mongoutils::str::stream() <<
+                              "Failed  to rename \"" << _writer->_fileName << "\" to \"" <<
+                              renameTarget << "\": " << strerror(errno) << " (" << errno << ')');
+                //TODO(schwerin): Make errnoWithDescription() available in the logger library, and
+                //use it here.
+            }
+        }
+        return _openFileStream(false);
+    }
+
+    Status RotatableFileWriter::Use::status() {
+        if (!_writer->_stream) {
+            return Status(ErrorCodes::FileNotOpen,
+                          mongoutils::str::stream() << "File \"" << _writer->_fileName <<
+                          "\" not open");
+        }
+        if (_writer->_stream->fail()) {
+            return Status(ErrorCodes::FileStreamFailed,
+                          mongoutils::str::stream() << "File \"" << _writer->_fileName <<
+                          "\" in failed state");
+        }
+        return Status::OK();
+    }
+
+    Status RotatableFileWriter::Use::_openFileStream(bool append) {
+        using std::swap;
+
+#ifdef _WIN32
+        boost::scoped_ptr<std::ostream> newStream(
+                new Win32FileOStream(_writer->_fileName, append));
+#else
+        std::ios::openmode mode = std::ios::out;
+        if (append) {
+            mode |= std::ios::app;
+        }
+        else {
+            mode |= std::ios::trunc;
+        }
+        boost::scoped_ptr<std::ostream> newStream(
+                new std::ofstream(_writer->_fileName.c_str(), mode));
+#endif
+
+        if (newStream->fail()) {
+            return Status(ErrorCodes::FileNotOpen, "Failed to open \"" + _writer->_fileName + "\"");
+        }
+        swap(_writer->_stream, newStream);
+        return status();
+    }
+
+}  // namespace logger
+}  // namespace mongo
diff --git a/src/mongo/logger/rotatable_file_writer.h b/src/mongo/logger/rotatable_file_writer.h
new file mode 100644
index 00000000000..7cc2b3df0c4
--- /dev/null
+++ b/src/mongo/logger/rotatable_file_writer.h
@@ -0,0 +1,113 @@
+/*    Copyright 2013 10gen Inc.
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+
+#pragma once
+
+#include <boost/scoped_ptr.hpp>
+#include <boost/thread/mutex.hpp>
+#include <iostream>
+#include <string>
+
+#include "mongo/base/disallow_copying.h"
+#include "mongo/base/status.h"
+
+namespace mongo {
+namespace logger {
+
+    /**
+     * A synchronized file output stream writer, with support for file rotation.
+     *
+     * To enforce proper locking, instances of RotatableFileWriter may only be manipulated by
+     * instantiating a RotatableFileWriter::Use guard object, which exposes the relevant
+     * manipulation methods for the stream.  For any instance of RotatableFileWriter, at most one
+     * fully constructed instance of RotatableFileWriter::Use exists at a time, providing mutual
+     * exclusion.
+     */
+    class RotatableFileWriter {
+        MONGO_DISALLOW_COPYING(RotatableFileWriter);
+    public:
+        /**
+         * Guard class representing synchronous use of an instance of RotatableFileWriter.
+         */
+        class Use {
+            MONGO_DISALLOW_COPYING(Use);
+        public:
+            /**
+             * Constructs a Use object for "writer", and lock "writer".
+             */
+            explicit Use(RotatableFileWriter* writer);
+
+            /**
+             * Sets the name of the target file to which stream() writes to "name".
+             *
+             * May be called repeatedly.
+             *
+             * If this method does not return Status::OK(), it is not safe to call rotate() or
+             * stream().
+             *
+             * Set "append" to true to open "name" in append mode.  Otherwise, it is truncated.
+             */
+            Status setFileName(const std::string& name, bool append);
+
+            /**
+             * Rotates the currently opened file into "renameTarget", and open a new file
+             * with the name previously set via setFileName().
+             *
+             * Returns Status::OK() on success.  If the rename fails, returns
+             * ErrorCodes::FileRenameFailed, and the stream continues to write to the unrotated
+             * file.  If the rename succeeds but the subsequent file open fails, returns
+             * ErrorCodes::FileNotOpen, and the stream continues to target the original file, though
+             * under its new name.
+             */
+            Status rotate(const std::string& renameTarget);
+
+            /**
+             * Returns the status of the stream.
+             *
+             * One of Status::OK(), ErrorCodes::FileNotOpen and ErrorCodes::FileStreamFailed.
+             */
+            Status status();
+
+            /**
+             * Returns a reference to the std::ostream() through which users may write to the file.
+             */
+            std::ostream& stream() { return *_writer->_stream; }
+
+        private:
+            /**
+             * Helper that opens the file named by setFileName(), in the mode specified by "append".
+             *
+             * Returns Status::OK() on success and ErrorCodes::FileNotOpen on failure.
+             */
+            Status _openFileStream(bool append);
+
+            RotatableFileWriter* _writer;
+            boost::unique_lock<boost::mutex> _lock;
+        };
+
+        /**
+         * Constructs an instance of RotatableFileWriter.
+         */
+        RotatableFileWriter();
+
+    private:
+        friend class RotatableFileWriter::Use;
+        boost::mutex _mutex;
+        std::string _fileName;
+        boost::scoped_ptr<std::ostream> _stream;
+    };
+
+}  // namespace logger
+}  // namespace mongo
diff --git a/src/mongo/logger/rotatable_file_writer_test.cpp b/src/mongo/logger/rotatable_file_writer_test.cpp
new file mode 100644
index 00000000000..f303d8fd737
--- /dev/null
+++ b/src/mongo/logger/rotatable_file_writer_test.cpp
@@ -0,0 +1,102 @@
+#include "mongo/platform/basic.h"
+
+#include <fstream>
+
+#include "mongo/logger/rotatable_file_writer.h"
+#include "mongo/unittest/unittest.h"
+
+namespace {
+    using namespace mongo;
+    using namespace mongo::logger;
+
+    TEST(RotatableFileWriter, RotationTest) {
+        using namespace logger;
+
+        const std::string logFileName("LogTest_RotatableFileAppender.txt");
+        const std::string logFileNameRotated("LogTest_RotatableFileAppender_Rotated.txt");
+
+        {
+            RotatableFileWriter writer;
+            RotatableFileWriter::Use writerUse(&writer);
+            ASSERT_OK(writerUse.setFileName(logFileName, false));
+            ASSERT_TRUE(writerUse.stream() << "Level 1 message." << std::endl);
+            ASSERT_TRUE(writerUse.stream() << "Level 2 message." << std::endl);
+            ASSERT_OK(writerUse.rotate(logFileNameRotated));
+            ASSERT_TRUE(writerUse.stream() << "Level 3 message." << std::endl);
+            ASSERT_TRUE(writerUse.stream() << "Level 4 message." << std::endl);
+        }
+
+        {
+            std::ifstream ifs(logFileNameRotated.c_str());
+            ASSERT_TRUE(ifs.is_open());
+            ASSERT_TRUE(ifs.good());
+            std::string input;
+            ASSERT_TRUE(std::getline(ifs, input));
+            ASSERT_EQUALS(input, "Level 1 message.");
+            ASSERT_TRUE(std::getline(ifs, input));
+            ASSERT_EQUALS(input, "Level 2 message.");
+            ASSERT_FALSE(std::getline(ifs, input));
+            ASSERT_TRUE(ifs.eof());
+        }
+
+        {
+            std::ifstream ifs(logFileName.c_str());
+            ASSERT_TRUE(ifs.is_open());
+            ASSERT_TRUE(ifs.good());
+            std::string input;
+            ASSERT_TRUE(std::getline(ifs, input));
+            ASSERT_EQUALS(input, "Level 3 message.");
+            ASSERT_TRUE(std::getline(ifs, input));
+            ASSERT_EQUALS(input, "Level 4 message.");
+            ASSERT_FALSE(std::getline(ifs, input));
+            ASSERT_TRUE(ifs.eof());
+        }
+
+        {
+            RotatableFileWriter writer;
+            RotatableFileWriter::Use writerUse(&writer);
+            ASSERT_OK(writerUse.setFileName(logFileName, true));
+            ASSERT_TRUE(writerUse.stream() << "Level 5 message." << std::endl);
+            ASSERT_TRUE(writerUse.stream() << "Level 6 message." << std::endl);
+        }
+
+       {
+            std::ifstream ifs(logFileName.c_str());
+            ASSERT_TRUE(ifs.is_open());
+            ASSERT_TRUE(ifs.good());
+            std::string input;
+            ASSERT_TRUE(std::getline(ifs, input));
+            ASSERT_EQUALS(input, "Level 3 message.");
+            ASSERT_TRUE(std::getline(ifs, input));
+            ASSERT_EQUALS(input, "Level 4 message.");
+            ASSERT_TRUE(std::getline(ifs, input));
+            ASSERT_EQUALS(input, "Level 5 message.");
+            ASSERT_TRUE(std::getline(ifs, input));
+            ASSERT_EQUALS(input, "Level 6 message.");
+            ASSERT_FALSE(std::getline(ifs, input));
+            ASSERT_TRUE(ifs.eof());
+        }
+
+        {
+            RotatableFileWriter writer;
+            RotatableFileWriter::Use writerUse(&writer);
+            ASSERT_OK(writerUse.setFileName(logFileName, false));
+            ASSERT_TRUE(writerUse.stream() << "Level 7 message." << std::endl);
+            ASSERT_TRUE(writerUse.stream() << "Level 8 message." << std::endl);
+        }
+
+        {
+            std::ifstream ifs(logFileName.c_str());
+            ASSERT_TRUE(ifs.is_open());
+            ASSERT_TRUE(ifs.good());
+            std::string input;
+            ASSERT_TRUE(std::getline(ifs, input));
+            ASSERT_EQUALS(input, "Level 7 message.");
+            ASSERT_TRUE(std::getline(ifs, input));
+            ASSERT_EQUALS(input, "Level 8 message.");
+            ASSERT_FALSE(std::getline(ifs, input));
+            ASSERT_TRUE(ifs.eof());
+        }
+    }
+
+}  // namespace mongo