SERVER-42726 enable logging via logv2

7 files changed, 317 insertions, 78 deletions

diff --git a/src/mongo/logv2/SConscript b/src/mongo/logv2/SConscript
index ef31e488ce7..9772cb9c60b 100644
--- a/src/mongo/logv2/SConscript
+++ b/src/mongo/logv2/SConscript
@@ -4,43 +4,22 @@ Import("env")
 
 env = env.Clone()
 
-env.Library(
-    target='logv2',
-    source=[
-        'attributes.cpp',
-        'console.cpp',
-        'log.cpp',
-        'log_component.cpp',
-        'log_component_settings.cpp',
-        'log_domain.cpp',
-        'log_domain_global.cpp',
-        'log_manager.cpp',
-        'log_record.cpp',
-        'log_severity.cpp',
-        'log_tag.cpp',
-        'ramlog.cpp',
-    ],
-    LIBDEPS=[
-        '$BUILD_DIR/mongo/base',
-    ],
-    LIBDEPS_PRIVATE=[
-    ],
-)
-
 env.CppUnitTest(
     target='log_test_v2',
     source=[
         'log_test_v2.cpp',
     ],
     LIBDEPS=[
-        'logv2',
+        '$BUILD_DIR/mongo/base',
     ]
 )
 
 env.Benchmark(
     target='logv2_bm',
-    source='logv2_bm.cpp',
+    source=[
+        'logv2_bm.cpp',
+    ],
     LIBDEPS=[
-        'logv2',
+        '$BUILD_DIR/mongo/base',
     ],
 )
diff --git a/src/mongo/logv2/log.h b/src/mongo/logv2/log.h
index adc47130aec..a27265e6283 100644
--- a/src/mongo/logv2/log.h
+++ b/src/mongo/logv2/log.h
@@ -44,6 +44,8 @@
 #else  // MONGO_UTIL_LOGV2_H_
 #define MONGO_UTIL_LOGV2_H_
 
+#include <boost/preprocessor/variadic/size.hpp>
+
 #include "mongo/base/status.h"
 #include "mongo/bson/util/builder.h"
 #include "mongo/logv2/attribute_argument_set.h"
@@ -52,8 +54,6 @@
 #include "mongo/logv2/log_severity.h"
 #include "mongo/util/errno_util.h"
 
-#include <boost/preprocessor/variadic/size.hpp>
-
 // Provide log component in global scope so that MONGO_LOG will always have a valid component.
 // Global log component will be kDefault unless overridden by MONGO_LOGV2_DEFAULT_COMPONENT.
 #if defined(MONGO_LOGV2_DEFAULT_COMPONENT)
@@ -65,52 +65,11 @@ const ::mongo::logv2::LogComponent MongoLogV2DefaultComponent_component =
        "Please see http://www.mongodb.org/about/contributors/reference/server-logging-rules/ "
 #endif  // MONGO_LOGV2_DEFAULT_COMPONENT
 
+// include log_detail.h and log_options.h after MONGO_LOGV2_DEFAULT_COMPONENT gets set
+#include "mongo/logv2/log_detail.h"
 #include "mongo/logv2/log_options.h"
 
 namespace mongo {
-namespace logv2 {
-namespace detail {
-void doLogImpl(LogSeverity const& severity,
-               LogOptions const& options,
-               StringData stable_id,
-               StringData message,
-               AttributeArgumentSet const& attrs);
-
-void doLogRecordImpl(LogRecord&& debugRecord,
-                     LogDomain& domain,
-                     StringData message,
-                     AttributeArgumentSet const& attrs);
-
-template <typename S, typename... Args>
-void doLog(LogSeverity const& severity,
-           LogOptions const& options,
-           StringData stable_id,
-           S const& message,
-           fmt::internal::named_arg<Args, char>&&... args) {
-    AttributeArgumentSet attr_set;
-    auto arg_store = fmt::internal::make_args_checked(message, (args.value)...);
-    attr_set._values = arg_store;
-    (attr_set._names.push_back(::mongo::StringData(args.name.data(), args.name.size())), ...);
-    auto msg = static_cast<fmt::string_view>(message);
-    doLogImpl(severity, options, stable_id, ::mongo::StringData(msg.data(), msg.size()), attr_set);
-}
-
-template <typename S, typename... Args>
-void doLogRecord(LogRecord&& record,
-                 LogDomain& domain,
-                 S const& message,
-                 fmt::internal::named_arg<Args, char>&&... args) {
-    AttributeArgumentSet attr_set;
-    auto arg_store = fmt::internal::make_args_checked(message, (args.value)...);
-    attr_set._values = arg_store;
-    (attr_set._names.push_back(::mongo::StringData(args.name.data(), args.name.size())), ...);
-    auto msg = static_cast<fmt::string_view>(message);
-    doLogRecordImpl(
-        std::move(record), domain, ::mongo::StringData(msg.data(), msg.size()), attr_set);
-}
-
-}  // namespace detail
-}  // namespace logv2
 
 #define LOGV2_IMPL_0(SEVERITY, OPTIONS, ID, MESSAGE)                      \
     do {                                                                  \
@@ -245,10 +204,6 @@ void doLogRecord(LogRecord&& record,
 
 #endif  // BOOST_PP_VARIADICS_MSVC
 
-inline fmt::internal::udl_arg<char> operator"" _attr(const char* s, std::size_t) {
-    return {s};
-}
-
 }  // namespace mongo
 
 #endif  // MONGO_UTIL_LOGV2_H_
diff --git a/src/mongo/logv2/log_detail.cpp b/src/mongo/logv2/log_detail.cpp
new file mode 100644
index 00000000000..8b0b2aab30f
--- /dev/null
+++ b/src/mongo/logv2/log_detail.cpp
@@ -0,0 +1,91 @@
+/**
+ *    Copyright (C) 2018-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kControl
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/logv2/attributes.h"
+#include "mongo/logv2/log.h"
+#include "mongo/logv2/log_domain.h"
+#include "mongo/logv2/log_domain_impl.h"
+#include "mongo/logv2/log_options.h"
+#include "mongo/logv2/log_record_impl.h"
+#include "mongo/logv2/log_source.h"
+
+namespace mongo {
+namespace logv2 {
+namespace detail {
+
+void doLogImpl(LogSeverity const& severity,
+               LogOptions const& options,
+               StringData stable_id,
+               StringData message,
+               AttributeArgumentSet const& attrs) {
+    auto& source = options.domain().impl().source();
+    auto record = source.open_record(severity, options.component(), options.tags(), stable_id);
+    if (record) {
+        record.attribute_values().insert(
+            attributes::message(),
+            boost::log::attribute_value(
+                new boost::log::attributes::attribute_value_impl<StringData>(message)));
+
+        record.attribute_values().insert(
+            attributes::attributes(),
+            boost::log::attribute_value(
+                new boost::log::attributes::attribute_value_impl<AttributeArgumentSet>(attrs)));
+
+        source.push_record(std::move(record));
+    }
+}
+
+void doLogRecordImpl(LogRecord&& record,
+                     LogDomain& domain,
+                     StringData message,
+                     AttributeArgumentSet const& attrs) {
+    auto& source = domain.impl().source();
+    auto rec = std::move(record.impl()->_record);
+    if (rec) {
+        rec.attribute_values().insert(
+            attributes::message(),
+            boost::log::attribute_value(
+                new boost::log::attributes::attribute_value_impl<StringData>(message)));
+
+        rec.attribute_values().insert(
+            attributes::attributes(),
+            boost::log::attribute_value(
+                new boost::log::attributes::attribute_value_impl<AttributeArgumentSet>(attrs)));
+
+        source.push_record(std::move(rec));
+    }
+}
+
+}  // namespace detail
+}  // namespace logv2
+}  // namespace mongo
diff --git a/src/mongo/logv2/log_detail.h b/src/mongo/logv2/log_detail.h
new file mode 100644
index 00000000000..b1a1948b08e
--- /dev/null
+++ b/src/mongo/logv2/log_detail.h
@@ -0,0 +1,88 @@
+/**
+ *    Copyright (C) 2018-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#include "mongo/base/status.h"
+#include "mongo/bson/util/builder.h"
+#include "mongo/logv2/attribute_argument_set.h"
+#include "mongo/logv2/log_component.h"
+#include "mongo/logv2/log_domain.h"
+#include "mongo/logv2/log_options.h"
+#include "mongo/logv2/log_severity.h"
+#include "mongo/util/errno_util.h"
+
+namespace mongo {
+namespace logv2 {
+namespace detail {
+void doLogImpl(LogSeverity const& severity,
+               LogOptions const& options,
+               StringData stable_id,
+               StringData message,
+               AttributeArgumentSet const& attrs);
+
+void doLogRecordImpl(LogRecord&& debugRecord,
+                     LogDomain& domain,
+                     StringData message,
+                     AttributeArgumentSet const& attrs);
+
+template <typename S, typename... Args>
+void doLog(LogSeverity const& severity,
+           LogOptions const& options,
+           StringData stable_id,
+           S const& message,
+           fmt::internal::named_arg<Args, char>&&... args) {
+    AttributeArgumentSet attr_set;
+    auto arg_store = fmt::internal::make_args_checked(message, (args.value)...);
+    attr_set._values = arg_store;
+    (attr_set._names.push_back(::mongo::StringData(args.name.data(), args.name.size())), ...);
+    auto msg = static_cast<fmt::string_view>(message);
+    doLogImpl(severity, options, stable_id, ::mongo::StringData(msg.data(), msg.size()), attr_set);
+}
+
+template <typename S, typename... Args>
+void doLogRecord(LogRecord&& record,
+                 LogDomain& domain,
+                 S const& message,
+                 fmt::internal::named_arg<Args, char>&&... args) {
+    AttributeArgumentSet attr_set;
+    auto arg_store = fmt::internal::make_args_checked(message, (args.value)...);
+    attr_set._values = arg_store;
+    (attr_set._names.push_back(::mongo::StringData(args.name.data(), args.name.size())), ...);
+    auto msg = static_cast<fmt::string_view>(message);
+    doLogRecordImpl(
+        std::move(record), domain, ::mongo::StringData(msg.data(), msg.size()), attr_set);
+}
+
+}  // namespace detail
+}  // namespace logv2
+
+inline fmt::internal::udl_arg<char> operator"" _attr(const char* s, std::size_t) {
+    return {s};
+}
+
+}  // namespace mongo
diff --git a/src/mongo/logv2/log_manager.cpp b/src/mongo/logv2/log_manager.cpp
index 4cd8e7d8382..420665fa62f 100644
--- a/src/mongo/logv2/log_manager.cpp
+++ b/src/mongo/logv2/log_manager.cpp
@@ -39,21 +39,61 @@
 #include "mongo/logv2/ramlog_sink.h"
 #include "mongo/logv2/tagged_severity_filter.h"
 #include "mongo/logv2/text_formatter.h"
-
+#include "mongo/util/time_support.h"
 
 #include <boost/core/null_deleter.hpp>
+#include <boost/filesystem/operations.hpp>
 #include <boost/log/core.hpp>
 #include <boost/log/sinks.hpp>
+#include <iostream>
+#include <string>
 
+#ifndef _WIN32
+#include <boost/log/sinks/syslog_backend.hpp>
+#endif
 
 namespace mongo {
 namespace logv2 {
 
+namespace {
+
+class RotateCollector : public boost::log::sinks::file::collector {
+public:
+    explicit RotateCollector(bool renameOnRotate) : _renameOnRotate{renameOnRotate} {}
+
+    void store_file(boost::filesystem::path const& file) override {
+        if (_renameOnRotate) {
+            auto renameTarget = file.string() + "." + terseCurrentTime(false);
+            boost::system::error_code ec;
+            boost::filesystem::rename(file, renameTarget, ec);
+            if (ec) {
+                // throw here or propagate this error in another way?
+            }
+        }
+    }
+
+    uintmax_t scan_for_files(boost::log::sinks::file::scan_method,
+                             boost::filesystem::path const&,
+                             unsigned int*) override {
+        return 0;
+    }
+
+private:
+    bool _renameOnRotate;
+};
+
+}  // namespace
+
+
 struct LogManager::Impl {
     typedef boost::log::sinks::synchronous_sink<boost::log::sinks::text_ostream_backend>
         ConsoleBackend;
-
     typedef boost::log::sinks::unlocked_sink<RamLogSink> RamLogBackend;
+#ifndef _WIN32
+    typedef boost::log::sinks::synchronous_sink<boost::log::sinks::syslog_backend> SyslogBackend;
+#endif
+    typedef boost::log::sinks::synchronous_sink<boost::log::sinks::text_file_backend>
+        RotatableFileBackend;
 
     Impl() {
         _consoleBackend = boost::make_shared<ConsoleBackend>();
@@ -75,14 +115,57 @@ struct LogManager::Impl {
         _startupWarningsBackend->set_formatter(TextFormatter());
     }
 
+    void setupSyslogBackend(int syslogFacility) {
+#ifndef _WIN32
+        // Create a backend
+        auto backend = boost::make_shared<boost::log::sinks::syslog_backend>(
+            boost::log::keywords::facility =
+                boost::log::sinks::syslog::make_facility(syslogFacility),
+            boost::log::keywords::use_impl = boost::log::sinks::syslog::native);
+
+        // // Set the straightforward level translator for the "Severity" attribute of type int
+        // backend->set_severity_mapper(
+        //     boost::log::sinks::syslog::direct_severity_mapping<int>("Severity"));
+        _syslogBackend = boost::make_shared<SyslogBackend>(backend);
+        _syslogBackend->set_filter(ComponentSettingsFilter(_globalDomain.settings()));
+        _syslogBackend->set_formatter(TextFormatter());
+#endif
+    }
+
+    void setupRotatableFileBackend(std::string path, bool append) {
+
+        auto backend = boost::make_shared<boost::log::sinks::text_file_backend>(
+            boost::log::keywords::file_name = path);
+        backend->auto_flush(true);
+
+        backend->set_file_collector(boost::make_shared<RotateCollector>(!append));
+
+        _rotatableFileBackend = boost::make_shared<RotatableFileBackend>(backend);
+        _rotatableFileBackend->set_filter(ComponentSettingsFilter(_globalDomain.settings()));
+        _rotatableFileBackend->set_formatter(TextFormatter());
+    }
+
     LogDomain _globalDomain{std::make_unique<LogDomainGlobal>()};
+    // I think that, technically, these are logging front ends
+    // and that they get to hold or wrap a backend
     boost::shared_ptr<ConsoleBackend> _consoleBackend;
+    boost::shared_ptr<RotatableFileBackend> _rotatableFileBackend;
+#ifndef _WIN32
+    boost::shared_ptr<SyslogBackend> _syslogBackend;
+#endif
     boost::shared_ptr<RamLogBackend> _globalLogCacheBackend;
     boost::shared_ptr<RamLogBackend> _startupWarningsBackend;
     bool _defaultBackendsAttached{false};
 };
 
 
+void LogManager::rotate() {
+    if (_impl->_rotatableFileBackend) {
+        auto backend = _impl->_rotatableFileBackend->locked_backend();
+        backend->rotate_file();
+    }
+}
+
 LogManager::LogManager() {
     _impl = std::make_unique<Impl>();
     reattachDefaultBackends();
@@ -108,6 +191,32 @@ void LogManager::detachDefaultBackends() {
     _impl->_defaultBackendsAttached = false;
 }
 
+void LogManager::detachConsoleBackend() {
+    _impl->_globalDomain.impl().core()->remove_sink(_impl->_consoleBackend);
+}
+
+void LogManager::setupRotatableFileBackend(std::string path, bool append) {
+    _impl->setupRotatableFileBackend(path, append);
+}
+
+void LogManager::setupSyslogBackend(int syslogFacility) {
+    _impl->setupSyslogBackend(syslogFacility);
+}
+
+void LogManager::reattachSyslogBackend() {
+#ifndef _WIN32
+    _impl->_globalDomain.impl().core()->add_sink(_impl->_syslogBackend);
+#endif
+}
+
+void LogManager::reattachRotatableFileBackend() {
+    _impl->_globalDomain.impl().core()->add_sink(_impl->_rotatableFileBackend);
+}
+
+void LogManager::reattachConsoleBackend() {
+    _impl->_globalDomain.impl().core()->add_sink(_impl->_consoleBackend);
+}
+
 void LogManager::reattachDefaultBackends() {
     invariant(!isDefaultBackendsAttached());
 
diff --git a/src/mongo/logv2/log_manager.h b/src/mongo/logv2/log_manager.h
index 059a3e7d019..69df533ab9a 100644
--- a/src/mongo/logv2/log_manager.h
+++ b/src/mongo/logv2/log_manager.h
@@ -30,6 +30,7 @@
 #pragma once
 
 #include <memory>
+#include <string>
 
 namespace mongo {
 namespace logv2 {
@@ -42,10 +43,10 @@ class LogDomain;
  * Use this while setting up the logging system, before launching any threads.
  */
 class LogManager {
+public:
     LogManager(const LogManager&) = delete;
     LogManager& operator=(const LogManager&) = delete;
 
-public:
     LogManager();
     ~LogManager();
 
@@ -63,6 +64,7 @@ public:
      * @note This function is not thread safe.
      */
     void detachDefaultBackends();
+    void detachConsoleBackend();
 
     /**
      * Reattaches the default log backends
@@ -70,6 +72,14 @@ public:
      * @note This function is not thread safe.
      */
     void reattachDefaultBackends();
+    void reattachConsoleBackend();
+
+    void setupSyslogBackend(int syslogFacility);
+    void reattachSyslogBackend();
+
+    void setupRotatableFileBackend(std::string path, bool append);
+    void reattachRotatableFileBackend();
+    void rotate();
 
     /**
      * Checks if the default log backends are attached
diff --git a/src/mongo/logv2/log_options.h b/src/mongo/logv2/log_options.h
index c5d2b073762..481c98db403 100644
--- a/src/mongo/logv2/log_options.h
+++ b/src/mongo/logv2/log_options.h
@@ -46,6 +46,9 @@ public:
 
     LogOptions(LogTag tags) : _tags(tags) {}
 
+    LogOptions(LogComponent component, LogDomain* domain, LogTag tags)
+        : _domain(domain), _tags(tags), _component(component) {}
+
     LogComponent component() const {
         return _component;
     }
@@ -61,7 +64,11 @@ public:
 private:
     LogDomain* _domain = &LogManager::global().getGlobalDomain();
     LogTag _tags;
+#ifdef MONGO_LOGV2_DEFAULT_COMPONENT
     LogComponent _component = MongoLogV2DefaultComponent_component;
+#else
+    LogComponent _component = LogComponent::kDefault;
+#endif  // MONGO_LOGV2_DEFAULT_COMPONENT
 };
 
 }  // namespace logv2