SERVER-38233 Abort if we fail to write log to output stream

5 files changed, 112 insertions, 32 deletions

diff --git a/src/mongo/logv2/file_rotate_sink.cpp b/src/mongo/logv2/file_rotate_sink.cpp
index 941ee7ae7f4..eefc8da6543 100644
--- a/src/mongo/logv2/file_rotate_sink.cpp
+++ b/src/mongo/logv2/file_rotate_sink.cpp
@@ -30,10 +30,14 @@
 #include "mongo/logv2/file_rotate_sink.h"
 
 #include <boost/filesystem/operations.hpp>
+#include <boost/iterator/filter_iterator.hpp>
+#include <boost/iterator/transform_iterator.hpp>
 #include <boost/make_shared.hpp>
 #include <fmt/format.h>
 #include <fstream>
 
+#include "mongo/logv2/json_formatter.h"
+#include "mongo/logv2/log_detail.h"
 #include "mongo/logv2/shared_access_fstream.h"
 #include "mongo/util/string_map.h"
 
@@ -64,10 +68,13 @@ StatusWith<boost::shared_ptr<stream_t>> openFile(const std::string& filename, bo
 }  // namespace
 
 struct FileRotateSink::Impl {
+    Impl(LogTimestampFormat tsFormat) : timestampFormat(tsFormat) {}
     StringMap<boost::shared_ptr<stream_t>> files;
+    LogTimestampFormat timestampFormat;
 };
 
-FileRotateSink::FileRotateSink() : _impl(std::make_unique<Impl>()) {}
+FileRotateSink::FileRotateSink(LogTimestampFormat timestampFormat)
+    : _impl(std::make_unique<Impl>(timestampFormat)) {}
 FileRotateSink::~FileRotateSink() {}
 
 Status FileRotateSink::addFile(const std::string& filename, bool append) {
@@ -131,4 +138,49 @@ Status FileRotateSink::rotate(bool rename, StringData renameSuffix) {
     return Status::OK();
 }
 
+void FileRotateSink::consume(const boost::log::record_view& rec,
+                             const string_type& formatted_string) {
+    auto isFailed = [](const auto& file) { return file.second->fail(); };
+    boost::log::sinks::text_ostream_backend::consume(rec, formatted_string);
+    if (std::any_of(_impl->files.begin(), _impl->files.end(), isFailed)) {
+        try {
+            auto failedBegin =
+                boost::make_filter_iterator(isFailed, _impl->files.begin(), _impl->files.end());
+            auto failedEnd =
+                boost::make_filter_iterator(isFailed, _impl->files.begin(), _impl->files.end());
+
+            auto getFilename = [](const auto& file) -> const auto& {
+                return file.first;
+            };
+            auto begin = boost::make_transform_iterator(failedBegin, getFilename);
+            auto end = boost::make_transform_iterator(failedEnd, getFilename);
+            auto sequence = logv2::seqLog(begin, end);
+
+            DynamicAttributes attrs;
+            attrs.add("files", sequence);
+
+            fmt::memory_buffer buffer;
+            JSONFormatter(nullptr, _impl->timestampFormat)
+                .format(buffer,
+                        LogSeverity::Severe(),
+                        LogComponent::kControl,
+                        Date_t::now(),
+                        4522200,
+                        getThreadName(),
+                        "Writing to log file failed, aborting application",
+                        TypeErasedAttributeStorage(attrs),
+                        LogTag::kNone,
+                        LogTruncation::Disabled);
+            // Commented out log line below to get validation of the log id with the errorcodes
+            // linter LOGV2(4522200, "Writing to log file failed, aborting application");
+            std::cout << StringData(buffer.data(), buffer.size()) << std::endl;
+        } catch (...) {
+            // If the formatting code throws for any reason, ignore and proceed with aborting the
+            // application.
+        }
+
+        std::abort();
+    }
+}
+
 }  // namespace mongo::logv2
diff --git a/src/mongo/logv2/file_rotate_sink.h b/src/mongo/logv2/file_rotate_sink.h
index 56f12215750..d0ae4527b4c 100644
--- a/src/mongo/logv2/file_rotate_sink.h
+++ b/src/mongo/logv2/file_rotate_sink.h
@@ -34,6 +34,7 @@
 #include <string>
 
 #include "mongo/base/status.h"
+#include "mongo/logv2/log_format.h"
 
 namespace mongo::logv2 {
 // boost::log backend sink to provide MongoDB style file rotation.
@@ -41,7 +42,7 @@ namespace mongo::logv2 {
 // boost file rotation sink does not do.
 class FileRotateSink : public boost::log::sinks::text_ostream_backend {
 public:
-    FileRotateSink();
+    FileRotateSink(LogTimestampFormat timestampFormat);
     ~FileRotateSink();
 
     Status addFile(const std::string& filename, bool append);
@@ -49,6 +50,8 @@ public:
 
     Status rotate(bool rename, StringData renameSuffix);
 
+    void consume(const boost::log::record_view& rec, const string_type& formatted_string);
+
 private:
     struct Impl;
     std::unique_ptr<Impl> _impl;
diff --git a/src/mongo/logv2/json_formatter.cpp b/src/mongo/logv2/json_formatter.cpp
index ebdf8596973..81007d59c6b 100644
--- a/src/mongo/logv2/json_formatter.cpp
+++ b/src/mongo/logv2/json_formatter.cpp
@@ -36,16 +36,10 @@
 
 #include "mongo/bson/bsonobj.h"
 #include "mongo/bson/bsonobjbuilder.h"
-#include "mongo/logv2/attribute_storage.h"
 #include "mongo/logv2/attributes.h"
 #include "mongo/logv2/constants.h"
-#include "mongo/logv2/log_component.h"
-#include "mongo/logv2/log_severity.h"
-#include "mongo/logv2/log_tag.h"
-#include "mongo/logv2/log_truncation.h"
 #include "mongo/logv2/name_extractor.h"
 #include "mongo/util/str_escape.h"
-#include "mongo/util/time_support.h"
 
 #include <fmt/compile.h>
 #include <fmt/format.h>
@@ -226,25 +220,23 @@ private:
 };
 }  // namespace
 
-void JSONFormatter::operator()(boost::log::record_view const& rec,
-                               boost::log::formatting_ostream& strm) const {
-    using namespace boost::log;
-
-    // Build a JSON object for the user attributes.
-    const auto& attrs = extract<TypeErasedAttributeStorage>(attributes::attributes(), rec).get();
-
-    StringData severity =
-        extract<LogSeverity>(attributes::severity(), rec).get().toStringDataCompact();
-    StringData component =
-        extract<LogComponent>(attributes::component(), rec).get().getNameForLog();
-
-    fmt::memory_buffer buffer;
+void JSONFormatter::format(fmt::memory_buffer& buffer,
+                           LogSeverity severity,
+                           LogComponent component,
+                           Date_t date,
+                           int32_t id,
+                           StringData context,
+                           StringData message,
+                           const TypeErasedAttributeStorage& attrs,
+                           LogTag tags,
+                           LogTruncation truncation) const {
+    StringData severityString = severity.toStringDataCompact();
+    StringData componentString = component.getNameForLog();
 
     // Put all fields up until the message value
     static const auto& fmtStrOpen = *new auto(fmt::compile<StringData>(R"({{)"
                                                                        R"("{}":{{"$date":")"));
     compiled_format_to(buffer, fmtStrOpen, constants::kTimestampFieldName);
-    Date_t date = extract<Date_t>(attributes::timeStamp(), rec).get();
     switch (_timestampFormat) {
         case LogTimestampFormat::kISO8601UTC:
             outputDateAsISOStringUTC(buffer, date);
@@ -278,24 +270,24 @@ void JSONFormatter::operator()(boost::log::record_view const& rec,
         fmtStrBody,
         // severity, left align the comma and add padding to create fixed column width
         constants::kSeverityFieldName,
-        severity,
+        severityString,
         ","_sd,
-        3 - severity.size(),
+        3 - severityString.size(),
         // component, left align the comma and add padding to create fixed column width
         constants::kComponentFieldName,
-        component,
+        componentString,
         ","_sd,
-        9 - component.size(),
+        9 - componentString.size(),
         // id
         constants::kIdFieldName,
-        extract<int32_t>(attributes::id(), rec).get(),
+        id,
         // context
         constants::kContextFieldName,
-        extract<StringData>(attributes::threadName(), rec).get(),
+        context,
         // message
         constants::kMessageFieldName);
 
-    str::escapeForJSON(buffer, extract<StringData>(attributes::message(), rec).get());
+    str::escapeForJSON(buffer, message);
     buffer.push_back('"');
 
     static const auto& fmtStrAttr = *new auto(fmt::compile<StringData>(R"(,"{}":{{)"));
@@ -304,7 +296,7 @@ void JSONFormatter::operator()(boost::log::record_view const& rec,
         compiled_format_to(buffer, fmtStrAttr, constants::kAttributesFieldName);
         // comma separated list of attributes (no opening/closing brace are added here)
         size_t attributeMaxSize = 0;
-        if (extract<LogTruncation>(attributes::truncation(), rec).get() == LogTruncation::Enabled) {
+        if (truncation == LogTruncation::Enabled) {
             if (_maxAttributeSizeKB)
                 attributeMaxSize = _maxAttributeSizeKB->loadRelaxed() * 1024;
             else
@@ -327,7 +319,6 @@ void JSONFormatter::operator()(boost::log::record_view const& rec,
         }
     }
 
-    LogTag tags = extract<LogTag>(attributes::tags(), rec).get();
     static const auto& fmtStrTags = *new auto(fmt::compile<StringData>(R"(,"{}":)"));
     if (tags != LogTag::kNone) {
         compiled_format_to(buffer, fmtStrTags, constants::kTagsFieldName);
@@ -336,6 +327,24 @@ void JSONFormatter::operator()(boost::log::record_view const& rec,
     }
 
     buffer.push_back('}');
+}
+
+void JSONFormatter::operator()(boost::log::record_view const& rec,
+                               boost::log::formatting_ostream& strm) const {
+    using boost::log::extract;
+
+    fmt::memory_buffer buffer;
+
+    format(buffer,
+           extract<LogSeverity>(attributes::severity(), rec).get(),
+           extract<LogComponent>(attributes::component(), rec).get(),
+           extract<Date_t>(attributes::timeStamp(), rec).get(),
+           extract<int32_t>(attributes::id(), rec).get(),
+           extract<StringData>(attributes::threadName(), rec).get(),
+           extract<StringData>(attributes::message(), rec).get(),
+           extract<TypeErasedAttributeStorage>(attributes::attributes(), rec).get(),
+           extract<LogTag>(attributes::tags(), rec).get(),
+           extract<LogTruncation>(attributes::truncation(), rec).get());
 
     // Write final JSON object to output stream
     strm.write(buffer.data(), buffer.size());
diff --git a/src/mongo/logv2/json_formatter.h b/src/mongo/logv2/json_formatter.h
index cd511516eb1..9e3cd666e1a 100644
--- a/src/mongo/logv2/json_formatter.h
+++ b/src/mongo/logv2/json_formatter.h
@@ -32,8 +32,14 @@
 #include <boost/log/core/record_view.hpp>
 #include <boost/log/utility/formatting_ostream_fwd.hpp>
 
+#include "mongo/logv2/attribute_storage.h"
 #include "mongo/logv2/constants.h"
+#include "mongo/logv2/log_component.h"
 #include "mongo/logv2/log_format.h"
+#include "mongo/logv2/log_severity.h"
+#include "mongo/logv2/log_tag.h"
+#include "mongo/logv2/log_truncation.h"
+#include "mongo/util/time_support.h"
 
 namespace mongo::logv2 {
 
@@ -43,6 +49,16 @@ public:
                   LogTimestampFormat timestampFormat = LogTimestampFormat::kISO8601UTC)
         : _maxAttributeSizeKB(maxAttributeSizeKB), _timestampFormat(timestampFormat) {}
 
+    void format(fmt::memory_buffer& buffer,
+                LogSeverity severity,
+                LogComponent component,
+                Date_t date,
+                int32_t id,
+                StringData context,
+                StringData message,
+                const TypeErasedAttributeStorage& attrs,
+                LogTag tags,
+                LogTruncation truncation) const;
     void operator()(boost::log::record_view const& rec, boost::log::formatting_ostream& strm) const;
 
 private:
diff --git a/src/mongo/logv2/log_domain_global.cpp b/src/mongo/logv2/log_domain_global.cpp
index 050dbe1958f..35d4d6427ba 100644
--- a/src/mongo/logv2/log_domain_global.cpp
+++ b/src/mongo/logv2/log_domain_global.cpp
@@ -168,7 +168,7 @@ Status LogDomainGlobal::Impl::configure(LogDomainGlobal::ConfigurationOptions co
 
     if (options.fileEnabled) {
         auto backend = boost::make_shared<RotatableFileBackend>(
-            boost::make_shared<FileRotateSink>(),
+            boost::make_shared<FileRotateSink>(options.timestampFormat),
             boost::make_shared<RamLogSink>(RamLog::get("global")),
             boost::make_shared<RamLogSink>(RamLog::get("startupWarnings")),
             boost::make_shared<UserAssertSink>());