SERVER-47198 rotateLogs: tolerate unexpectedly missing or unexpectedly existing files

7 files changed, 78 insertions, 38 deletions

diff --git a/src/mongo/logv2/file_rotate_sink.cpp b/src/mongo/logv2/file_rotate_sink.cpp
index af4b7256a0c..cf882cca235 100644
--- a/src/mongo/logv2/file_rotate_sink.cpp
+++ b/src/mongo/logv2/file_rotate_sink.cpp
@@ -46,6 +46,9 @@
 
 namespace mongo::logv2 {
 namespace {
+
+using namespace fmt::literals;
+
 #if _WIN32
 using stream_t = Win32SharedAccessOfstream;
 #else
@@ -96,35 +99,51 @@ void FileRotateSink::removeFile(const std::string& filename) {
     }
 }
 
-Status FileRotateSink::rotate(bool rename, StringData renameSuffix) {
+Status FileRotateSink::rotate(bool rename,
+                              StringData renameSuffix,
+                              std::function<void(Status)> onMinorError) {
     for (auto& file : _impl->files) {
         const std::string& filename = file.first;
         if (rename) {
             std::string renameTarget = filename + renameSuffix;
-            try {
-                if (boost::filesystem::exists(renameTarget)) {
-                    return Status(
-                        ErrorCodes::FileRenameFailed,
-                        fmt::format("Renaming file {} to {} failed; destination already exists",
-                                    filename,
-                                    renameTarget));
+
+            auto targetExists = [&]() -> StatusWith<bool> {
+                try {
+                    return boost::filesystem::exists(renameTarget);
+                } catch (const boost::exception&) {
+                    return exceptionToStatus();
+                }
+            }();
+
+            if (!targetExists.isOK()) {
+                return Status(ErrorCodes::FileRenameFailed, targetExists.getStatus().reason())
+                    .withContext("Cannot verify whether destination already exists: {}"_format(
+                        renameTarget));
+            }
+
+            if (targetExists.getValue()) {
+                if (onMinorError) {
+                    onMinorError({ErrorCodes::FileRenameFailed,
+                                  "Target already exists during log rotation. Skipping this file. "
+                                  "target={}, file={}"_format(renameTarget, filename)});
                 }
-            } catch (const std::exception& e) {
-                return Status(ErrorCodes::FileRenameFailed,
-                              fmt::format("Renaming file {} to {} failed; Cannot verify whether "
-                                          "destination already exists: {}",
-                                          filename,
-                                          renameTarget,
-                                          e.what()));
+                continue;
             }
 
             boost::system::error_code ec;
             boost::filesystem::rename(filename, renameTarget, ec);
             if (ec) {
-                return Status(
-                    ErrorCodes::FileRenameFailed,
-                    fmt::format(
-                        "Failed  to rename {} to {}: {}", filename, renameTarget, ec.message()));
+                if (ec == boost::system::errc::no_such_file_or_directory) {
+                    if (onMinorError)
+                        onMinorError(
+                            {ErrorCodes::FileRenameFailed,
+                             "Source file was missing during log rotation. Creating a new one. "
+                             "file={}"_format(filename)});
+                } else {
+                    return Status(ErrorCodes::FileRenameFailed,
+                                  "Failed to rename {} to {}: {}"_format(
+                                      filename, renameTarget, ec.message()));
+                }
             }
         }
 
diff --git a/src/mongo/logv2/file_rotate_sink.h b/src/mongo/logv2/file_rotate_sink.h
index d0ae4527b4c..f4c492fa417 100644
--- a/src/mongo/logv2/file_rotate_sink.h
+++ b/src/mongo/logv2/file_rotate_sink.h
@@ -48,7 +48,7 @@ public:
     Status addFile(const std::string& filename, bool append);
     void removeFile(const std::string& filename);
 
-    Status rotate(bool rename, StringData renameSuffix);
+    Status rotate(bool rename, StringData renameSuffix, std::function<void(Status)> onMinorError);
 
     void consume(const boost::log::record_view& rec, const string_type& formatted_string);
 
diff --git a/src/mongo/logv2/log_domain_global.cpp b/src/mongo/logv2/log_domain_global.cpp
index d9591e9cf19..e1124cb7463 100644
--- a/src/mongo/logv2/log_domain_global.cpp
+++ b/src/mongo/logv2/log_domain_global.cpp
@@ -27,6 +27,8 @@
  *    it in the license file.
  */
 
+#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kDefault
+
 #include "log_domain_global.h"
 
 #include "mongo/config.h"
@@ -35,6 +37,7 @@
 #include "mongo/logv2/console.h"
 #include "mongo/logv2/file_rotate_sink.h"
 #include "mongo/logv2/json_formatter.h"
+#include "mongo/logv2/log.h"
 #include "mongo/logv2/log_source.h"
 #include "mongo/logv2/ramlog_sink.h"
 #include "mongo/logv2/shared_access_fstream.h"
@@ -71,7 +74,7 @@ struct LogDomainGlobal::Impl {
 
     Impl(LogDomainGlobal& parent);
     Status configure(LogDomainGlobal::ConfigurationOptions const& options);
-    Status rotate(bool rename, StringData renameSuffix);
+    Status rotate(bool rename, StringData renameSuffix, std::function<void(Status)> onMinorError);
 
     const ConfigurationOptions& config() const;
 
@@ -222,12 +225,21 @@ const LogDomainGlobal::ConfigurationOptions& LogDomainGlobal::Impl::config() con
     return _config;
 }
 
-Status LogDomainGlobal::Impl::rotate(bool rename, StringData renameSuffix) {
-    if (_rotatableFileSink) {
-        auto backend = _rotatableFileSink->locked_backend()->lockedBackend<0>();
-        return backend->rotate(rename, renameSuffix);
-    }
-    return Status::OK();
+Status LogDomainGlobal::Impl::rotate(bool rename,
+                                     StringData renameSuffix,
+                                     std::function<void(Status)> onMinorError) {
+    if (!_rotatableFileSink)
+        return Status::OK();
+    std::vector<Status> errors;
+    Status result = _rotatableFileSink->locked_backend()->lockedBackend<0>()->rotate(
+        rename, renameSuffix, [&](Status s) {
+            errors.push_back(s);
+            if (onMinorError)
+                onMinorError(s);
+        });
+    if (!errors.empty())
+        LOGV2_WARNING(4719804, "Errors occurred during log rotate", "errors"_attr = errors);
+    return result;
 }
 
 LogSource& LogDomainGlobal::Impl::source() {
@@ -279,8 +291,10 @@ const LogDomainGlobal::ConfigurationOptions& LogDomainGlobal::config() const {
     return _impl->config();
 }
 
-Status LogDomainGlobal::rotate(bool rename, StringData renameSuffix) {
-    return _impl->rotate(rename, renameSuffix);
+Status LogDomainGlobal::rotate(bool rename,
+                               StringData renameSuffix,
+                               std::function<void(Status)> onMinorError) {
+    return _impl->rotate(rename, renameSuffix, onMinorError);
 }
 
 LogComponentSettings& LogDomainGlobal::settings() {
diff --git a/src/mongo/logv2/log_domain_global.h b/src/mongo/logv2/log_domain_global.h
index 58467c33b86..59b47f5f6a8 100644
--- a/src/mongo/logv2/log_domain_global.h
+++ b/src/mongo/logv2/log_domain_global.h
@@ -60,7 +60,7 @@ public:
     LogSource& source() override;
 
     Status configure(ConfigurationOptions const& options);
-    Status rotate(bool rename, StringData renameSuffix);
+    Status rotate(bool rename, StringData renameSuffix, std::function<void(Status)> onMinorError);
 
     const ConfigurationOptions& config() const;
 
diff --git a/src/mongo/logv2/log_manager.cpp b/src/mongo/logv2/log_manager.cpp
index 4edc4eac83e..f9841d7fd79 100644
--- a/src/mongo/logv2/log_manager.cpp
+++ b/src/mongo/logv2/log_manager.cpp
@@ -101,9 +101,12 @@ LogComponentSettings& LogManager::getGlobalSettings() {
 }
 
 MONGO_INITIALIZER(GlobalLogRotator)(InitializerContext*) {
-    addLogRotator(logv2::kServerLogTag, [](bool renameFiles, StringData suffix) {
-        return LogManager::global().getGlobalDomainInternal().rotate(renameFiles, suffix);
-    });
+    addLogRotator(
+        logv2::kServerLogTag,
+        [](bool renameFiles, StringData suffix, std::function<void(Status)> onMinorError) {
+            return LogManager::global().getGlobalDomainInternal().rotate(
+                renameFiles, suffix, onMinorError);
+        });
 }
 
 }  // namespace mongo::logv2
diff --git a/src/mongo/logv2/log_util.cpp b/src/mongo/logv2/log_util.cpp
index 63aadc11735..ded9e390e89 100644
--- a/src/mongo/logv2/log_util.cpp
+++ b/src/mongo/logv2/log_util.cpp
@@ -49,7 +49,9 @@ void addLogRotator(StringData logType, LogRotateCallback cb) {
     logRotateCallbacks.emplace(logType, std::move(cb));
 }
 
-bool rotateLogs(bool renameFiles, boost::optional<StringData> logType) {
+bool rotateLogs(bool renameFiles,
+                boost::optional<StringData> logType,
+                std::function<void(Status)> onMinorError) {
     std::string suffix = "." + terseCurrentTimeForFilename();
 
     LOGV2(23166, "Log rotation initiated", "suffix"_attr = suffix, "logType"_attr = logType);
@@ -61,7 +63,7 @@ bool rotateLogs(bool renameFiles, boost::optional<StringData> logType) {
                 ErrorCodes::NoSuchKey, "Unknown log type for rotate", "logType"_attr = logType);
             return false;
         }
-        auto status = it->second(renameFiles, suffix);
+        auto status = it->second(renameFiles, suffix, onMinorError);
         if (!status.isOK()) {
             LOGV2_WARNING(
                 1947001, "Log rotation failed", "reason"_attr = status, "logType"_attr = logType);
@@ -71,7 +73,7 @@ bool rotateLogs(bool renameFiles, boost::optional<StringData> logType) {
     } else {
         bool ret = true;
         for (const auto& entry : logRotateCallbacks) {
-            auto status = entry.second(renameFiles, suffix);
+            auto status = entry.second(renameFiles, suffix, onMinorError);
             if (!status.isOK()) {
                 LOGV2_WARNING(23168,
                               "Log rotation failed",
diff --git a/src/mongo/logv2/log_util.h b/src/mongo/logv2/log_util.h
index 1bbafffda85..a60e5f1dd67 100644
--- a/src/mongo/logv2/log_util.h
+++ b/src/mongo/logv2/log_util.h
@@ -40,7 +40,7 @@ namespace mongo::logv2 {
 constexpr auto kServerLogTag = "server"_sd;
 constexpr auto kAuditLogTag = "audit"_sd;
 
-using LogRotateCallback = std::function<Status(bool, StringData)>;
+using LogRotateCallback = std::function<Status(bool, StringData, std::function<void(Status)>)>;
 
 /**
  * logType param needs to have static lifetime. If a new logType needs to be defined, add it above
@@ -59,7 +59,9 @@ void addLogRotator(StringData logType, LogRotateCallback cb);
  * We expect logrotate to rename the existing file before we rotate, and so the next open
  * we do should result in a file create.
  */
-bool rotateLogs(bool renameFiles, boost::optional<StringData> logType = boost::none);
+bool rotateLogs(bool renameFiles,
+                boost::optional<StringData> logType,
+                std::function<void(Status)> onMinorError);
 
 /**
  * Returns true if system logs should be redacted.