diff options
author | Billy Donahue <billy.donahue@mongodb.com> | 2020-09-22 18:52:22 +0000 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2021-06-28 21:41:58 +0000 |
commit | 75fdaf5d32a8ece4de9ed673b10d1d02256f4172 (patch) | |
tree | 2f2dd4bfc1699475fc73a1fc0c0e0e4eea93ee58 /src/mongo/logv2 | |
parent | 0e68f3ec099396bb26a121728915cc1efefb8002 (diff) | |
download | mongo-75fdaf5d32a8ece4de9ed673b10d1d02256f4172.tar.gz |
SERVER-47198 rotateLogs: tolerate unexpectedly missing or unexpectedly existing files
Diffstat (limited to 'src/mongo/logv2')
-rw-r--r-- | src/mongo/logv2/file_rotate_sink.cpp | 57 | ||||
-rw-r--r-- | src/mongo/logv2/file_rotate_sink.h | 2 | ||||
-rw-r--r-- | src/mongo/logv2/log_domain_global.cpp | 32 | ||||
-rw-r--r-- | src/mongo/logv2/log_domain_global.h | 2 | ||||
-rw-r--r-- | src/mongo/logv2/log_manager.cpp | 9 | ||||
-rw-r--r-- | src/mongo/logv2/log_util.cpp | 8 | ||||
-rw-r--r-- | src/mongo/logv2/log_util.h | 6 |
7 files changed, 78 insertions, 38 deletions
diff --git a/src/mongo/logv2/file_rotate_sink.cpp b/src/mongo/logv2/file_rotate_sink.cpp index af4b7256a0c..cf882cca235 100644 --- a/src/mongo/logv2/file_rotate_sink.cpp +++ b/src/mongo/logv2/file_rotate_sink.cpp @@ -46,6 +46,9 @@ namespace mongo::logv2 { namespace { + +using namespace fmt::literals; + #if _WIN32 using stream_t = Win32SharedAccessOfstream; #else @@ -96,35 +99,51 @@ void FileRotateSink::removeFile(const std::string& filename) { } } -Status FileRotateSink::rotate(bool rename, StringData renameSuffix) { +Status FileRotateSink::rotate(bool rename, + StringData renameSuffix, + std::function<void(Status)> onMinorError) { for (auto& file : _impl->files) { const std::string& filename = file.first; if (rename) { std::string renameTarget = filename + renameSuffix; - try { - if (boost::filesystem::exists(renameTarget)) { - return Status( - ErrorCodes::FileRenameFailed, - fmt::format("Renaming file {} to {} failed; destination already exists", - filename, - renameTarget)); + + auto targetExists = [&]() -> StatusWith<bool> { + try { + return boost::filesystem::exists(renameTarget); + } catch (const boost::exception&) { + return exceptionToStatus(); + } + }(); + + if (!targetExists.isOK()) { + return Status(ErrorCodes::FileRenameFailed, targetExists.getStatus().reason()) + .withContext("Cannot verify whether destination already exists: {}"_format( + renameTarget)); + } + + if (targetExists.getValue()) { + if (onMinorError) { + onMinorError({ErrorCodes::FileRenameFailed, + "Target already exists during log rotation. Skipping this file. " + "target={}, file={}"_format(renameTarget, filename)}); } - } catch (const std::exception& e) { - return Status(ErrorCodes::FileRenameFailed, - fmt::format("Renaming file {} to {} failed; Cannot verify whether " - "destination already exists: {}", - filename, - renameTarget, - e.what())); + continue; } boost::system::error_code ec; boost::filesystem::rename(filename, renameTarget, ec); if (ec) { - return Status( - ErrorCodes::FileRenameFailed, - fmt::format( - "Failed to rename {} to {}: {}", filename, renameTarget, ec.message())); + if (ec == boost::system::errc::no_such_file_or_directory) { + if (onMinorError) + onMinorError( + {ErrorCodes::FileRenameFailed, + "Source file was missing during log rotation. Creating a new one. " + "file={}"_format(filename)}); + } else { + return Status(ErrorCodes::FileRenameFailed, + "Failed to rename {} to {}: {}"_format( + filename, renameTarget, ec.message())); + } } } diff --git a/src/mongo/logv2/file_rotate_sink.h b/src/mongo/logv2/file_rotate_sink.h index d0ae4527b4c..f4c492fa417 100644 --- a/src/mongo/logv2/file_rotate_sink.h +++ b/src/mongo/logv2/file_rotate_sink.h @@ -48,7 +48,7 @@ public: Status addFile(const std::string& filename, bool append); void removeFile(const std::string& filename); - Status rotate(bool rename, StringData renameSuffix); + Status rotate(bool rename, StringData renameSuffix, std::function<void(Status)> onMinorError); void consume(const boost::log::record_view& rec, const string_type& formatted_string); diff --git a/src/mongo/logv2/log_domain_global.cpp b/src/mongo/logv2/log_domain_global.cpp index d9591e9cf19..e1124cb7463 100644 --- a/src/mongo/logv2/log_domain_global.cpp +++ b/src/mongo/logv2/log_domain_global.cpp @@ -27,6 +27,8 @@ * it in the license file. */ +#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kDefault + #include "log_domain_global.h" #include "mongo/config.h" @@ -35,6 +37,7 @@ #include "mongo/logv2/console.h" #include "mongo/logv2/file_rotate_sink.h" #include "mongo/logv2/json_formatter.h" +#include "mongo/logv2/log.h" #include "mongo/logv2/log_source.h" #include "mongo/logv2/ramlog_sink.h" #include "mongo/logv2/shared_access_fstream.h" @@ -71,7 +74,7 @@ struct LogDomainGlobal::Impl { Impl(LogDomainGlobal& parent); Status configure(LogDomainGlobal::ConfigurationOptions const& options); - Status rotate(bool rename, StringData renameSuffix); + Status rotate(bool rename, StringData renameSuffix, std::function<void(Status)> onMinorError); const ConfigurationOptions& config() const; @@ -222,12 +225,21 @@ const LogDomainGlobal::ConfigurationOptions& LogDomainGlobal::Impl::config() con return _config; } -Status LogDomainGlobal::Impl::rotate(bool rename, StringData renameSuffix) { - if (_rotatableFileSink) { - auto backend = _rotatableFileSink->locked_backend()->lockedBackend<0>(); - return backend->rotate(rename, renameSuffix); - } - return Status::OK(); +Status LogDomainGlobal::Impl::rotate(bool rename, + StringData renameSuffix, + std::function<void(Status)> onMinorError) { + if (!_rotatableFileSink) + return Status::OK(); + std::vector<Status> errors; + Status result = _rotatableFileSink->locked_backend()->lockedBackend<0>()->rotate( + rename, renameSuffix, [&](Status s) { + errors.push_back(s); + if (onMinorError) + onMinorError(s); + }); + if (!errors.empty()) + LOGV2_WARNING(4719804, "Errors occurred during log rotate", "errors"_attr = errors); + return result; } LogSource& LogDomainGlobal::Impl::source() { @@ -279,8 +291,10 @@ const LogDomainGlobal::ConfigurationOptions& LogDomainGlobal::config() const { return _impl->config(); } -Status LogDomainGlobal::rotate(bool rename, StringData renameSuffix) { - return _impl->rotate(rename, renameSuffix); +Status LogDomainGlobal::rotate(bool rename, + StringData renameSuffix, + std::function<void(Status)> onMinorError) { + return _impl->rotate(rename, renameSuffix, onMinorError); } LogComponentSettings& LogDomainGlobal::settings() { diff --git a/src/mongo/logv2/log_domain_global.h b/src/mongo/logv2/log_domain_global.h index 58467c33b86..59b47f5f6a8 100644 --- a/src/mongo/logv2/log_domain_global.h +++ b/src/mongo/logv2/log_domain_global.h @@ -60,7 +60,7 @@ public: LogSource& source() override; Status configure(ConfigurationOptions const& options); - Status rotate(bool rename, StringData renameSuffix); + Status rotate(bool rename, StringData renameSuffix, std::function<void(Status)> onMinorError); const ConfigurationOptions& config() const; diff --git a/src/mongo/logv2/log_manager.cpp b/src/mongo/logv2/log_manager.cpp index 4edc4eac83e..f9841d7fd79 100644 --- a/src/mongo/logv2/log_manager.cpp +++ b/src/mongo/logv2/log_manager.cpp @@ -101,9 +101,12 @@ LogComponentSettings& LogManager::getGlobalSettings() { } MONGO_INITIALIZER(GlobalLogRotator)(InitializerContext*) { - addLogRotator(logv2::kServerLogTag, [](bool renameFiles, StringData suffix) { - return LogManager::global().getGlobalDomainInternal().rotate(renameFiles, suffix); - }); + addLogRotator( + logv2::kServerLogTag, + [](bool renameFiles, StringData suffix, std::function<void(Status)> onMinorError) { + return LogManager::global().getGlobalDomainInternal().rotate( + renameFiles, suffix, onMinorError); + }); } } // namespace mongo::logv2 diff --git a/src/mongo/logv2/log_util.cpp b/src/mongo/logv2/log_util.cpp index 63aadc11735..ded9e390e89 100644 --- a/src/mongo/logv2/log_util.cpp +++ b/src/mongo/logv2/log_util.cpp @@ -49,7 +49,9 @@ void addLogRotator(StringData logType, LogRotateCallback cb) { logRotateCallbacks.emplace(logType, std::move(cb)); } -bool rotateLogs(bool renameFiles, boost::optional<StringData> logType) { +bool rotateLogs(bool renameFiles, + boost::optional<StringData> logType, + std::function<void(Status)> onMinorError) { std::string suffix = "." + terseCurrentTimeForFilename(); LOGV2(23166, "Log rotation initiated", "suffix"_attr = suffix, "logType"_attr = logType); @@ -61,7 +63,7 @@ bool rotateLogs(bool renameFiles, boost::optional<StringData> logType) { ErrorCodes::NoSuchKey, "Unknown log type for rotate", "logType"_attr = logType); return false; } - auto status = it->second(renameFiles, suffix); + auto status = it->second(renameFiles, suffix, onMinorError); if (!status.isOK()) { LOGV2_WARNING( 1947001, "Log rotation failed", "reason"_attr = status, "logType"_attr = logType); @@ -71,7 +73,7 @@ bool rotateLogs(bool renameFiles, boost::optional<StringData> logType) { } else { bool ret = true; for (const auto& entry : logRotateCallbacks) { - auto status = entry.second(renameFiles, suffix); + auto status = entry.second(renameFiles, suffix, onMinorError); if (!status.isOK()) { LOGV2_WARNING(23168, "Log rotation failed", diff --git a/src/mongo/logv2/log_util.h b/src/mongo/logv2/log_util.h index 1bbafffda85..a60e5f1dd67 100644 --- a/src/mongo/logv2/log_util.h +++ b/src/mongo/logv2/log_util.h @@ -40,7 +40,7 @@ namespace mongo::logv2 { constexpr auto kServerLogTag = "server"_sd; constexpr auto kAuditLogTag = "audit"_sd; -using LogRotateCallback = std::function<Status(bool, StringData)>; +using LogRotateCallback = std::function<Status(bool, StringData, std::function<void(Status)>)>; /** * logType param needs to have static lifetime. If a new logType needs to be defined, add it above @@ -59,7 +59,9 @@ void addLogRotator(StringData logType, LogRotateCallback cb); * We expect logrotate to rename the existing file before we rotate, and so the next open * we do should result in a file create. */ -bool rotateLogs(bool renameFiles, boost::optional<StringData> logType = boost::none); +bool rotateLogs(bool renameFiles, + boost::optional<StringData> logType, + std::function<void(Status)> onMinorError); /** * Returns true if system logs should be redacted. |