From 2810db8896aef992209ba13563104ed4d12c53a0 Mon Sep 17 00:00:00 2001 From: Henrik Edin Date: Tue, 19 Nov 2019 21:44:48 +0000 Subject: SERVER-44556 Move logging formatters into cpp files. --- src/mongo/SConscript | 3 + src/mongo/logv2/json_formatter.cpp | 240 ++++++++++++++++++++++++++++++++++++ src/mongo/logv2/json_formatter.h | 213 +------------------------------- src/mongo/logv2/plain_formatter.cpp | 92 ++++++++++++++ src/mongo/logv2/plain_formatter.h | 66 +--------- src/mongo/logv2/text_formatter.cpp | 65 ++++++++++ src/mongo/logv2/text_formatter.h | 27 +--- 7 files changed, 411 insertions(+), 295 deletions(-) create mode 100644 src/mongo/logv2/json_formatter.cpp create mode 100644 src/mongo/logv2/plain_formatter.cpp create mode 100644 src/mongo/logv2/text_formatter.cpp diff --git a/src/mongo/SConscript b/src/mongo/SConscript index 66e0eca7806..8cbe6309cd8 100644 --- a/src/mongo/SConscript +++ b/src/mongo/SConscript @@ -111,6 +111,7 @@ baseEnv.Library( 'logger/rotatable_file_writer.cpp', 'logv2/attributes.cpp', 'logv2/console.cpp', + 'logv2/json_formatter.cpp', 'logv2/log_detail.cpp', 'logv2/log_component.cpp', 'logv2/log_component_settings.cpp', @@ -120,7 +121,9 @@ baseEnv.Library( 'logv2/log_manager.cpp', 'logv2/log_severity.cpp', 'logv2/log_tag.cpp', + 'logv2/plain_formatter.cpp', 'logv2/ramlog.cpp', + 'logv2/text_formatter.cpp', 'platform/decimal128.cpp', 'platform/mutex.cpp', 'platform/posix_fadvise.cpp', diff --git a/src/mongo/logv2/json_formatter.cpp b/src/mongo/logv2/json_formatter.cpp new file mode 100644 index 00000000000..38d879c4f90 --- /dev/null +++ b/src/mongo/logv2/json_formatter.cpp @@ -0,0 +1,240 @@ +/** + * Copyright (C) 2019-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/logv2/json_formatter.h" + +#include +#include +#include +#include + +#include "mongo/bson/bsonobj.h" +#include "mongo/logv2/attribute_storage.h" +#include "mongo/logv2/attributes.h" +#include "mongo/logv2/constants.h" +#include "mongo/logv2/log_component.h" +#include "mongo/logv2/log_severity.h" +#include "mongo/logv2/log_tag.h" +#include "mongo/util/time_support.h" + +#include + +namespace mongo::logv2 { +namespace { +struct JsonValueExtractor { + void operator()(StringData name, CustomAttributeValue const& val) { + if (val.toBSON) { + // This is a JSON subobject, select overload that does not surround value with quotes + operator()(name, val.toBSON().jsonString()); + } else { + // This is a string, select overload that surrounds value with quotes + operator()(name, StringData(val.toString())); + } + } + + void operator()(StringData name, const BSONObj* val) { + // This is a JSON subobject, select overload that does not surround value with quotes + operator()(name, val->jsonString()); + } + + void operator()(StringData name, StringData value) { + // The first {} is for the member separator, added by store() + store(R"({}"{}":"{}")", name, value); + } + + template + void operator()(StringData name, const T& value) { + // The first {} is for the member separator, added by store() + store(R"({}"{}":{})", name, value); + } + + template + void store(const char* fmt_str, StringData name, const T& value) { + nameArgs.push_back(fmt::internal::make_arg(name)); + fmt::format_to(buffer, fmt_str, _separator, name, value); + _separator = ","_sd; + } + + fmt::memory_buffer buffer; + boost::container::small_vector, + constants::kNumStaticAttrs> + nameArgs; + +private: + StringData _separator = ""_sd; +}; + +class NamedArgFormatter : public fmt::arg_formatter> { +public: + typedef fmt::arg_formatter> arg_formatter; + + NamedArgFormatter(fmt::format_context& ctx, + fmt::basic_parse_context* parse_ctx = nullptr, + fmt::format_specs* spec = nullptr) + : arg_formatter(ctx, parse_ctx, nullptr) { + // Pretend that we have no format_specs, but store so we can re-construct the format + // specifier + if (spec) { + spec_ = *spec; + } + } + + + using arg_formatter::operator(); + + auto operator()(fmt::string_view name) { + using namespace fmt; + + write("{"); + arg_formatter::operator()(name); + // If formatting spec was provided, reconstruct the string. Libfmt does not provide this + // unfortunately + if (spec_) { + char str[2] = {'\0', '\0'}; + write(":"); + if (spec_->fill() != ' ') { + str[0] = static_cast(spec_->fill()); + write(str); + } + + switch (spec_->align()) { + case ALIGN_LEFT: + write("<"); + break; + case ALIGN_RIGHT: + write(">"); + break; + case ALIGN_CENTER: + write("^"); + break; + case ALIGN_NUMERIC: + write("="); + break; + default: + break; + }; + + if (spec_->has(PLUS_FLAG)) + write("+"); + else if (spec_->has(MINUS_FLAG)) + write("-"); + else if (spec_->has(SIGN_FLAG)) + write(" "); + else if (spec_->has(HASH_FLAG)) + write("#"); + + if (spec_->align() == ALIGN_NUMERIC && spec_->fill() == 0) + write("0"); + + if (spec_->width() > 0) + write(std::to_string(spec_->width()).c_str()); + + if (spec_->has_precision()) { + write("."); + write(std::to_string(spec_->precision).c_str()); + } + str[0] = spec_->type; + write(str); + } + write("}"); + return out(); + } + +private: + boost::optional spec_; +}; +} // namespace + +void JsonFormatter::operator()(boost::log::record_view const& rec, + boost::log::formatting_ostream& strm) const { + using namespace boost::log; + + // Build a JSON object for the user attributes. + const auto& attrs = extract(attributes::attributes(), rec).get(); + + JsonValueExtractor extractor; + attrs.apply(extractor); + + std::string id; + auto stable_id = extract(attributes::stableId(), rec).get(); + if (!stable_id.empty()) { + id = fmt::format("\"id\":\"{}\",", stable_id); + } + + std::string message; + fmt::memory_buffer buffer; + fmt::vformat_to( + buffer, + extract(attributes::message(), rec).get().toString(), + fmt::basic_format_args(extractor.nameArgs.data(), + extractor.nameArgs.size())); + message = fmt::to_string(buffer); + + StringData severity = + extract(attributes::severity(), rec).get().toStringDataCompact(); + StringData component = + extract(attributes::component(), rec).get().getNameForLog(); + std::string tag; + LogTag tags = extract(attributes::tags(), rec).get(); + if (tags != LogTag::kNone) { + tag = fmt::format(",\"tags\":{}", tags.toJSONArray()); + } + + strm << fmt::format(R"({{)" + R"("t":"{}",)" // timestamp + R"("s":"{}"{: <{}})" // severity with padding for the comma + R"("c":"{}"{: <{}})" // component with padding for the comma + R"("ctx":"{}",)" // context + R"({})" // optional stable id + R"("msg":"{}")" // message + R"({})", // optional attribute key + dateToISOStringUTC(extract(attributes::timeStamp(), rec).get()), + severity, + ",", + 3 - severity.size(), + component, + ",", + 9 - component.size(), + extract(attributes::threadName(), rec).get(), + id, + message, + attrs.empty() ? "" : ",\"attr\":{"); + + if (!attrs.empty()) { + strm << fmt::to_string(extractor.buffer); + } + + strm << fmt::format(R"({})" // optional attribute closing + R"({})" // optional tags + R"(}})", + attrs.empty() ? "" : "}", + tag); +} + +} // namespace mongo::logv2 diff --git a/src/mongo/logv2/json_formatter.h b/src/mongo/logv2/json_formatter.h index 1d266fd871a..428398aab16 100644 --- a/src/mongo/logv2/json_formatter.h +++ b/src/mongo/logv2/json_formatter.h @@ -29,68 +29,10 @@ #pragma once -#include -#include #include -#include -#include +#include -#include "mongo/bson/bsonobj.h" -#include "mongo/logv2/attribute_storage.h" -#include "mongo/logv2/attributes.h" -#include "mongo/logv2/constants.h" -#include "mongo/logv2/log_component.h" -#include "mongo/logv2/log_severity.h" -#include "mongo/logv2/log_tag.h" -#include "mongo/util/time_support.h" - -#include - -namespace mongo { -namespace logv2 { - -struct JsonValueExtractor { - void operator()(StringData name, CustomAttributeValue const& val) { - if (val.toBSON) { - // This is a JSON subobject, select overload that does not surround value with quotes - operator()(name, val.toBSON().jsonString()); - } else { - // This is a string, select overload that surrounds value with quotes - operator()(name, StringData(val.toString())); - } - } - - void operator()(StringData name, const BSONObj* val) { - // This is a JSON subobject, select overload that does not surround value with quotes - operator()(name, val->jsonString()); - } - - void operator()(StringData name, StringData value) { - // The first {} is for the member separator, added by store() - store(R"({}"{}":"{}")", name, value); - } - - template - void operator()(StringData name, const T& value) { - // The first {} is for the member separator, added by store() - store(R"({}"{}":{})", name, value); - } - - template - void store(const char* fmt_str, StringData name, const T& value) { - nameArgs.push_back(fmt::internal::make_arg(name)); - fmt::format_to(buffer, fmt_str, _separator, name, value); - _separator = ","_sd; - } - - fmt::memory_buffer buffer; - boost::container::small_vector, - constants::kNumStaticAttrs> - nameArgs; - -private: - StringData _separator = ""_sd; -}; +namespace mongo::logv2 { class JsonFormatter { public: @@ -98,154 +40,7 @@ public: return false; }; - void operator()(boost::log::record_view const& rec, - boost::log::formatting_ostream& strm) const { - using namespace boost::log; - - // Build a JSON object for the user attributes. - const auto& attrs = - extract(attributes::attributes(), rec).get(); - - JsonValueExtractor extractor; - attrs.apply(extractor); - - std::string id; - auto stable_id = extract(attributes::stableId(), rec).get(); - if (!stable_id.empty()) { - id = fmt::format("\"id\":\"{}\",", stable_id); - } - - std::string message; - fmt::memory_buffer buffer; - fmt::vformat_to( - buffer, - extract(attributes::message(), rec).get().toString(), - fmt::basic_format_args(extractor.nameArgs.data(), - extractor.nameArgs.size())); - message = fmt::to_string(buffer); - - StringData severity = - extract(attributes::severity(), rec).get().toStringDataCompact(); - StringData component = - extract(attributes::component(), rec).get().getNameForLog(); - std::string tag; - LogTag tags = extract(attributes::tags(), rec).get(); - if (tags != LogTag::kNone) { - tag = fmt::format(",\"tags\":{}", tags.toJSONArray()); - } - - strm << fmt::format(R"({{)" - R"("t":"{}",)" // timestamp - R"("s":"{}"{: <{}})" // severity with padding for the comma - R"("c":"{}"{: <{}})" // component with padding for the comma - R"("ctx":"{}",)" // context - R"({})" // optional stable id - R"("msg":"{}")" // message - R"({})", // optional attribute key - dateToISOStringUTC(extract(attributes::timeStamp(), rec).get()), - severity, - ",", - 3 - severity.size(), - component, - ",", - 9 - component.size(), - extract(attributes::threadName(), rec).get(), - id, - message, - attrs.empty() ? "" : ",\"attr\":{"); - - if (!attrs.empty()) { - strm << fmt::to_string(extractor.buffer); - } - - strm << fmt::format(R"({})" // optional attribute closing - R"({})" // optional tags - R"(}})", - attrs.empty() ? "" : "}", - tag); - } - -private: - class NamedArgFormatter : public fmt::arg_formatter> { - public: - typedef fmt::arg_formatter> arg_formatter; - - NamedArgFormatter(fmt::format_context& ctx, - fmt::basic_parse_context* parse_ctx = nullptr, - fmt::format_specs* spec = nullptr) - : arg_formatter(ctx, parse_ctx, nullptr) { - // Pretend that we have no format_specs, but store so we can re-construct the format - // specifier - if (spec) { - spec_ = *spec; - } - } - - - using arg_formatter::operator(); - - auto operator()(fmt::string_view name) { - using namespace fmt; - - write("{"); - arg_formatter::operator()(name); - // If formatting spec was provided, reconstruct the string. Libfmt does not provide this - // unfortunately - if (spec_) { - char str[2] = {'\0', '\0'}; - write(":"); - if (spec_->fill() != ' ') { - str[0] = static_cast(spec_->fill()); - write(str); - } - - switch (spec_->align()) { - case ALIGN_LEFT: - write("<"); - break; - case ALIGN_RIGHT: - write(">"); - break; - case ALIGN_CENTER: - write("^"); - break; - case ALIGN_NUMERIC: - write("="); - break; - default: - break; - }; - - if (spec_->has(PLUS_FLAG)) - write("+"); - else if (spec_->has(MINUS_FLAG)) - write("-"); - else if (spec_->has(SIGN_FLAG)) - write(" "); - else if (spec_->has(HASH_FLAG)) - write("#"); - - if (spec_->align() == ALIGN_NUMERIC && spec_->fill() == 0) - write("0"); - - if (spec_->width() > 0) - write(std::to_string(spec_->width()).c_str()); - - if (spec_->has_precision()) { - write("."); - write(std::to_string(spec_->precision).c_str()); - } - str[0] = spec_->type; - write(str); - } - write("}"); - return out(); - } - - private: - boost::optional spec_; - }; + void operator()(boost::log::record_view const& rec, boost::log::formatting_ostream& strm) const; }; -} // namespace logv2 -} // namespace mongo +} // namespace mongo::logv2 diff --git a/src/mongo/logv2/plain_formatter.cpp b/src/mongo/logv2/plain_formatter.cpp new file mode 100644 index 00000000000..235172e6dda --- /dev/null +++ b/src/mongo/logv2/plain_formatter.cpp @@ -0,0 +1,92 @@ +/** + * Copyright (C) 2019-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/logv2/plain_formatter.h" + +#include "mongo/bson/bsonobj.h" +#include "mongo/logv2/attribute_storage.h" +#include "mongo/logv2/attributes.h" +#include "mongo/logv2/constants.h" + +#include +#include +#include +#include + +#include +#include + +namespace mongo::logv2 { +namespace { + +struct TextValueExtractor { + void operator()(StringData name, CustomAttributeValue const& val) { + _storage.push_back(val.toString()); + operator()(name, _storage.back()); + } + + void operator()(StringData name, const BSONObj* val) { + _storage.push_back(val->jsonString()); + operator()(name, _storage.back()); + } + + template + void operator()(StringData name, const T& val) { + args.push_back(fmt::internal::make_arg(val)); + } + + boost::container::small_vector, + constants::kNumStaticAttrs> + args; + +private: + std::deque _storage; +}; + +} // namespace + +void PlainFormatter::operator()(boost::log::record_view const& rec, + boost::log::formatting_ostream& strm) const { + using namespace boost::log; + + StringData message = extract(attributes::message(), rec).get(); + const auto& attrs = extract(attributes::attributes(), rec).get(); + + TextValueExtractor extractor; + extractor.args.reserve(attrs.size()); + attrs.apply(extractor); + fmt::memory_buffer buffer; + fmt::internal::vformat_to( + buffer, + to_string_view(message), + fmt::basic_format_args(extractor.args.data(), extractor.args.size())); + strm.write(buffer.data(), buffer.size()); +} + +} // namespace mongo::logv2 diff --git a/src/mongo/logv2/plain_formatter.h b/src/mongo/logv2/plain_formatter.h index bf1fb71536c..7b869703058 100644 --- a/src/mongo/logv2/plain_formatter.h +++ b/src/mongo/logv2/plain_formatter.h @@ -29,50 +29,10 @@ #pragma once -#include -#include #include -#include -#include +#include -#include "mongo/bson/bsonobj.h" -#include "mongo/logv2/attribute_storage.h" -#include "mongo/logv2/attributes.h" -#include "mongo/logv2/constants.h" -#include "mongo/logv2/log_component.h" -#include "mongo/logv2/log_severity.h" -#include "mongo/logv2/log_tag.h" -#include "mongo/util/time_support.h" - -#include -#include - -namespace mongo { -namespace logv2 { - -struct TextValueExtractor { - void operator()(StringData name, CustomAttributeValue const& val) { - _storage.push_back(val.toString()); - operator()(name, _storage.back()); - } - - void operator()(StringData name, const BSONObj* val) { - _storage.push_back(val->jsonString()); - operator()(name, _storage.back()); - } - - template - void operator()(StringData name, const T& val) { - args.push_back(fmt::internal::make_arg(val)); - } - - boost::container::small_vector, - constants::kNumStaticAttrs> - args; - -private: - std::deque _storage; -}; +namespace mongo::logv2 { // Text formatter without metadata. Just contains the formatted message. class PlainFormatter { @@ -81,25 +41,7 @@ public: return false; }; - void operator()(boost::log::record_view const& rec, - boost::log::formatting_ostream& strm) const { - using namespace boost::log; - - StringData message = extract(attributes::message(), rec).get(); - const auto& attrs = - extract(attributes::attributes(), rec).get(); - - TextValueExtractor extractor; - extractor.args.reserve(attrs.size()); - attrs.apply(extractor); - fmt::memory_buffer buffer; - fmt::internal::vformat_to(buffer, - to_string_view(message), - fmt::basic_format_args( - extractor.args.data(), extractor.args.size())); - strm.write(buffer.data(), buffer.size()); - } + void operator()(boost::log::record_view const& rec, boost::log::formatting_ostream& strm) const; }; -} // namespace logv2 -} // namespace mongo +} // namespace mongo::logv2 diff --git a/src/mongo/logv2/text_formatter.cpp b/src/mongo/logv2/text_formatter.cpp new file mode 100644 index 00000000000..da8bff39f6b --- /dev/null +++ b/src/mongo/logv2/text_formatter.cpp @@ -0,0 +1,65 @@ +/** + * Copyright (C) 2019-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/logv2/text_formatter.h" + +#include "mongo/logv2/attributes.h" +#include "mongo/logv2/log_component.h" +#include "mongo/logv2/log_severity.h" +#include "mongo/logv2/log_tag.h" +#include "mongo/util/time_support.h" + +#include +#include + +#include + +namespace mongo::logv2 { + +void TextFormatter::operator()(boost::log::record_view const& rec, + boost::log::formatting_ostream& strm) const { + using namespace boost::log; + + fmt::memory_buffer buffer; + fmt::format_to(buffer, + "{} {:<2} {:<8} [{}] ", + extract(attributes::timeStamp(), rec).get().toString(), + extract(attributes::severity(), rec).get().toStringDataCompact(), + extract(attributes::component(), rec).get().getNameForLog(), + extract(attributes::threadName(), rec).get()); + strm.write(buffer.data(), buffer.size()); + + if (extract(attributes::tags(), rec).get().has(LogTag::kStartupWarnings)) { + strm << "** WARNING: "; + } + + PlainFormatter::operator()(rec, strm); +} + +} // namespace mongo::logv2 diff --git a/src/mongo/logv2/text_formatter.h b/src/mongo/logv2/text_formatter.h index 017dbdc34b0..ccc61ffb0e0 100644 --- a/src/mongo/logv2/text_formatter.h +++ b/src/mongo/logv2/text_formatter.h @@ -31,8 +31,7 @@ #include "mongo/logv2/plain_formatter.h" -namespace mongo { -namespace logv2 { +namespace mongo::logv2 { class TextFormatter : protected PlainFormatter { public: @@ -40,27 +39,7 @@ public: return false; }; - void operator()(boost::log::record_view const& rec, - boost::log::formatting_ostream& strm) const { - using namespace boost::log; - - fmt::memory_buffer buffer; - fmt::format_to( - buffer, - "{} {:<2} {:<8} [{}] ", - extract(attributes::timeStamp(), rec).get().toString(), - extract(attributes::severity(), rec).get().toStringDataCompact(), - extract(attributes::component(), rec).get().getNameForLog(), - extract(attributes::threadName(), rec).get()); - strm.write(buffer.data(), buffer.size()); - - if (extract(attributes::tags(), rec).get().has(LogTag::kStartupWarnings)) { - strm << "** WARNING: "; - } - - PlainFormatter::operator()(rec, strm); - } + void operator()(boost::log::record_view const& rec, boost::log::formatting_ostream& strm) const; }; -} // namespace logv2 -} // namespace mongo +} // namespace mongo::logv2 -- cgit v1.2.1