From 5976013e3655bd769833f32463048ffd4f45b065 Mon Sep 17 00:00:00 2001 From: Henrik Edin Date: Tue, 19 Nov 2019 18:41:45 +0000 Subject: SERVER-44541 Attribute packing and extraction for logv2 Basic integral and floating point types, string, BSONObj supported. Custom types supported via toString() and toBSON() hooks. --- src/mongo/logger/logv2_appender.h | 1 - src/mongo/logv2/attribute_argument_set.h | 48 ------ src/mongo/logv2/attribute_storage.h | 177 ++++++++++++++++++++ src/mongo/logv2/constants.h | 38 +++++ src/mongo/logv2/json_formatter.h | 138 ++++++++++------ src/mongo/logv2/log.h | 1 - src/mongo/logv2/log_detail.cpp | 5 +- src/mongo/logv2/log_detail.h | 15 +- src/mongo/logv2/log_test_v2.cpp | 275 +++++++++++++++++++++++++------ src/mongo/logv2/plain_formatter.h | 105 ++++++++++++ src/mongo/logv2/text_formatter.h | 26 +-- src/mongo/shell/dbshell.cpp | 10 +- 12 files changed, 648 insertions(+), 191 deletions(-) delete mode 100644 src/mongo/logv2/attribute_argument_set.h create mode 100644 src/mongo/logv2/attribute_storage.h create mode 100644 src/mongo/logv2/constants.h create mode 100644 src/mongo/logv2/plain_formatter.h diff --git a/src/mongo/logger/logv2_appender.h b/src/mongo/logger/logv2_appender.h index 67567ea1af1..d728cdf780f 100644 --- a/src/mongo/logger/logv2_appender.h +++ b/src/mongo/logger/logv2_appender.h @@ -32,7 +32,6 @@ #include "mongo/base/status.h" #include "mongo/logger/appender.h" #include "mongo/logger/log_version_util.h" -#include "mongo/logv2/attribute_argument_set.h" #include "mongo/logv2/log_component.h" #include "mongo/logv2/log_detail.h" #include "mongo/logv2/log_domain.h" diff --git a/src/mongo/logv2/attribute_argument_set.h b/src/mongo/logv2/attribute_argument_set.h deleted file mode 100644 index 039cc7a6b95..00000000000 --- a/src/mongo/logv2/attribute_argument_set.h +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Copyright (C) 2019-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * . - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#pragma once - -#include -#include - -#include "mongo/base/string_data.h" - -namespace mongo { -namespace logv2 { - -// Type erased set of provided libfmt named arguments. Index match between names and values. -struct AttributeArgumentSet { - boost::container::small_vector _names; - fmt::format_args _values; -}; - - -} // namespace logv2 -} // namespace mongo diff --git a/src/mongo/logv2/attribute_storage.h b/src/mongo/logv2/attribute_storage.h new file mode 100644 index 00000000000..cd6c37b785e --- /dev/null +++ b/src/mongo/logv2/attribute_storage.h @@ -0,0 +1,177 @@ +/** + * Copyright (C) 2019-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/base/string_data.h" +#include "mongo/stdx/variant.h" + +#include + +namespace mongo { +namespace logv2 { + +class TypeErasedAttributeStorage; + +// Custom type, storage for how to call its formatters +struct CustomAttributeValue { + std::function toBSON; + std::function toString; +}; + +namespace detail { +namespace { + +// Helper traits to figure out capabilities on custom types +template +struct HasToBSON : std::false_type {}; + +template +struct HasToBSON().toBSON())>> : std::true_type {}; + +template +struct HasToString : std::false_type {}; + +template +struct HasToString().toString())>> : std::true_type {}; + +} // namespace + +// Named attribute, storage for a name-value attribute. +class NamedAttribute { +public: + NamedAttribute() = default; + NamedAttribute(StringData n, int val) : name(n), value(val) {} + NamedAttribute(StringData n, unsigned int val) : name(n), value(val) {} + // long is 32bit on Windows and 64bit on posix. To avoid ambiguity where different platforms we + // treat long as 64bit always + NamedAttribute(StringData n, long val) : name(n), value(static_cast(val)) {} + NamedAttribute(StringData n, unsigned long val) + : name(n), value(static_cast(val)) {} + NamedAttribute(StringData n, long long val) : name(n), value(val) {} + NamedAttribute(StringData n, unsigned long long val) : name(n), value(val) {} + NamedAttribute(StringData n, double val) : name(n), value(val) {} + NamedAttribute(StringData n, bool val) : name(n), value(val) {} + NamedAttribute(StringData n, StringData val) : name(n), value(val) {} + NamedAttribute(StringData n, BSONObj const& val) : name(n), value(&val) {} + NamedAttribute(StringData n, const char* val) : NamedAttribute(n, StringData(val)) {} + NamedAttribute(StringData n, float val) : NamedAttribute(n, static_cast(val)) {} + NamedAttribute(StringData n, std::string const& val) : NamedAttribute(n, StringData(val)) {} + NamedAttribute(StringData n, long double val) = delete; + + template && !std::is_floating_point_v>> + NamedAttribute(StringData n, const T& val) : name(n) { + static_assert(HasToString::value, "custom type needs toString() implementation"); + + CustomAttributeValue custom; + if constexpr (HasToBSON::value) { + custom.toBSON = std::bind(&T::toBSON, val); + } + if constexpr (HasToString::value) { + custom.toString = std::bind(&T::toString, val); + } + + value = std::move(custom); + } + + StringData name; + stdx::variant + value; +}; + +// Attribute Storage stores an array of Named Attributes. +template +class AttributeStorage { +public: + AttributeStorage(const Args&... args) + : _data{detail::NamedAttribute(StringData(args.name.data(), args.name.size()), + args.value)...} {} + +private: + static const size_t kNumArgs = sizeof...(Args); + + // Arrays need a size of at least 1, add dummy element if needed (kNumArgs above is still 0) + NamedAttribute _data[kNumArgs ? kNumArgs : 1]; + + // This class is meant to be wrapped by TypeErasedAttributeStorage below that provides public + // accessors. Let it access all our internals. + friend class mongo::logv2::TypeErasedAttributeStorage; +}; + +template +AttributeStorage makeAttributeStorage(const Args&... args) { + return {args...}; +} + +} // namespace detail + +// Wrapper around internal pointer of AttributeStorage so it does not need any template parameters +class TypeErasedAttributeStorage { +public: + TypeErasedAttributeStorage() : _size(0) {} + + template + TypeErasedAttributeStorage(const detail::AttributeStorage& store) { + _data = store._data; + _size = store.kNumArgs; + } + + bool empty() const { + return _size == 0; + } + + std::size_t size() const { + return _size; + } + + // Applies a function to every stored named attribute in order they are captured + template + void apply(Func&& f) const { + std::for_each(_data, _data + _size, [&f](const detail::NamedAttribute& attr) { + StringData name = attr.name; + stdx::visit([name, &f](auto&& val) { f(name, val); }, attr.value); + }); + } + +private: + const detail::NamedAttribute* _data; + size_t _size; +}; + +} // namespace logv2 +} // namespace mongo diff --git a/src/mongo/logv2/constants.h b/src/mongo/logv2/constants.h new file mode 100644 index 00000000000..a235bd8081b --- /dev/null +++ b/src/mongo/logv2/constants.h @@ -0,0 +1,38 @@ +/** + * Copyright (C) 2019-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +namespace mongo::logv2::constants { + +// Used in data structures to indicate number of attributes to store without having to allocate +// memory. +constexpr size_t kNumStaticAttrs = 16; + +} // namespace mongo::logv2::constants diff --git a/src/mongo/logv2/json_formatter.h b/src/mongo/logv2/json_formatter.h index d045a93c2ce..1d266fd871a 100644 --- a/src/mongo/logv2/json_formatter.h +++ b/src/mongo/logv2/json_formatter.h @@ -29,13 +29,16 @@ #pragma once +#include #include #include #include #include -#include "mongo/logv2/attribute_argument_set.h" +#include "mongo/bson/bsonobj.h" +#include "mongo/logv2/attribute_storage.h" #include "mongo/logv2/attributes.h" +#include "mongo/logv2/constants.h" #include "mongo/logv2/log_component.h" #include "mongo/logv2/log_severity.h" #include "mongo/logv2/log_tag.h" @@ -46,44 +49,65 @@ namespace mongo { namespace logv2 { +struct JsonValueExtractor { + void operator()(StringData name, CustomAttributeValue const& val) { + if (val.toBSON) { + // This is a JSON subobject, select overload that does not surround value with quotes + operator()(name, val.toBSON().jsonString()); + } else { + // This is a string, select overload that surrounds value with quotes + operator()(name, StringData(val.toString())); + } + } + + void operator()(StringData name, const BSONObj* val) { + // This is a JSON subobject, select overload that does not surround value with quotes + operator()(name, val->jsonString()); + } + + void operator()(StringData name, StringData value) { + // The first {} is for the member separator, added by store() + store(R"({}"{}":"{}")", name, value); + } + + template + void operator()(StringData name, const T& value) { + // The first {} is for the member separator, added by store() + store(R"({}"{}":{})", name, value); + } + + template + void store(const char* fmt_str, StringData name, const T& value) { + nameArgs.push_back(fmt::internal::make_arg(name)); + fmt::format_to(buffer, fmt_str, _separator, name, value); + _separator = ","_sd; + } + + fmt::memory_buffer buffer; + boost::container::small_vector, + constants::kNumStaticAttrs> + nameArgs; + +private: + StringData _separator = ""_sd; +}; + class JsonFormatter { public: static bool binary() { return false; }; - void operator()(boost::log::record_view const& rec, boost::log::formatting_ostream& strm) { + void operator()(boost::log::record_view const& rec, + boost::log::formatting_ostream& strm) const { using namespace boost::log; // Build a JSON object for the user attributes. - const auto& attrs = extract(attributes::attributes(), rec).get(); - std::stringstream ss; - bool first = true; - ss << "{"; - for (std::size_t i = 0; i < attrs._names.size(); ++i) { - if (!first) - ss << ","; - first = false; - bool is_string = attrs._values.get(i).type() == fmt::internal::type::string_type || - attrs._values.get(i).type() == fmt::internal::type::cstring_type; - ss << "\"" << attrs._names[i] << "\":"; - if (is_string) - ss << "\""; - - // Call libfmt to extract formatted value from type erased format_args. If we have a - // custom formatter, ask for formatting in JSON using :j format specifier. - fmt::memory_buffer buffer; - auto format_str = fmt::format( - attrs._values.get(i).type() == fmt::internal::type::custom_type ? "{{{}:j}}" - : "{{{}}}", - i); - fmt::vformat_to(buffer, format_str, attrs._values); - ss << fmt::to_string(buffer); - - if (is_string) - ss << "\""; - } - ss << "}"; + const auto& attrs = + extract(attributes::attributes(), rec).get(); + + JsonValueExtractor extractor; + attrs.apply(extractor); std::string id; auto stable_id = extract(attributes::stableId(), rec).get(); @@ -93,17 +117,13 @@ public: std::string message; fmt::memory_buffer buffer; - std::vector> name_args; - for (auto&& attr_name : attrs._names) { - name_args.emplace_back(fmt::internal::make_arg(attr_name)); - } fmt::vformat_to( buffer, extract(attributes::message(), rec).get().toString(), - fmt::basic_format_args(name_args.data(), name_args.size())); + fmt::basic_format_args(extractor.nameArgs.data(), + extractor.nameArgs.size())); message = fmt::to_string(buffer); - StringData severity = extract(attributes::severity(), rec).get().toStringDataCompact(); StringData component = @@ -114,23 +134,35 @@ public: tag = fmt::format(",\"tags\":{}", tags.toJSONArray()); } - auto formatted_body = fmt::format( - "{{\"t\":\"{}\",\"s\":\"{}\"{: <{}}\"c\":\"{}\"{: " - "<{}}\"ctx\":\"{}\",{}\"msg\":\"{}\"{}{}{}}}", - dateToISOStringUTC(extract(attributes::timeStamp(), rec).get()), - severity, - ",", - 3 - severity.size(), - component, - ",", - 9 - component.size(), - extract(attributes::threadName(), rec).get(), - id, - message, - attrs._names.empty() ? "" : ",\"attr\":", - attrs._names.empty() ? "" : ss.str(), - tag); - strm << formatted_body; + strm << fmt::format(R"({{)" + R"("t":"{}",)" // timestamp + R"("s":"{}"{: <{}})" // severity with padding for the comma + R"("c":"{}"{: <{}})" // component with padding for the comma + R"("ctx":"{}",)" // context + R"({})" // optional stable id + R"("msg":"{}")" // message + R"({})", // optional attribute key + dateToISOStringUTC(extract(attributes::timeStamp(), rec).get()), + severity, + ",", + 3 - severity.size(), + component, + ",", + 9 - component.size(), + extract(attributes::threadName(), rec).get(), + id, + message, + attrs.empty() ? "" : ",\"attr\":{"); + + if (!attrs.empty()) { + strm << fmt::to_string(extractor.buffer); + } + + strm << fmt::format(R"({})" // optional attribute closing + R"({})" // optional tags + R"(}})", + attrs.empty() ? "" : "}", + tag); } private: diff --git a/src/mongo/logv2/log.h b/src/mongo/logv2/log.h index bb3178b3d79..9a313535ce2 100644 --- a/src/mongo/logv2/log.h +++ b/src/mongo/logv2/log.h @@ -46,7 +46,6 @@ #include "mongo/base/status.h" #include "mongo/bson/util/builder.h" -#include "mongo/logv2/attribute_argument_set.h" #include "mongo/logv2/log_component.h" #include "mongo/logv2/log_domain.h" #include "mongo/logv2/log_severity.h" diff --git a/src/mongo/logv2/log_detail.cpp b/src/mongo/logv2/log_detail.cpp index 15a0fb7caf2..579ba741e67 100644 --- a/src/mongo/logv2/log_detail.cpp +++ b/src/mongo/logv2/log_detail.cpp @@ -46,7 +46,7 @@ void doLogImpl(LogSeverity const& severity, StringData stable_id, LogOptions const& options, StringData message, - AttributeArgumentSet const& attrs) { + TypeErasedAttributeStorage const& attrs) { auto& source = options.domain().internal().source(); auto record = source.open_record(severity, options.component(), options.tags(), stable_id); if (record) { @@ -58,7 +58,8 @@ void doLogImpl(LogSeverity const& severity, record.attribute_values().insert( attributes::attributes(), boost::log::attribute_value( - new boost::log::attributes::attribute_value_impl(attrs))); + new boost::log::attributes::attribute_value_impl( + attrs))); source.push_record(std::move(record)); } diff --git a/src/mongo/logv2/log_detail.h b/src/mongo/logv2/log_detail.h index 8a9cbdd40d1..e5ffbc48fd8 100644 --- a/src/mongo/logv2/log_detail.h +++ b/src/mongo/logv2/log_detail.h @@ -29,7 +29,7 @@ #include "mongo/base/status.h" #include "mongo/bson/util/builder.h" -#include "mongo/logv2/attribute_argument_set.h" +#include "mongo/logv2/attribute_storage.h" #include "mongo/logv2/log_component.h" #include "mongo/logv2/log_domain.h" #include "mongo/logv2/log_options.h" @@ -39,11 +39,12 @@ namespace mongo { namespace logv2 { namespace detail { + void doLogImpl(LogSeverity const& severity, StringData stable_id, LogOptions const& options, StringData message, - AttributeArgumentSet const& attrs); + TypeErasedAttributeStorage const& attrs); template @@ -51,13 +52,11 @@ void doLog(LogSeverity const& severity, StringData stable_id, LogOptions const& options, S const& message, - fmt::internal::named_arg&&... args) { - AttributeArgumentSet attr_set; - auto arg_store = fmt::internal::make_args_checked(message, (args.value)...); - attr_set._values = arg_store; - (attr_set._names.push_back(::mongo::StringData(args.name.data(), args.name.size())), ...); + const fmt::internal::named_arg&... args) { + auto attributes = makeAttributeStorage(args...); + auto msg = static_cast(message); - doLogImpl(severity, stable_id, options, ::mongo::StringData(msg.data(), msg.size()), attr_set); + doLogImpl(severity, stable_id, options, StringData(msg.data(), msg.size()), attributes); } } // namespace detail diff --git a/src/mongo/logv2/log_test_v2.cpp b/src/mongo/logv2/log_test_v2.cpp index 98ce2063949..bdbe1225c83 100644 --- a/src/mongo/logv2/log_test_v2.cpp +++ b/src/mongo/logv2/log_test_v2.cpp @@ -34,26 +34,35 @@ #include "mongo/logv2/log_test_v2.h" #include +#include #include #include +#include "mongo/bson/bsonobj.h" +#include "mongo/bson/bsonobjbuilder.h" #include "mongo/bson/json.h" #include "mongo/logv2/component_settings_filter.h" #include "mongo/logv2/formatter_base.h" #include "mongo/logv2/json_formatter.h" #include "mongo/logv2/log.h" #include "mongo/logv2/log_component.h" +#include "mongo/logv2/plain_formatter.h" #include "mongo/logv2/ramlog_sink.h" #include "mongo/logv2/text_formatter.h" #include "mongo/stdx/thread.h" #include "mongo/unittest/temp_dir.h" #include +#include +#include +namespace mongo { +namespace logv2 { namespace { -struct TypeWithCustomFormatting { - TypeWithCustomFormatting() {} - TypeWithCustomFormatting(double x, double y) : _x(x), _y(y) {} + +struct TypeWithoutBSON { + TypeWithoutBSON() {} + TypeWithoutBSON(double x, double y) : _x(x), _y(y) {} double _x{0.0}; double _y{0.0}; @@ -61,37 +70,19 @@ struct TypeWithCustomFormatting { std::string toString() const { return fmt::format("(x: {}, y: {})", _x, _y); } - - std::string toJson() const { - return fmt::format("{{\"x\": {}, \"y\": {}}}", _x, _y); - } }; -} // namespace - - -namespace fmt { -template <> -struct formatter : public mongo::logv2::FormatterBase { - template - auto format(const TypeWithCustomFormatting& obj, FormatContext& ctx) { - switch (output_format()) { - case OutputFormat::kJson: - return format_to(ctx.out(), "{}", obj.toJson()); - case OutputFormat::kBson: - return format_to(ctx.out(), "{}", "bson impl here"); +struct TypeWithBSON : public TypeWithoutBSON { + using TypeWithoutBSON::TypeWithoutBSON; - case OutputFormat::kText: - default: - return format_to(ctx.out(), "{}", obj.toString()); - } + BSONObj toBSON() const { + BSONObjBuilder builder; + builder.append("x"_sd, _x); + builder.append("y"_sd, _y); + return builder.obj(); } }; -} // namespace fmt -namespace mongo { -namespace logv2 { -namespace { class LogTestBackend : public boost::log::sinks:: basic_formatted_sink_backend { @@ -113,21 +104,6 @@ private: std::vector& _logLines; }; -class PlainFormatter { -public: - static bool binary() { - return false; - }; - - void operator()(boost::log::record_view const& rec, boost::log::formatting_ostream& strm) { - StringData message = boost::log::extract(attributes::message(), rec).get(); - const auto& attrs = - boost::log::extract(attributes::attributes(), rec).get(); - - strm << fmt::internal::vformat(to_string_view(message), attrs._values); - } -}; - class LogDuringInitTester { public: LogDuringInitTester() { @@ -179,12 +155,192 @@ TEST_F(LogTestV2, Basic) { LOGV2_OPTIONS({LogTag::kStartupWarnings}, "test"); ASSERT(lines.back() == "test"); - TypeWithCustomFormatting t(1.0, 2.0); + TypeWithBSON t(1.0, 2.0); LOGV2("{} custom formatting", "name"_attr = t); ASSERT(lines.back() == t.toString() + " custom formatting"); - LOGV2("{:j} custom formatting, force json", "name"_attr = t); - ASSERT(lines.back() == t.toJson() + " custom formatting, force json"); + TypeWithoutBSON t2(1.0, 2.0); + LOGV2("{} custom formatting, no bson", "name"_attr = t2); + ASSERT(lines.back() == t.toString() + " custom formatting, no bson"); +} + +TEST_F(LogTestV2, Types) { + std::vector text; + auto text_sink = LogTestBackend::create(text); + text_sink->set_filter(ComponentSettingsFilter(LogManager::global().getGlobalDomain(), + LogManager::global().getGlobalSettings())); + text_sink->set_formatter(PlainFormatter()); + attach(text_sink); + + std::vector json; + auto json_sink = LogTestBackend::create(json); + json_sink->set_filter(ComponentSettingsFilter(LogManager::global().getGlobalDomain(), + LogManager::global().getGlobalSettings())); + json_sink->set_formatter(JsonFormatter()); + attach(json_sink); + + // The JSON formatter should make the types round-trippable without data loss + auto validateJSON = [&](auto expected) { + namespace pt = boost::property_tree; + + std::istringstream json_stream(json.back()); + pt::ptree ptree; + pt::json_parser::read_json(json_stream, ptree); + ASSERT(ptree.get("attr.name") == expected); + }; + + // bool + bool b = true; + LOGV2("bool {}", "name"_attr = b); + ASSERT(text.back() == "bool true"); + validateJSON(b); + + // char gets promoted to int + char c = 1; + LOGV2("char {}", "name"_attr = c); + ASSERT(text.back() == "char 1"); + validateJSON(static_cast( + c)); // cast, boost property_tree will try and parse as ascii otherwise + + // signed char gets promoted to int + signed char sc = -1; + LOGV2("signed char {}", "name"_attr = sc); + ASSERT(text.back() == "signed char -1"); + validateJSON(sc); + + // unsigned char gets promoted to unsigned int + unsigned char uc = -1; + LOGV2("unsigned char {}", "name"_attr = uc); + ASSERT(text.back() == "unsigned char 255"); + validateJSON(uc); + + // short gets promoted to int + short s = 1; + LOGV2("short {}", "name"_attr = s); + ASSERT(text.back() == "short 1"); + validateJSON(s); + + // signed short gets promoted to int + signed short ss = -1; + LOGV2("signed short {}", "name"_attr = ss); + ASSERT(text.back() == "signed short -1"); + validateJSON(ss); + + // unsigned short gets promoted to unsigned int + unsigned short us = -1; + LOGV2("unsigned short {}", "name"_attr = us); + ASSERT(text.back() == "unsigned short 65535"); + validateJSON(us); + + // int types are preserved + int i = 1; + LOGV2("int {}", "name"_attr = i); + ASSERT(text.back() == "int 1"); + validateJSON(i); + + signed int si = -1; + LOGV2("signed int {}", "name"_attr = si); + ASSERT(text.back() == "signed int -1"); + validateJSON(si); + + unsigned int ui = -1; + LOGV2("unsigned int {}", "name"_attr = ui); + ASSERT(text.back() == "unsigned int 4294967295"); + validateJSON(ui); + + // int is treated as long long + long l = 1; + LOGV2("long {}", "name"_attr = l); + ASSERT(text.back() == "long 1"); + validateJSON(l); + + signed long sl = -1; + LOGV2("signed long {}", "name"_attr = sl); + ASSERT(text.back() == "signed long -1"); + validateJSON(sl); + + unsigned long ul = -1; + LOGV2("unsigned long {}", "name"_attr = ul); + ASSERT(text.back() == fmt::format("unsigned long {}", ul)); + validateJSON(ul); + + // long long types are preserved + long long ll = std::numeric_limits::max(); + ll += 1; + LOGV2("long long {}", "name"_attr = ll); + ASSERT(text.back() == std::string("long long ") + std::to_string(ll)); + validateJSON(ll); + + signed long long sll = -1; + LOGV2("signed long long {}", "name"_attr = sll); + ASSERT(text.back() == "signed long long -1"); + validateJSON(sll); + + unsigned long long ull = -1; + LOGV2("unsigned long long {}", "name"_attr = ull); + ASSERT(text.back() == "unsigned long long 18446744073709551615"); + validateJSON(ull); + + // int64_t, uint64_t, size_t are fine + int64_t int64 = 1; + LOGV2("int64_t {}", "name"_attr = int64); + ASSERT(text.back() == "int64_t 1"); + validateJSON(int64); + + uint64_t uint64 = -1; + LOGV2("uint64_t {}", "name"_attr = uint64); + ASSERT(text.back() == "uint64_t 18446744073709551615"); + validateJSON(uint64); + + size_t size = 1; + LOGV2("size_t {}", "name"_attr = size); + ASSERT(text.back() == "size_t 1"); + validateJSON(size); + + // floating point types + float f = 1.0; + LOGV2("float {}", "name"_attr = f); + ASSERT(text.back() == "float 1.0"); + validateJSON(f); + + double d = 1.0; + LOGV2("double {}", "name"_attr = d); + ASSERT(text.back() == "double 1.0"); + validateJSON(d); + + // long double is prohibited, we don't use this type and favors Decimal128 instead. + + // string types + const char* c_str = "a c string"; + LOGV2("c string {}", "name"_attr = c_str); + ASSERT(text.back() == "c string a c string"); + validateJSON(std::string(c_str)); + + std::string str = "a std::string"; + LOGV2("std::string {}", "name"_attr = str); + ASSERT(text.back() == "std::string a std::string"); + validateJSON(str); + + StringData str_data = "a StringData"_sd; + LOGV2("StringData {}", "name"_attr = str_data); + ASSERT(text.back() == "StringData a StringData"); + validateJSON(str_data.toString()); + + // BSONObj + BSONObjBuilder builder; + builder.append("int32"_sd, i); + builder.append("int64"_sd, ll); + builder.append("double"_sd, d); + builder.append("str"_sd, str_data); + BSONObj bson = builder.obj(); + LOGV2("bson {}", "name"_attr = bson); + ASSERT(text.back() == std::string("bson ") + bson.jsonString()); + ASSERT(mongo::fromjson(json.back()) + .getField("attr"_sd) + .Obj() + .getField("name") + .Obj() + .woCompare(bson) == 0); } TEST_F(LogTestV2, TextFormat) { @@ -205,9 +361,16 @@ TEST_F(LogTestV2, TextFormat) { "warning"); ASSERT(lines.back().rfind("** WARNING: warning") != std::string::npos); - TypeWithCustomFormatting t(1.0, 2.0); + TypeWithBSON t(1.0, 2.0); LOGV2("{} custom formatting", "name"_attr = t); ASSERT(lines.back().rfind(t.toString() + " custom formatting") != std::string::npos); + + LOGV2("{} bson", "name"_attr = t.toBSON()); + ASSERT(lines.back().rfind(t.toBSON().jsonString() + " bson") != std::string::npos); + + TypeWithoutBSON t2(1.0, 2.0); + LOGV2("{} custom formatting, no bson", "name"_attr = t2); + ASSERT(lines.back().rfind(t.toString() + " custom formatting, no bson") != std::string::npos); } TEST_F(LogTestV2, JSONFormat) { @@ -255,13 +418,27 @@ TEST_F(LogTestV2, JSONFormat) { ASSERT(log.getField("c"_sd).String() == LogComponent(LogComponent::kControl).getNameForLog()); ASSERT(log.getField("msg"_sd).String() == "different component"); - TypeWithCustomFormatting t(1.0, 2.0); + TypeWithBSON t(1.0, 2.0); LOGV2("{} custom formatting", "name"_attr = t); log = mongo::fromjson(lines.back()); ASSERT(log.getField("msg"_sd).String() == "{name} custom formatting"); ASSERT(log.getField("attr"_sd).Obj().nFields() == 1); ASSERT(log.getField("attr"_sd).Obj().getField("name").Obj().woCompare( - mongo::fromjson(t.toJson())) == 0); + mongo::fromjson(t.toBSON().jsonString())) == 0); + + LOGV2("{} bson", "name"_attr = t.toBSON()); + log = mongo::fromjson(lines.back()); + ASSERT(log.getField("msg"_sd).String() == "{name} bson"); + ASSERT(log.getField("attr"_sd).Obj().nFields() == 1); + ASSERT(log.getField("attr"_sd).Obj().getField("name").Obj().woCompare( + mongo::fromjson(t.toBSON().jsonString())) == 0); + + TypeWithoutBSON t2(1.0, 2.0); + LOGV2("{} custom formatting", "name"_attr = t2); + log = mongo::fromjson(lines.back()); + ASSERT(log.getField("msg"_sd).String() == "{name} custom formatting"); + ASSERT(log.getField("attr"_sd).Obj().nFields() == 1); + ASSERT(log.getField("attr"_sd).Obj().getField("name").String() == t.toString()); } TEST_F(LogTestV2, Threads) { diff --git a/src/mongo/logv2/plain_formatter.h b/src/mongo/logv2/plain_formatter.h new file mode 100644 index 00000000000..bf1fb71536c --- /dev/null +++ b/src/mongo/logv2/plain_formatter.h @@ -0,0 +1,105 @@ +/** + * Copyright (C) 2019-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "mongo/bson/bsonobj.h" +#include "mongo/logv2/attribute_storage.h" +#include "mongo/logv2/attributes.h" +#include "mongo/logv2/constants.h" +#include "mongo/logv2/log_component.h" +#include "mongo/logv2/log_severity.h" +#include "mongo/logv2/log_tag.h" +#include "mongo/util/time_support.h" + +#include +#include + +namespace mongo { +namespace logv2 { + +struct TextValueExtractor { + void operator()(StringData name, CustomAttributeValue const& val) { + _storage.push_back(val.toString()); + operator()(name, _storage.back()); + } + + void operator()(StringData name, const BSONObj* val) { + _storage.push_back(val->jsonString()); + operator()(name, _storage.back()); + } + + template + void operator()(StringData name, const T& val) { + args.push_back(fmt::internal::make_arg(val)); + } + + boost::container::small_vector, + constants::kNumStaticAttrs> + args; + +private: + std::deque _storage; +}; + +// Text formatter without metadata. Just contains the formatted message. +class PlainFormatter { +public: + static bool binary() { + return false; + }; + + void operator()(boost::log::record_view const& rec, + boost::log::formatting_ostream& strm) const { + using namespace boost::log; + + StringData message = extract(attributes::message(), rec).get(); + const auto& attrs = + extract(attributes::attributes(), rec).get(); + + TextValueExtractor extractor; + extractor.args.reserve(attrs.size()); + attrs.apply(extractor); + fmt::memory_buffer buffer; + fmt::internal::vformat_to(buffer, + to_string_view(message), + fmt::basic_format_args( + extractor.args.data(), extractor.args.size())); + strm.write(buffer.data(), buffer.size()); + } +}; + +} // namespace logv2 +} // namespace mongo diff --git a/src/mongo/logv2/text_formatter.h b/src/mongo/logv2/text_formatter.h index 4d9a65d7b37..017dbdc34b0 100644 --- a/src/mongo/logv2/text_formatter.h +++ b/src/mongo/logv2/text_formatter.h @@ -29,35 +29,21 @@ #pragma once -#include -#include -#include -#include - -#include "mongo/logv2/attribute_argument_set.h" -#include "mongo/logv2/attributes.h" -#include "mongo/logv2/log_component.h" -#include "mongo/logv2/log_severity.h" -#include "mongo/logv2/log_tag.h" -#include "mongo/util/time_support.h" - -#include +#include "mongo/logv2/plain_formatter.h" namespace mongo { namespace logv2 { -class TextFormatter { +class TextFormatter : protected PlainFormatter { public: static bool binary() { return false; }; - void operator()(boost::log::record_view const& rec, boost::log::formatting_ostream& strm) { + void operator()(boost::log::record_view const& rec, + boost::log::formatting_ostream& strm) const { using namespace boost::log; - StringData message = extract(attributes::message(), rec).get(); - const auto& attrs = extract(attributes::attributes(), rec).get(); - fmt::memory_buffer buffer; fmt::format_to( buffer, @@ -72,9 +58,7 @@ public: strm << "** WARNING: "; } - buffer.clear(); - fmt::internal::vformat_to(buffer, to_string_view(message), attrs._values); - strm.write(buffer.data(), buffer.size()); + PlainFormatter::operator()(rec, strm); } }; diff --git a/src/mongo/shell/dbshell.cpp b/src/mongo/shell/dbshell.cpp index b573a3c5546..eca95ae9f92 100644 --- a/src/mongo/shell/dbshell.cpp +++ b/src/mongo/shell/dbshell.cpp @@ -57,7 +57,6 @@ #include "mongo/logger/logger.h" #include "mongo/logger/logv2_appender.h" #include "mongo/logger/message_event_utf8_encoder.h" -#include "mongo/logv2/attribute_argument_set.h" #include "mongo/logv2/attributes.h" #include "mongo/logv2/component_settings_filter.h" #include "mongo/logv2/console.h" @@ -198,14 +197,9 @@ public: using boost::log::extract; if (extract(attributes::tags(), rec).get().has(LogTag::kJavascript)) { - StringData message = extract(attributes::message(), rec).get(); - const auto& attrs = extract(attributes::attributes(), rec).get(); - - fmt::memory_buffer buffer; - fmt::internal::vformat_to(buffer, to_string_view(message), attrs._values); - strm.write(buffer.data(), buffer.size()); + PlainFormatter::operator()(rec, strm); } else { - logv2::TextFormatter::operator()(rec, strm); + TextFormatter::operator()(rec, strm); } } }; -- cgit v1.2.1