SERVER-24605 Add support for collecting information from /proc/meminfo

(cherry picked from commit b602fbbe46fa4e09cc488428acfd963771f7ebff)

4 files changed, 287 insertions, 0 deletions

diff --git a/src/mongo/db/ftdc/ftdc_system_stats.cpp b/src/mongo/db/ftdc/ftdc_system_stats.cpp
index 35764e7768f..16d13a98174 100644
--- a/src/mongo/db/ftdc/ftdc_system_stats.cpp
+++ b/src/mongo/db/ftdc/ftdc_system_stats.cpp
@@ -50,6 +50,9 @@ constexpr auto kSystemMetricsCollector = "systemMetrics";
 static const std::vector<StringData> kCpuKeys{
     "btime", "cpu", "ctxt", "processes", "procs_blocked", "procs_running"};
 
+// Collect all the memory keys by specifying an empty set.
+static const std::vector<StringData> kMemKeys{};
+
 /**
  *  Collect metrics from the Linux /proc file system.
  */
@@ -63,6 +66,14 @@ public:
                 &subObjBuilder);
             subObjBuilder.doneFast();
         }
+
+        {
+            BSONObjBuilder subObjBuilder(builder.subobjStart("memory"));
+            processStatusErrors(
+                procparser::parseProcMemInfoFile("/proc/meminfo", kMemKeys, &subObjBuilder),
+                &subObjBuilder);
+            subObjBuilder.doneFast();
+        }
     }
 
     std::string name() const override {
diff --git a/src/mongo/util/procparser.cpp b/src/mongo/util/procparser.cpp
index 1a1de263eb3..6eb3faa70a9 100644
--- a/src/mongo/util/procparser.cpp
+++ b/src/mongo/util/procparser.cpp
@@ -275,5 +275,110 @@ Status parseProcStatFile(StringData filename,
     return parseProcStat(keys, swString.getValue(), getTicksPerSecond(), builder);
 }
 
+// Here is an example of the type of string it supports:
+// Note: output has been trimmed
+//
+// For more information, see:
+// Documentation/filesystems/proc.txt in the Linux kernel
+// proc(5) man page
+//
+// > cat /proc/meminfo
+// MemTotal:       12294392 kB
+// MemFree:         3652612 kB
+// MemAvailable:   11831704 kB
+// Buffers:          568536 kB
+// Cached:          6421520 kB
+// SwapCached:            0 kB
+// HugePages_Total:       0
+//
+// Note: HugePages_* do not end in kB, it is not a typo
+//
+Status parseProcMemInfo(const std::vector<StringData>& keys,
+                        StringData data,
+                        BSONObjBuilder* builder) {
+    bool foundKeys = false;
+
+    using string_split_iterator = boost::split_iterator<StringData::const_iterator>;
+
+    // Split the file by lines.
+    // token_compress_on means the iterator skips over consecutive '\n'. This should not be a
+    // problem in normal /proc/memInfo output.
+    for (string_split_iterator lineIt = string_split_iterator(
+             data.begin(),
+             data.end(),
+             boost::token_finder([](char c) { return c == '\n'; }, boost::token_compress_on));
+         lineIt != string_split_iterator();
+         ++lineIt) {
+        StringData line((*lineIt).begin(), (*lineIt).end());
+
+        // Split the line by spaces and colons since these are the delimiters for meminfo files.
+        // token_compress_on means the iterator skips over consecutive ' '. This is needed for
+        // every line.
+        string_split_iterator partIt = string_split_iterator(line.begin(),
+                                                             line.end(),
+                                                             boost::token_finder([](char c) {
+                                                                 return c == ' ' || c == ':';
+                                                             }, boost::token_compress_on));
+
+        // Skip processing this line if we do not have a key.
+        if (partIt == string_split_iterator()) {
+            continue;
+        }
+
+        StringData key((*partIt).begin(), (*partIt).end());
+
+        ++partIt;
+
+        // Skip processing this line if we only have a key, and no number.
+        if (partIt == string_split_iterator()) {
+            continue;
+        }
+
+        // Check if the key is in the list. /proc/meminfo will have extra keys, and may not have the
+        // keys we want.
+        if (keys.empty() || std::find(keys.begin(), keys.end(), key) != keys.end()) {
+            foundKeys = true;
+
+            StringData stringValue((*partIt).begin(), (*partIt).end());
+
+            uint64_t value;
+
+            if (!parseNumberFromString(stringValue, &value).isOK()) {
+                value = 0;
+            }
+
+            // Check if the line ends in "kB"
+            ++partIt;
+
+            // If there is one last token, check if it is actually "kB"
+            if (partIt != string_split_iterator()) {
+                StringData kb_token((*partIt).begin(), (*partIt).end());
+                auto keyWithSuffix = key.toString();
+
+                if (kb_token == "kB") {
+                    keyWithSuffix.append("_kb");
+                }
+
+                builder->appendNumber(keyWithSuffix, value);
+            } else {
+                builder->appendNumber(key, value);
+            }
+        }
+    }
+
+    return foundKeys ? Status::OK()
+                     : Status(ErrorCodes::NoSuchKey, "Failed to find any keys in meminfo string");
+}
+
+Status parseProcMemInfoFile(StringData filename,
+                            const std::vector<StringData>& keys,
+                            BSONObjBuilder* builder) {
+    auto swString = readFileAsString(filename);
+    if (!swString.isOK()) {
+        return swString.getStatus();
+    }
+
+    return parseProcMemInfo(keys, swString.getValue(), builder);
+}
 }  // namespace procparser
 }  // namespace mongo
diff --git a/src/mongo/util/procparser.h b/src/mongo/util/procparser.h
index 3bbb0ece16e..ed18b8bd197 100644
--- a/src/mongo/util/procparser.h
+++ b/src/mongo/util/procparser.h
@@ -69,5 +69,23 @@ Status parseProcStatFile(StringData filename,
                          const std::vector<StringData>& keys,
                          BSONObjBuilder* builder);
 
+/**
+ * Read a string matching /proc/meminfo format, and write the specified list of keys in builder.
+ *
+ * keys - list of keys to output in BSON. If keys is empty, all keys are outputed.
+ * data - string to parsee
+ * builder - BSON output
+ */
+Status parseProcMemInfo(const std::vector<StringData>& keys,
+                        StringData data,
+                        BSONObjBuilder* builder);
+
+/**
+ * Read from file, and write the specified list of keys in builder.
+ */
+Status parseProcMemInfoFile(StringData filename,
+                            const std::vector<StringData>& keys,
+                            BSONObjBuilder* builder);
+
 }  // namespace procparser
 }  // namespace mongo
diff --git a/src/mongo/util/procparser_test.cpp b/src/mongo/util/procparser_test.cpp
index c19734a7998..659eca1b911 100644
--- a/src/mongo/util/procparser_test.cpp
+++ b/src/mongo/util/procparser_test.cpp
@@ -64,6 +64,11 @@ StringMap toStringMap(BSONObj& obj) {
     ASSERT_OK(procparser::parseProcStat(_keys, _x, 1000, &builder)); \
     auto obj = builder.obj();                                        \
     auto stringMap = toStringMap(obj);
+#define ASSERT_PARSE_MEMINFO(_keys, _x)                           \
+    BSONObjBuilder builder;                                       \
+    ASSERT_OK(procparser::parseProcMemInfo(_keys, _x, &builder)); \
+    auto obj = builder.obj();                                     \
+    auto stringMap = toStringMap(obj);
 
 TEST(FTDCProcStat, TestStat) {
     std::vector<StringData> keys{"cpu", "ctxt", "processes"};
@@ -211,5 +216,153 @@ TEST(FTDCProcStat, TestLocalNonExistentStat) {
     ASSERT_NOT_OK(procparser::parseProcStatFile("/proc/does_not_exist", keys, &builder));
 }
 
+TEST(FTDCProcMemInfo, TestMemInfo) {
+    std::vector<StringData> keys{"Key1", "Key2", "Key3"};
+
+    // Normal case
+    {
+        ASSERT_PARSE_MEMINFO(keys, "Key1: 123 kB\nKey2: 456 kB");
+        ASSERT_KEY_AND_VALUE("Key1_kb", 123UL);
+        ASSERT_KEY_AND_VALUE("Key2_kb", 456UL);
+    }
+
+    // Space in key name
+    {
+        ASSERT_PARSE_MEMINFO(keys, "Key1: 123 kB\nKey 2: 456 kB");
+        ASSERT_KEY_AND_VALUE("Key1_kb", 123UL);
+        ASSERT_NO_KEY("Key2_kb");
+    }
+
+    // No newline
+    {
+        ASSERT_PARSE_MEMINFO(keys, "Key1: 123 kB Key2: 456 kB");
+        ASSERT_KEY_AND_VALUE("Key1_kb", 123UL);
+        ASSERT_NO_KEY("Key2_kb");
+    }
+
+    // Missing colon on first key
+    {
+        ASSERT_PARSE_MEMINFO(keys, "Key1 123 kB\nKey2: 456 kB");
+        ASSERT_KEY_AND_VALUE("Key1_kb", 123UL);
+        ASSERT_KEY_AND_VALUE("Key2_kb", 456UL);
+    }
+
+    // One token missing kB, HugePages is not size in kB
+    {
+        ASSERT_PARSE_MEMINFO(keys, "Key1: 123 kB\nKey2: 456\nKey3: 789 kB\nKey4: 789 kB");
+        ASSERT_KEY_AND_VALUE("Key1_kb", 123UL);
+        ASSERT_KEY_AND_VALUE("Key2", 456UL);
+        ASSERT_KEY_AND_VALUE("Key3_kb", 789UL);
+        ASSERT_NO_KEY("Key4_kb");
+    }
+
+    // Empty string
+    {
+        BSONObjBuilder builder;
+        ASSERT_NOT_OK(procparser::parseProcMemInfo(keys, "", &builder));
+    }
+}
+
+// Test we can parse the /proc/meminfo on this machine. Also assert we have the expected fields
+// This tests is designed to exercise our parsing code on various Linuxes and fail
+// Normally when run in the FTDC loop we return a non-fatal error so we may not notice the failure
+// otherwise.
+TEST(FTDCProcMemInfo, TestLocalMemInfo) {
+    std::vector<StringData> keys{
+        "Active",
+        "Active(anon)",
+        "Active(file)",
+        "AnonHugePages",
+        "AnonPages",
+        "Bounce",
+        "Buffers",
+        "Cached",
+        "CmaFree",
+        "CmaTotal",
+        "CommitLimit",
+        "Committed_AS",
+        "Dirty",
+        "HardwareCorrupted",
+        "Inactive",
+        "Inactive(anon)",
+        "Inactive(file)",
+        "KernelStack",
+        "Mapped",
+        "MemAvailable",
+        "MemFree",
+        "MemTotal",
+        "Mlocked",
+        "NFS_Unstable",
+        "PageTables",
+        "SReclaimable",
+        "SUnreclaim",
+        "Shmem",
+        "Slab",
+        "SwapCached",
+        "SwapFree",
+        "SwapTotal",
+        "Unevictable",
+        "VmallocChunk",
+        "VmallocTotal",
+        "VmallocUsed",
+        "Writeback",
+        "WritebackTmp",
+    };
+
+    BSONObjBuilder builder;
+
+    ASSERT_OK(procparser::parseProcMemInfoFile("/proc/meminfo", keys, &builder));
+
+    BSONObj obj = builder.obj();
+    auto stringMap = toStringMap(obj);
+    log() << "OBJ:" << obj;
+    ASSERT_KEY("MemTotal_kb");
+    ASSERT_KEY("MemFree_kb");
+    // Needs in 3.15+ - ASSERT_KEY("MemAvailable_kb");
+    ASSERT_KEY("Buffers_kb");
+    ASSERT_KEY("Cached_kb");
+    ASSERT_KEY("SwapCached_kb");
+    ASSERT_KEY("Active_kb");
+    ASSERT_KEY("Inactive_kb");
+    // Needs 2.6.28+ - ASSERT_KEY("Active(anon)_kb");
+    // Needs 2.6.28+ - ASSERT_KEY("Inactive(anon)_kb");
+    // Needs 2.6.28+ - ASSERT_KEY("Active(file)_kb");
+    // Needs 2.6.28+ - ASSERT_KEY("Inactive(file)_kb");
+    // Needs 2.6.28+ - ASSERT_KEY("Unevictable_kb");
+    // Needs 2.6.28+ - ASSERT_KEY("Mlocked_kb");
+    ASSERT_KEY("SwapTotal_kb");
+    ASSERT_KEY("SwapFree_kb");
+    ASSERT_KEY("Dirty_kb");
+    ASSERT_KEY("Writeback_kb");
+    ASSERT_KEY("AnonPages_kb");
+    ASSERT_KEY("Mapped_kb");
+    // Needs 2.6.32+ - ASSERT_KEY("Shmem_kb");
+    ASSERT_KEY("Slab_kb");
+    // Needs 2.6.19+ - ASSERT_KEY("SReclaimable_kb");
+    // Needs 2.6.19+ - ASSERT_KEY("SUnreclaim_kb");
+    // Needs 2.6.32+ - ASSERT_KEY("KernelStack_kb");
+    ASSERT_KEY("PageTables_kb");
+    ASSERT_KEY("NFS_Unstable_kb");
+    ASSERT_KEY("Bounce_kb");
+    // Needs 2.6.19+ - ASSERT_KEY("WritebackTmp_kb");
+    ASSERT_KEY("CommitLimit_kb");
+    ASSERT_KEY("Committed_AS_kb");
+    ASSERT_KEY("VmallocTotal_kb");
+    ASSERT_KEY("VmallocUsed_kb");
+    ASSERT_KEY("VmallocChunk_kb");
+    // Needs CONFIG_MEMORY_FAILURE & 2.6.32+ ASSERT_KEY("HardwareCorrupted_kb");
+    // Needs CONFIG_TRANSPARENT_HUGEPAGE - ASSERT_KEY("AnonHugePages_kb");
+    // Needs CONFIG_CMA & 3.19+ - ASSERT_KEY("CmaTotal_kb");
+    // Needs CONFIG_CMA & 3.19+ - ASSERT_KEY("CmaFree_kb");
+}
+
+
+TEST(FTDCProcMemInfo, TestLocalNonExistentMemInfo) {
+    std::vector<StringData> keys{};
+    BSONObjBuilder builder;
+
+    ASSERT_NOT_OK(procparser::parseProcMemInfoFile("/proc/does_not_exist", keys, &builder));
+}
+
 }  // namespace
 }  // namespace mongo