SERVER-24606 Add support for collecting information from /proc/diskstats

4 files changed, 441 insertions, 0 deletions

diff --git a/src/mongo/db/ftdc/ftdc_system_stats.cpp b/src/mongo/db/ftdc/ftdc_system_stats.cpp
index 16d13a98174..f0ffb319284 100644
--- a/src/mongo/db/ftdc/ftdc_system_stats.cpp
+++ b/src/mongo/db/ftdc/ftdc_system_stats.cpp
@@ -58,6 +58,12 @@ static const std::vector<StringData> kMemKeys{};
  */
 class LinuxSystemMetricsCollector final : public FTDCCollectorInterface {
 public:
+    LinuxSystemMetricsCollector() : _disks(procparser::findPhysicalDisks("/sys/block")) {
+        for (const auto& disk : _disks) {
+            _disksStringData.emplace_back(disk);
+        }
+    }
+
     void collect(OperationContext* txn, BSONObjBuilder& builder) override {
         {
             BSONObjBuilder subObjBuilder(builder.subobjStart("cpu"));
@@ -74,6 +80,16 @@ public:
                 &subObjBuilder);
             subObjBuilder.doneFast();
         }
+
+        // Skip the disks section if we could not find any disks.
+        // This can happen when we do not have permission to /sys/block for instance.
+        if (!_disksStringData.empty()) {
+            BSONObjBuilder subObjBuilder(builder.subobjStart("disks"));
+            processStatusErrors(procparser::parseProcDiskStatsFile(
+                                    "/proc/diskstats", _disksStringData, &subObjBuilder),
+                                &subObjBuilder);
+            subObjBuilder.doneFast();
+        }
     }
 
     std::string name() const override {
@@ -92,6 +108,13 @@ private:
             builder->append("error", s.toString());
         }
     }
+
+private:
+    // List of physical disks to collect stats from as string from findPhysicalDisks.
+    std::vector<std::string> _disks;
+
+    // List of physical disks to collect stats from as StringData to pass to parseProcDiskStatsFile.
+    std::vector<StringData> _disksStringData;
 };
 
 void installSystemMetricsCollector(FTDCController* controller) {
diff --git a/src/mongo/util/procparser.cpp b/src/mongo/util/procparser.cpp
index 3929c40f6ee..36f2ae0254e 100644
--- a/src/mongo/util/procparser.cpp
+++ b/src/mongo/util/procparser.cpp
@@ -36,6 +36,7 @@
 #include <array>
 #include <boost/algorithm/string/finder.hpp>
 #include <boost/algorithm/string/split.hpp>
+#include <boost/filesystem.hpp>
 #include <fcntl.h>
 #include <string>
 #include <sys/stat.h>
@@ -74,6 +75,8 @@ double convertTicksToMilliSeconds(const int64_t ticks, const int64_t ticksPerSec
 const size_t kFileBufferSize = 16384;
 const size_t kFileReadRetryCount = 5;
 
+constexpr auto kSysBlockDeviceDirectoryName = "device";
+
 /**
  * Read a file from disk as a string with a null-terminating byte using the POSIX file api.
  *
@@ -149,6 +152,22 @@ const char* const kAdditionCpuFields[] = {"user_ms",
                                           "guest_nice_ms"};
 const size_t kAdditionCpuFieldCount = std::extent<decltype(kAdditionCpuFields)>::value;
 
+const char* const kDiskFields[] = {
+    "reads",
+    "reads_merged",
+    "read_sectors",
+    "read_time_ms",
+    "writes",
+    "writes_merged",
+    "write_sectors",
+    "write_time_ms",
+    "io_in_progress",
+    "io_time_ms",
+    "io_queued_ms",
+};
+
+const size_t kDiskFieldCount = std::extent<decltype(kDiskFields)>::value;
+
 }  // namespace
 
 namespace procparser {
@@ -386,5 +405,216 @@ Status parseProcMemInfoFile(StringData filename,
 
     return parseProcMemInfo(keys, swString.getValue(), builder);
 }
+
+// Here is an example of the type of string it supports:
+//
+// For more information, see:
+// Documentation/iostats.txt in the Linux kernel
+// proc(5) man page
+//
+// > cat /proc/diskstats
+//   8       0 sda 120611 33630 6297628 96550 349797 167398 11311562 2453603 0 117514 2554160
+//   8       1 sda1 138 37 8642 315 3 0 18 14 0 292 329
+//   8       2 sda2 120409 33593 6285754 96158 329029 167398 11311544 2450573 0 115611 2550739
+//   8      16 sdb 12707 3876 1525418 57507 997 3561 297576 97976 0 37870 155619
+//   8      17 sdb1 12601 3876 1521090 57424 992 3561 297576 97912 0 37738 155468
+//  11       0 sr0 0 0 0 0 0 0 0 0 0 0 0
+// 253       0 dm-0 154910 0 6279522 177681 506513 0 11311544 5674418 0 117752 5852275
+// 253       1 dm-1 109 0 4584 226 0 0 0 0 0 172 226
+//
+Status parseProcDiskStats(const std::vector<StringData>& disks,
+                          StringData data,
+                          BSONObjBuilder* builder) {
+    bool foundKeys = false;
+    std::vector<uint64_t> stats;
+    stats.reserve(kDiskFieldCount);
+
+    using string_split_iterator = boost::split_iterator<StringData::const_iterator>;
+
+    // Split the file by lines.
+    // token_compress_on means the iterator skips over consecutive '\n'. This should not be a
+    // problem in normal /proc/diskstats output.
+    for (string_split_iterator lineIt = string_split_iterator(
+             data.begin(),
+             data.end(),
+             boost::token_finder([](char c) { return c == '\n'; }, boost::token_compress_on));
+         lineIt != string_split_iterator();
+         ++lineIt) {
+
+        StringData line((*lineIt).begin(), (*lineIt).end());
+
+        // Skip leading whitespace so that the split_iterator starts on non-whitespace otherwise we
+        // get an empty first token. Device major numbers (the first number on each line) are right
+        // aligned to 4 spaces and start from
+        // single digits.
+        auto beginNonWhitespace =
+            std::find_if_not(line.begin(), line.end(), [](char c) { return c == ' '; });
+
+        // Split the line by spaces since that is the only delimiter for diskstats files.
+        // token_compress_on means the iterator skips over consecutive ' '.
+        string_split_iterator partIt = string_split_iterator(
+            beginNonWhitespace,
+            line.end(),
+            boost::token_finder([](char c) { return c == ' '; }, boost::token_compress_on));
+
+        // Skip processing this line if the line is blank
+        if (partIt == string_split_iterator()) {
+            continue;
+        }
+
+        ++partIt;
+
+        // Skip processing this line if we only have a device major number.
+        if (partIt == string_split_iterator()) {
+            continue;
+        }
+
+        ++partIt;
+
+        // Skip processing this line if we only have a device major minor.
+        if (partIt == string_split_iterator()) {
+            continue;
+        }
+
+        StringData disk((*partIt).begin(), (*partIt).end());
+
+        // Skip processing this line if we only have a block device name.
+        if (partIt == string_split_iterator()) {
+            continue;
+        }
+
+        ++partIt;
+
+        // Check if the disk is in the list. /proc/diskstats will have extra disks, and may not have
+        // the disk we want.
+        if (disks.empty() || std::find(disks.begin(), disks.end(), disk) != disks.end()) {
+            foundKeys = true;
+
+            stats.clear();
+
+            // Only generate a disk document if the disk has some activity. For instance, there
+            // could be a CD-ROM drive that is not used.
+            bool hasSomeNonZeroStats = false;
+
+            for (size_t index = 0; partIt != string_split_iterator() && index < kDiskFieldCount;
+                 ++partIt, ++index) {
+
+                StringData stringValue((*partIt).begin(), (*partIt).end());
+
+                uint64_t value;
+
+                if (!parseNumberFromString(stringValue, &value).isOK()) {
+                    value = 0;
+                }
+
+                if (value != 0) {
+                    hasSomeNonZeroStats = true;
+                }
+
+                stats.push_back(value);
+            }
+
+            if (hasSomeNonZeroStats) {
+                // Start a new document with disk as the name.
+                BSONObjBuilder sub(builder->subobjStart(disk));
+
+                for (size_t index = 0; index < stats.size() && index < kDiskFieldCount; ++index) {
+                    sub.appendNumber(kDiskFields[index], stats[index]);
+                }
+
+                sub.doneFast();
+            }
+        }
+    }
+
+    return foundKeys ? Status::OK()
+                     : Status(ErrorCodes::NoSuchKey, "Failed to find any keys in diskstats string");
+}
+
+Status parseProcDiskStatsFile(StringData filename,
+                              const std::vector<StringData>& disks,
+                              BSONObjBuilder* builder) {
+    auto swString = readFileAsString(filename);
+    if (!swString.isOK()) {
+        return swString.getStatus();
+    }
+
+    return parseProcDiskStats(disks, swString.getValue(), builder);
+}
+
+namespace {
+
+/**
+ * Is this a disk that is interesting to us? We only want physical disks, not multiple disk devices,
+ * LVM2 devices, partitions, or RAM disks.
+ *
+ * A physical disk has a symlink to a directory at /sys/block/<device name>/device.
+ *
+ * Note: returns false upon any errors such as access denied.
+ */
+bool isInterestingDisk(const boost::filesystem::path& path) {
+    boost::filesystem::path blockDevicePath(path);
+    blockDevicePath /= kSysBlockDeviceDirectoryName;
+
+    boost::system::error_code ec;
+    auto statusSysBlock = boost::filesystem::status(blockDevicePath, ec);
+    if (!boost::filesystem::exists(statusSysBlock)) {
+        return false;
+    }
+
+    if (ec) {
+        warning() << "Error checking directory '" << blockDevicePath.generic_string()
+                  << "': " << ec.message();
+        return false;
+    }
+
+    if (!boost::filesystem::is_directory(statusSysBlock)) {
+        return false;
+    }
+
+    return true;
+}
+
+}  // namespace
+
+std::vector<std::string> findPhysicalDisks(StringData sysBlockPath) {
+    boost::system::error_code ec;
+    auto sysBlockPathStr = sysBlockPath.toString();
+
+    auto statusSysBlock = boost::filesystem::status(sysBlockPathStr, ec);
+    if (ec) {
+        warning() << "Error checking directory '" << sysBlockPathStr << "': " << ec.message();
+        return {};
+    }
+
+    if (!(boost::filesystem::exists(statusSysBlock) &&
+          boost::filesystem::is_directory(statusSysBlock))) {
+        warning() << "Could not find directory '" << sysBlockPathStr << "': " << ec.message();
+        return {};
+    }
+
+    std::vector<std::string> files;
+
+    // Iterate through directories in /sys/block. The directories in this directory can be physical
+    // block devices (like SSD or HDD) or virtual devices like the LVM2 device mapper or a multiple
+    // disk device. It does not contain disk partitions.
+    boost::filesystem::directory_iterator di(sysBlockPathStr, ec);
+    if (ec) {
+        warning() << "Error getting directory iterator '" << sysBlockPathStr
+                  << "': " << ec.message();
+        return {};
+    }
+
+    for (; di != boost::filesystem::directory_iterator(); di++) {
+        auto path = (*di).path();
+
+        if (isInterestingDisk(path)) {
+            files.push_back(path.filename().generic_string());
+        }
+    }
+
+    return files;
+}
+
 }  // namespace procparser
 }  // namespace mongo
diff --git a/src/mongo/util/procparser.h b/src/mongo/util/procparser.h
index ed18b8bd197..2f9e97e343c 100644
--- a/src/mongo/util/procparser.h
+++ b/src/mongo/util/procparser.h
@@ -87,5 +87,34 @@ Status parseProcMemInfoFile(StringData filename,
                             const std::vector<StringData>& keys,
                             BSONObjBuilder* builder);
 
+/**
+ * Read a string matching /proc/diskstats format, and write the specified list of disks in builder.
+ *
+ * disks - vector of block devices to include in output. For each disk selected, 11 fields are
+ *         output in a nested document. There is no error if the disk is not found in the data. Also
+ *         a disk is excluded if it has no activity since startup (i.e. an idle CD-ROM drive). If
+ *         disks is empty, all non-zero block devices are outputed (this will include partitions,
+ *         etc).
+ * data - string to parsee
+ * builder - BSON output
+ */
+Status parseProcDiskStats(const std::vector<StringData>& disks,
+                          StringData data,
+                          BSONObjBuilder* builder);
+
+/**
+ * Read from file, and write the specified list of disks in builder.
+ */
+Status parseProcDiskStatsFile(StringData filename,
+                              const std::vector<StringData>& disks,
+                              BSONObjBuilder* builder);
+
+/**
+ * Get a vector of disks to monitor by enumerating the specified directory.
+ *
+ * If the directory does not exist, or otherwise permission is denied, returns an empty vector.
+ */
+std::vector<std::string> findPhysicalDisks(StringData directory);
+
 }  // namespace procparser
 }  // namespace mongo
diff --git a/src/mongo/util/procparser_test.cpp b/src/mongo/util/procparser_test.cpp
index d5bc1c2358e..81bf17ac8df 100644
--- a/src/mongo/util/procparser_test.cpp
+++ b/src/mongo/util/procparser_test.cpp
@@ -55,6 +55,22 @@ StringMap toStringMap(BSONObj& obj) {
     return map;
 }
 
+StringMap toNestedStringMap(BSONObj& obj) {
+    StringMap map;
+
+    for (const auto& e : obj) {
+        if (e.isABSONObj()) {
+            std::string prefix = std::string(e.fieldName()) + ".";
+
+            for (const auto& child : e.Obj()) {
+                map[prefix + child.fieldName()] = child.numberLong();
+            }
+        }
+    }
+
+    return map;
+}
+
 #define ASSERT_KEY(_key) ASSERT_TRUE(stringMap.find(_key) != stringMap.end());
 #define ASSERT_NO_KEY(_key) ASSERT_TRUE(stringMap.find(_key) == stringMap.end());
 #define ASSERT_KEY_AND_VALUE(_key, _value) ASSERT_EQUALS(stringMap.at(_key), _value);
@@ -69,6 +85,11 @@ StringMap toStringMap(BSONObj& obj) {
     ASSERT_OK(procparser::parseProcMemInfo(_keys, _x, &builder)); \
     auto obj = builder.obj();                                     \
     auto stringMap = toStringMap(obj);
+#define ASSERT_PARSE_DISKSTATS(_disks, _x)                           \
+    BSONObjBuilder builder;                                          \
+    ASSERT_OK(procparser::parseProcDiskStats(_disks, _x, &builder)); \
+    auto obj = builder.obj();                                        \
+    auto stringMap = toNestedStringMap(obj);
 
 TEST(FTDCProcStat, TestStat) {
 
@@ -347,5 +368,143 @@ TEST(FTDCProcMemInfo, TestLocalNonExistentMemInfo) {
     ASSERT_NOT_OK(procparser::parseProcMemInfoFile("/proc/does_not_exist", keys, &builder));
 }
 
+
+TEST(FTDCProcDiskStats, TestDiskStats) {
+
+    std::vector<StringData> disks{"dm-1", "sda", "sdb"};
+
+    // Normal case including high device major numbers.
+    {
+        ASSERT_PARSE_DISKSTATS(
+            disks,
+            "   8       0 sda 120611 33630 6297628 96550 349797 167398 11311562 2453603 0 117514 "
+            "2554160\n"
+            "   8       1 sda1 138 37 8642 315 3 0 18 14 0 292 329\n"
+            "   8       2 sda2 120409 33593 6285754 96158 329029 167398 11311544 2450573 0 115611 "
+            "2550739\n"
+            "   8      16 sdb 12707 3876 1525418 57507 997 3561 297576 97976 0 37870 155619\n"
+            "   8      17 sdb1 12601 3876 1521090 57424 992 3561 297576 97912 0 37738 155468\n"
+            "  11       0 sr0 0 0 0 0 0 0 0 0 0 0 0\n"
+            "2253       0 dm-0 154910 0 6279522 177681 506513 0 11311544 5674418 0 117752 5852275\n"
+            "2253       1 dm-1 109 0 4584 226 0 0 0 0 0 172 226");
+        ASSERT_KEY_AND_VALUE("sda.reads", 120611UL);
+        ASSERT_KEY_AND_VALUE("sda.writes", 349797UL);
+        ASSERT_KEY_AND_VALUE("sda.io_queued_ms", 2554160UL);
+        ASSERT_KEY_AND_VALUE("sdb.reads", 12707UL);
+        ASSERT_KEY_AND_VALUE("sdb.writes", 997UL);
+        ASSERT_KEY_AND_VALUE("sdb.io_queued_ms", 155619UL);
+        ASSERT_KEY_AND_VALUE("dm-1.reads", 109UL);
+        ASSERT_KEY_AND_VALUE("dm-1.writes", 0UL);
+        ASSERT_KEY_AND_VALUE("dm-1.io_queued_ms", 226UL);
+    }
+
+    // Exclude a block device without any activity
+    {
+        ASSERT_PARSE_DISKSTATS(
+            disks,
+            "   8       0 sda 120611 33630 6297628 96550 349797 167398 11311562 2453603 0 117514 "
+            "2554160\n"
+            "   8       1 sda1 138 37 8642 315 3 0 18 14 0 292 329\n"
+            "   8       2 sda2 120409 33593 6285754 96158 329029 167398 11311544 2450573 0 115611 "
+            "2550739\n"
+            "   8      16 sdb 0 0 0 0 0 0 0 0 0 0 0\n"
+            "   8      17 sdb1 12601 3876 1521090 57424 992 3561 297576 97912 0 37738 155468\n"
+            "  11       0 sr0 0 0 0 0 0 0 0 0 0 0 0\n"
+            "2253       0 dm-0 154910 0 6279522 177681 506513 0 11311544 5674418 0 117752 5852275\n"
+            "2253       1 dm-1 109 0 4584 226 0 0 0 0 0 172 226");
+        ASSERT_KEY_AND_VALUE("sda.reads", 120611UL);
+        ASSERT_KEY_AND_VALUE("sda.writes", 349797UL);
+        ASSERT_KEY_AND_VALUE("sda.io_queued_ms", 2554160UL);
+        ASSERT_NO_KEY("sdb.reads");
+        ASSERT_NO_KEY("sdb.writes");
+        ASSERT_NO_KEY("sdb.io_queued_ms");
+        ASSERT_KEY_AND_VALUE("dm-1.reads", 109UL);
+        ASSERT_KEY_AND_VALUE("dm-1.writes", 0UL);
+        ASSERT_KEY_AND_VALUE("dm-1.io_queued_ms", 226UL);
+    }
+
+
+    // Strings with less numbers
+    { ASSERT_PARSE_DISKSTATS(disks, "8       0 sda 120611 33630 6297628 96550 349797 "); }
+
+    // Strings with no numbers
+    { ASSERT_PARSE_DISKSTATS(disks, "8       0 sda"); }
+
+    // Strings that are too short
+    {
+        BSONObjBuilder builder;
+        ASSERT_NOT_OK(procparser::parseProcDiskStats(disks, "8       0", &builder));
+        ASSERT_NOT_OK(procparser::parseProcDiskStats(disks, "8", &builder));
+        ASSERT_NOT_OK(procparser::parseProcDiskStats(disks, "", &builder));
+    }
+}
+
+TEST(FTDCProcDiskStats, TestLocalNonExistentStat) {
+    std::vector<StringData> disks{"dm-1", "sda", "sdb"};
+    BSONObjBuilder builder;
+
+    ASSERT_NOT_OK(procparser::parseProcDiskStatsFile("/proc/does_not_exist", disks, &builder));
+}
+
+TEST(FTDCProcDiskStats, TestFindBadPhysicalDiskPaths) {
+    // Validate nothing goes wrong when we check a non-existent path.
+    {
+        auto disks = procparser::findPhysicalDisks("/proc/does_not_exist");
+        ASSERT_EQUALS(0UL, disks.size());
+    }
+
+    // Validate nothing goes wrong when we check a path we do not have permission.
+    {
+        auto disks = procparser::findPhysicalDisks("/sys/kernel/debug");
+        ASSERT_EQUALS(0UL, disks.size());
+    }
+}
+
+// Test we can parse the /proc/diskstats on this machine. Also assert we have the expected fields
+// This tests is designed to exercise our parsing code on various Linuxes and fail
+// Normally when run in the FTDC loop we return a non-fatal error so we may not notice the failure
+// otherwise.
+TEST(FTDCProcDiskStats, TestLocalDiskStats) {
+    auto disks = procparser::findPhysicalDisks("/sys/block");
+
+    std::vector<StringData> disks2;
+    for (const auto& disk : disks) {
+        log() << "DISK:" << disk;
+        disks2.emplace_back(disk);
+    }
+
+    ASSERT_NOT_EQUALS(0UL, disks.size());
+
+    BSONObjBuilder builder;
+
+    ASSERT_OK(procparser::parseProcDiskStatsFile("/proc/diskstats", disks2, &builder));
+
+    BSONObj obj = builder.obj();
+    auto stringMap = toNestedStringMap(obj);
+    log() << "OBJ:" << obj;
+
+    bool foundDisk = false;
+
+    for (const auto& disk : disks) {
+        std::string prefix(disk);
+        prefix += ".";
+
+        auto reads = prefix + "reads";
+        auto io_queued_ms = prefix + "io_queued_ms";
+
+        // Make sure that if have the first field, then we have the last field.
+        if (stringMap.find(reads) != stringMap.end()) {
+            foundDisk = true;
+            if (stringMap.find(io_queued_ms) == stringMap.end()) {
+                FAIL(std::string("Inconsistency for ") + disk);
+            }
+        }
+    }
+
+    if (!foundDisk) {
+        FAIL("Did not find any interesting disks on this machine.");
+    }
+}
+
 }  // namespace
 }  // namespace mongo