SERVER-46018 RamLog should use hard code length

3 files changed, 215 insertions, 106 deletions

diff --git a/src/mongo/logv2/log_test_v2.cpp b/src/mongo/logv2/log_test_v2.cpp
index 7bfb68ba601..0da56af42ea 100644
--- a/src/mongo/logv2/log_test_v2.cpp
+++ b/src/mongo/logv2/log_test_v2.cpp
@@ -1314,6 +1314,103 @@ TEST_F(LogTestV2, Ramlog) {
     ASSERT(verifyRamLog());
 }
 
+// Positive: Test that the ram log is properly circular
+TEST_F(LogTestV2, Ramlog_CircularBuffer) {
+    RamLog* ramlog = RamLog::get("test_ramlog2");
+
+    std::vector<std::string> lines;
+
+    constexpr size_t maxLines = 1024;
+    constexpr size_t testLines = 5000;
+
+    // Write enough lines to trigger wrapping
+    for (size_t i = 0; i < testLines; ++i) {
+        auto s = std::to_string(i);
+        lines.push_back(s);
+        ramlog->write(s);
+    }
+
+    lines.erase(lines.begin(), lines.begin() + (testLines - maxLines) + 1);
+
+    // Verify we circled correctly through the buffer
+    {
+        RamLog::LineIterator iter(ramlog);
+        ASSERT_EQ(iter.getTotalLinesWritten(), 5000UL);
+        for (const auto& line : lines) {
+            ASSERT_EQ(line, iter.next());
+        }
+    }
+
+    ramlog->clear();
+}
+
+// Positive: Test that the ram log has a max size cap
+TEST_F(LogTestV2, Ramlog_MaxSize) {
+    RamLog* ramlog = RamLog::get("test_ramlog3");
+
+    std::vector<std::string> lines;
+
+    constexpr size_t testLines = 2000;
+    constexpr size_t longStringLength = 2048;
+
+    std::string longStr(longStringLength, 'a');
+
+    // Write enough lines to trigger wrapping and trimming
+    for (size_t i = 0; i < testLines; ++i) {
+        auto s = std::to_string(10000 + i) + longStr;
+        lines.push_back(s);
+        ramlog->write(s);
+    }
+
+    constexpr size_t linesToFit = (1024 * 1024) / (5 + longStringLength);
+
+    lines.erase(lines.begin(), lines.begin() + (testLines - linesToFit));
+
+    // Verify we keep just enough lines that fit
+    {
+        RamLog::LineIterator iter(ramlog);
+        ASSERT_EQ(iter.getTotalLinesWritten(), 2000UL);
+        for (const auto& line : lines) {
+            ASSERT_EQ(line, iter.next());
+        }
+    }
+
+    ramlog->clear();
+}
+
+// Positive: Test that the ram log handles really large lines
+TEST_F(LogTestV2, Ramlog_GiantLine) {
+    RamLog* ramlog = RamLog::get("test_ramlog4");
+
+    std::vector<std::string> lines;
+
+    constexpr size_t testLines = 5000;
+
+    // Write enough lines to trigger wrapping
+    for (size_t i = 0; i < testLines; ++i) {
+        ramlog->write(std::to_string(i));
+    }
+
+    auto s = std::to_string(testLines);
+    lines.push_back(s);
+    ramlog->write(s);
+
+    std::string bigStr(2048 * 1024, 'a');
+    lines.push_back(bigStr);
+    ramlog->write(bigStr);
+
+    // Verify we keep 2 lines
+    {
+        RamLog::LineIterator iter(ramlog);
+        ASSERT_EQ(iter.getTotalLinesWritten(), testLines + 2);
+        for (const auto& line : lines) {
+            ASSERT_EQ(line, iter.next());
+        }
+    }
+
+    ramlog->clear();
+}
+
 TEST_F(LogTestV2, MultipleDomains) {
     std::vector<std::string> global_lines;
     auto sink = LogCaptureBackend::create(global_lines);
diff --git a/src/mongo/logv2/ramlog.cpp b/src/mongo/logv2/ramlog.cpp
index 38076ef1481..14c9a6d47f0 100644
--- a/src/mongo/logv2/ramlog.cpp
+++ b/src/mongo/logv2/ramlog.cpp
@@ -48,107 +48,102 @@ RM* _named = NULL;
 
 }  // namespace
 
-RamLog::RamLog(const std::string& name) : _name(name), _totalLinesWritten(0), _lastWrite(0) {
+RamLog::RamLog(StringData name) : _name(name) {
     clear();
-    for (int i = 0; i < N; i++)
-        lines[i][C - 1] = 0;
 }
 
 RamLog::~RamLog() {}
 
 void RamLog::write(const std::string& str) {
     stdx::lock_guard<stdx::mutex> lk(_mutex);
-    _lastWrite = time(0);
     _totalLinesWritten++;
 
-    char* p = lines[(h + n) % N];
+    if (0 == str.size()) {
+        return;
+    }
+
+    // Trim if we are going to go above the threshold
+    trimIfNeeded(str.size(), lk);
+
+    // Add the new line and adjust the space accounting
+    _totalSizeBytes -= _lines[_lastLinePosition].size();
+    _lines[_lastLinePosition] = str;
+    _totalSizeBytes += str.size();
+
+    // Advance the last line position to the next entry
+    _lastLinePosition = (_lastLinePosition + 1) % kMaxLines;
 
-    unsigned sz = str.size() + 1;
-    if (1 == sz)
+    // If _lastLinePosition is == _firstLinePosition, it means we wrapped around so advance
+    // firstLinePosition
+    if (_lastLinePosition == _firstLinePosition) {
+        _firstLinePosition = (_firstLinePosition + 1) % kMaxLines;
+    }
+}
+
+void RamLog::trimIfNeeded(size_t newStr, WithLock lock) {
+    // Check if we are going to go past the size limit
+    if ((_totalSizeBytes + newStr) < kMaxSizeBytes) {
         return;
-    if (sz < C) {
-        memcpy(p, str.c_str(), sz);
-    } else {
-        memcpy(p, str.c_str(), C - 1);
-        *(p + C - 1) = '\0';
     }
 
-    if (n < N)
-        n++;
-    else
-        h = (h + 1) % N;
+    // Worst case, if the user adds a really large line, we will keep just one line
+    if (getLineCount(lock) == 0) {
+        return;
+    }
+
+    // The buffer has grown large, so trim back enough to fit our new string
+    size_t trimmedSpace = 0;
+
+    // Trim down until we make enough space, keep at least one line though
+    // This means with the line we are about to have, the log will actually have 2 lines
+    while (getLineCount(lock) > 1 && trimmedSpace < newStr) {
+        size_t size = _lines[_firstLinePosition].size();
+        trimmedSpace += size;
+        _totalSizeBytes -= size;
+
+        _lines[_firstLinePosition].clear();
+        _lines[_firstLinePosition].shrink_to_fit();
+
+        _firstLinePosition = (_firstLinePosition + 1) % kMaxLines;
+    }
 }
 
 void RamLog::clear() {
     stdx::lock_guard<stdx::mutex> lk(_mutex);
     _totalLinesWritten = 0;
-    _lastWrite = 0;
-    h = 0;
-    n = 0;
-    for (int i = 0; i < N; i++)
-        lines[i][0] = 0;
-}
-
-time_t RamLog::LineIterator::lastWrite() {
-    return _ramlog->_lastWrite;
-}
+    _firstLinePosition = 0;
+    _lastLinePosition = 0;
+    _totalSizeBytes = 0;
 
-long long RamLog::LineIterator::getTotalLinesWritten() {
-    return _ramlog->_totalLinesWritten;
+    for (size_t i = 0; i < kMaxLines; i++) {
+        _lines[i].clear();
+        _lines[i].shrink_to_fit();
+    }
 }
 
-const char* RamLog::getLine_inlock(unsigned lineNumber) const {
-    if (lineNumber >= n)
+StringData RamLog::getLine(size_t lineNumber, WithLock lock) const {
+    if (lineNumber >= getLineCount(lock)) {
         return "";
-    return lines[(lineNumber + h) % N];  // h = 0 unless n == N, hence modulo N.
-}
-
-int RamLog::repeats(const std::vector<const char*>& v, int i) {
-    for (int j = i - 1; j >= 0 && j + 8 > i; j--) {
-        if (strcmp(v[i] + 24, v[j] + 24) == 0) {
-            for (int x = 1;; x++) {
-                if (j + x == i)
-                    return j;
-                if (i + x >= (int)v.size())
-                    return -1;
-                if (strcmp(v[i + x] + 24, v[j + x] + 24))
-                    return -1;
-            }
-            return -1;
-        }
     }
-    return -1;
-}
 
-
-string RamLog::clean(const std::vector<const char*>& v, int i, string line) {
-    if (line.empty())
-        line = v[i];
-    if (i > 0 && strncmp(v[i], v[i - 1], 11) == 0)
-        return string("           ") + line.substr(11);
-    return v[i];
+    return _lines[(lineNumber + _firstLinePosition) % kMaxLines].c_str();
 }
 
-/* turn http:... into an anchor */
-string RamLog::linkify(const char* s) {
-    const char* p = s;
-    const char* h = strstr(p, "http://");
-    if (h == 0)
-        return s;
-
-    const char* sp = h + 7;
-    while (*sp && *sp != ' ')
-        sp++;
-
-    string url(h, sp - h);
-    std::stringstream ss;
-    ss << string(s, h - s) << "<a href=\"" << url << "\">" << url << "</a>" << sp;
-    return ss.str();
+size_t RamLog::getLineCount(WithLock) const {
+    if (_lastLinePosition < _firstLinePosition) {
+        return (kMaxLines - _firstLinePosition) + _lastLinePosition;
+    }
+
+    return _lastLinePosition - _firstLinePosition;
 }
 
 RamLog::LineIterator::LineIterator(RamLog* ramlog)
     : _ramlog(ramlog), _lock(ramlog->_mutex), _nextLineIndex(0) {}
 
+size_t RamLog::LineIterator::getTotalLinesWritten() {
+    return _ramlog->_totalLinesWritten;
+}
+
 // ---------------
 // static things
 // ---------------
@@ -170,24 +165,28 @@ RamLog* RamLog::get(const std::string& name) {
         result = new RamLog(name);
         (*_named)[name] = result;
     }
+
     return result;
 }
 
 RamLog* RamLog::getIfExists(const std::string& name) {
-    if (!_named)
+    if (!_named) {
         return NULL;
+    }
     stdx::lock_guard<stdx::mutex> lk(*_namedLock);
     return mapFindWithDefault(*_named, name, static_cast<RamLog*>(NULL));
 }
 
 void RamLog::getNames(std::vector<string>& names) {
-    if (!_named)
+    if (!_named) {
         return;
+    }
 
     stdx::lock_guard<stdx::mutex> lk(*_namedLock);
     for (RM::iterator i = _named->begin(); i != _named->end(); ++i) {
-        if (i->second->n)
+        if (i->second->getLineCount(lk)) {
             names.push_back(i->first);
+        }
     }
 }
 
@@ -201,6 +200,7 @@ MONGO_INITIALIZER(RamLogCatalogV2)(InitializerContext*) {
             return Status(ErrorCodes::InternalError,
                           "Inconsistent intiailization of RamLogCatalog.");
         }
+
         _namedLock = new stdx::mutex();  // NOLINT
         _named = new RM();
     }
diff --git a/src/mongo/logv2/ramlog.h b/src/mongo/logv2/ramlog.h
index cefc1031f15..1dd29d4eae0 100644
--- a/src/mongo/logv2/ramlog.h
+++ b/src/mongo/logv2/ramlog.h
@@ -29,29 +29,33 @@
 
 #pragma once
 
-#include <boost/version.hpp>
-#include <sstream>
 #include <string>
 #include <vector>
 
-#include "mongo/base/status.h"
 #include "mongo/base/string_data.h"
 #include "mongo/platform/mutex.h"
 #include "mongo/util/concurrency/mutex.h"
+#include "mongo/util/concurrency/with_lock.h"
 
 namespace mongo {
 namespace logv2 {
 
 /**
- * Fixed-capacity log of line-oriented messages.
+ * Variable-capacity circular log of line-oriented messages.
  *
- * Holds up to RamLog::N lines of up to RamLog::C bytes, each.
+ * Holds up to RamLog::kMaxLines lines and caps total space to RamLog::kMaxSizeBytes [1]. There is
+ * no limit on the length of the line. RamLog expects the caller to truncate lines to a reasonable
+ * length.
  *
  * RamLogs are stored in a global registry, accessed via RamLog::get() and
  * RamLog::getIfExists().
  *
  * RamLogs and their registry are self-synchronizing.  See documentary comments.
  * To read a RamLog, instantiate a RamLog::LineIterator, documented below.
+ *
+ * Note:
+ * 1. In the degenerate case of a single log line being above RamLog::kMaxSizeBytes, it may
+ *    keep up to two log lines and exceed the size cap.
  */
 class RamLog {
     RamLog(const RamLog&) = delete;
@@ -84,8 +88,7 @@ public:
 
     /**
      * Writes "str" as a line into the RamLog.  If "str" is longer than the maximum
-     * line size, RamLog::C, truncates the line to the first C bytes.  If "str"
-     * is shorter than RamLog::C and has a terminal '\n', it omits that character.
+     * line size of the log, it keeps two lines.
      *
      * Synchronized on the instance's own mutex, _mutex.
      */
@@ -97,31 +100,42 @@ public:
     void clear();
 
 private:
-    static int repeats(const std::vector<const char*>& v, int i);
-    static std::string clean(const std::vector<const char*>& v, int i, std::string line = "");
+    explicit RamLog(StringData name);
+    ~RamLog();  // want this private as we want to leak so we can use them till the very end
 
-    /* turn http:... into an anchor */
-    static std::string linkify(const char* s);
+    StringData getLine(size_t lineNumber, WithLock lock) const;
 
-    explicit RamLog(const std::string& name);
-    ~RamLog();  // want this private as we want to leak so we can use them till the very end
+    size_t getLineCount(WithLock) const;
+
+    void trimIfNeeded(size_t newStr, WithLock lock);
 
-    enum {
-        N = 1024,  // number of lines
-        C = 3072   // max size of line
-    };
+private:
+    // Maximum number of lines
+    static constexpr size_t kMaxLines = 1024;
 
-    const char* getLine_inlock(unsigned lineNumber) const;
+    // Maximum capacity of RamLog of string data
+    static constexpr size_t kMaxSizeBytes = 1024 * 1024;
 
     // Guards all non-static data.
     stdx::mutex _mutex;  // NOLINT
-    char lines[N][C];
-    unsigned h;  // current position
-    unsigned n;  // number of lines stores 0 o N
+
+    // Array of lines
+    std::array<std::string, kMaxLines> _lines;
+
+    // First line of ram log
+    size_t _firstLinePosition;
+
+    // Last line of ram log
+    size_t _lastLinePosition;
+
+    // Total size of bytes written
+    size_t _totalSizeBytes;
+
+    // Name of Ram Log
     std::string _name;
-    long long _totalLinesWritten;
 
-    time_t _lastWrite;
+    // Total lines written since last clear, can be > kMaxLines
+    size_t _totalLinesWritten;
 };
 
 /**
@@ -143,30 +157,28 @@ public:
      * Returns true if there are more lines available to return by calls to next().
      */
     bool more() const {
-        return _nextLineIndex < _ramlog->n;
+        return _nextLineIndex < _ramlog->getLineCount(_lock);
     }
 
     /**
      * Returns the next line and advances the iterator.
      */
-    const char* next() {
-        return _ramlog->getLine_inlock(_nextLineIndex++);  // Postfix increment.
+    StringData next() {
+        return _ramlog->getLine(_nextLineIndex++, _lock);  // Postfix increment.
     }
 
     /**
-     * Returns the time of the last write to the ramlog.
-     */
-    time_t lastWrite();
-
-    /**
      * Returns the total number of lines ever written to the ramlog.
      */
-    long long getTotalLinesWritten();
+    size_t getTotalLinesWritten();
 
 private:
     const RamLog* _ramlog;
+
+    // Holds RamLog's mutex
     stdx::lock_guard<stdx::mutex> _lock;
-    unsigned _nextLineIndex;
+
+    size_t _nextLineIndex;
 };
 
 }  // namespace logv2