Merge pull request #526 from ceph/wip-5909

mon: Early warning system for monitor stores growing over predefined threshold

Reviewed-by: Sage Weil <sage@inktank.com>

9 files changed, 207 insertions, 12 deletions

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index 6453e5a207c..f526f80c929 100644
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -203,6 +203,7 @@ OPTION(mon_leveldb_max_open_files, OPT_INT, 0) // monitor's leveldb max open fil
 OPTION(mon_leveldb_compression, OPT_BOOL, false) // monitor's leveldb uses compression
 OPTION(mon_leveldb_paranoid, OPT_BOOL, false)   // monitor's leveldb paranoid flag
 OPTION(mon_leveldb_log, OPT_STR, "")
+OPTION(mon_leveldb_size_warn, OPT_U64, 40*1024*1024*1024) // issue a warning when the monitor's leveldb goes over 40GB (in bytes)
 OPTION(paxos_stash_full_interval, OPT_INT, 25)   // how often (in commits) to stash a full copy of the PaxosService state
 OPTION(paxos_max_join_drift, OPT_INT, 10) // max paxos iterations before we must first sync the monitor stores
 OPTION(paxos_propose_interval, OPT_DOUBLE, 1.0)  // gather updates for this long before proposing a map update
diff --git a/src/mon/DataHealthService.cc b/src/mon/DataHealthService.cc
index 6e8aa313a36..a05948d3ce7 100644
--- a/src/mon/DataHealthService.cc
+++ b/src/mon/DataHealthService.cc
@@ -81,6 +81,18 @@ health_status_t DataHealthService::get_health(
       health_detail = "low disk space!";
     }
 
+    if (stats.store_stats.bytes_total >= g_conf->mon_leveldb_size_warn) {
+      if (health_status > HEALTH_WARN)
+        health_status = HEALTH_WARN;
+      if (!health_detail.empty())
+        health_detail.append("; ");
+      stringstream ss;
+      ss << "store is getting too big! "
+         << prettybyte_t(stats.store_stats.bytes_total)
+         << " >= " << prettybyte_t(g_conf->mon_leveldb_size_warn);
+      health_detail.append(ss.str());
+    }
+
     if (overall_status > health_status)
       overall_status = health_status;
 
@@ -95,18 +107,16 @@ health_status_t DataHealthService::get_health(
     if (f) {
       f->open_object_section("mon");
       f->dump_string("name", mon_name.c_str());
-      f->dump_int("kb_total", stats.kb_total);
-      f->dump_int("kb_used", stats.kb_used);
-      f->dump_int("kb_avail", stats.kb_avail);
-      f->dump_int("avail_percent", stats.latest_avail_percent);
-      f->dump_stream("last_updated") << stats.last_update;
+      f->open_object_section("data_stats");
+      stats.dump(f);
+      f->close_section();
       f->dump_stream("health") << health_status;
       if (health_status != HEALTH_OK)
-	f->dump_string("health_detail", health_detail);
+        f->dump_string("health_detail", health_detail);
       f->close_section();
     }
   }
-  
+
   if (f) {
     f->close_section(); // mons
     f->close_section(); // data_health
@@ -115,6 +125,22 @@ health_status_t DataHealthService::get_health(
   return overall_status;
 }
 
+int DataHealthService::update_store_stats(DataStats &ours)
+{
+  map<string,uint64_t> extra;
+  uint64_t store_size = mon->store->get_estimated_size(extra);
+  assert(store_size > 0);
+
+  ours.store_stats.bytes_total = store_size;
+  ours.store_stats.bytes_sst = extra["sst"];
+  ours.store_stats.bytes_log = extra["log"];
+  ours.store_stats.bytes_misc = extra["misc"];
+  ours.last_update = ceph_clock_now(g_ceph_context);
+
+  return 0;
+}
+
+
 int DataHealthService::update_stats()
 {
   struct statfs stbuf;
@@ -135,7 +161,8 @@ int DataHealthService::update_stats()
           << " total " << ours.kb_total << " used " << ours.kb_used << " avail " << ours.kb_avail
           << dendl;
   ours.last_update = ceph_clock_now(g_ceph_context);
-  return 0;
+
+  return update_store_stats(ours);
 }
 
 void DataHealthService::share_stats()
diff --git a/src/mon/DataHealthService.h b/src/mon/DataHealthService.h
index 337e7a450f7..750c58e5f80 100644
--- a/src/mon/DataHealthService.h
+++ b/src/mon/DataHealthService.h
@@ -34,6 +34,7 @@ class DataHealthService :
   int last_warned_percent;
 
   void handle_tell(MMonHealth *m);
+  int update_store_stats(DataStats &ours);
   int update_stats();
   void share_stats();
 
diff --git a/src/mon/MonitorDBStore.h b/src/mon/MonitorDBStore.h
index 276620f7516..85f6c895145 100644
--- a/src/mon/MonitorDBStore.h
+++ b/src/mon/MonitorDBStore.h
@@ -509,6 +509,10 @@ class MonitorDBStore
     db->compact_prefix(prefix);
   }
 
+  uint64_t get_estimated_size(map<string, uint64_t> &extras) {
+    return db->get_estimated_size(extras);
+  }
+
   MonitorDBStore(const string& path) :
     db(0), do_dump(false), dump_fd(-1) {
     string::const_reverse_iterator rit;
@@ -523,8 +527,8 @@ class MonitorDBStore
 
     LevelDBStore *db_ptr = new LevelDBStore(g_ceph_context, full_path);
     if (!db_ptr) {
-      std::cout << __func__ << " error initializing level db back storage in "
-		<< full_path << std::endl;
+      derr << __func__ << " error initializing level db back storage in "
+		<< full_path << dendl;
       assert(0 != "MonitorDBStore: error initializing level db back storage");
     }
     db.reset(db_ptr);
diff --git a/src/mon/mon_types.h b/src/mon/mon_types.h
index 0eae3b172bf..0ae1aaf8d5e 100644
--- a/src/mon/mon_types.h
+++ b/src/mon/mon_types.h
@@ -40,6 +40,52 @@ inline const char *get_paxos_name(int p) {
 
 #define CEPH_MON_ONDISK_MAGIC "ceph mon volume v012"
 
+/**
+ * leveldb store stats
+ *
+ * If we ever decide to support multiple backends for the monitor store,
+ * we should then create an abstract class 'MonitorStoreStats' of sorts
+ * and inherit it on LevelDBStoreStats.  I'm sure you'll figure something
+ * out.
+ */
+struct LevelDBStoreStats {
+  uint64_t bytes_total;
+  uint64_t bytes_sst;
+  uint64_t bytes_log;
+  uint64_t bytes_misc;
+  utime_t last_update;
+
+  void dump(Formatter *f) const {
+    assert(f != NULL);
+    f->dump_int("bytes_total", bytes_total);
+    f->dump_int("bytes_sst", bytes_sst);
+    f->dump_int("bytes_log", bytes_log);
+    f->dump_int("bytes_misc", bytes_misc);
+    f->dump_stream("last_updated") << last_update;
+  }
+
+  void encode(bufferlist &bl) const {
+    ENCODE_START(1, 1, bl);
+    ::encode(bytes_total, bl);
+    ::encode(bytes_sst, bl);
+    ::encode(bytes_log, bl);
+    ::encode(bytes_misc, bl);
+    ::encode(last_update, bl);
+    ENCODE_FINISH(bl);
+  }
+
+  void decode(bufferlist::iterator &p) {
+    DECODE_START(1, p);
+    ::decode(bytes_total, p);
+    ::decode(bytes_sst, p);
+    ::decode(bytes_log, p);
+    ::decode(bytes_misc, p);
+    ::decode(last_update, p);
+    DECODE_FINISH(p);
+  }
+};
+WRITE_CLASS_ENCODER(LevelDBStoreStats);
+
 // data stats
 
 struct DataStats {
@@ -50,13 +96,29 @@ struct DataStats {
   int latest_avail_percent;
   utime_t last_update;
 
+  LevelDBStoreStats store_stats;
+
+  void dump(Formatter *f) const {
+    assert(f != NULL);
+    f->dump_int("kb_total", kb_total);
+    f->dump_int("kb_used", kb_used);
+    f->dump_int("kb_avail", kb_avail);
+    f->dump_int("avail_percent", latest_avail_percent);
+    f->dump_stream("last_updated") << last_update;
+
+    f->open_object_section("store_stats");
+    store_stats.dump(f);
+    f->close_section();
+  }
+
   void encode(bufferlist &bl) const {
-    ENCODE_START(1, 1, bl);
+    ENCODE_START(2, 1, bl);
     ::encode(kb_total, bl);
     ::encode(kb_used, bl);
     ::encode(kb_avail, bl);
     ::encode(latest_avail_percent, bl);
     ::encode(last_update, bl);
+    ::encode(store_stats, bl);
     ENCODE_FINISH(bl);
   }
   void decode(bufferlist::iterator &p) {
@@ -66,10 +128,12 @@ struct DataStats {
     ::decode(kb_avail, p);
     ::decode(latest_avail_percent, p);
     ::decode(last_update, p);
+    if (struct_v > 1)
+      ::decode(store_stats, p);
+
     DECODE_FINISH(p);
   }
 };
-
 WRITE_CLASS_ENCODER(DataStats);
 
 struct ScrubResult {
diff --git a/src/os/KeyValueDB.h b/src/os/KeyValueDB.h
index f62bca996a5..e98463aa763 100644
--- a/src/os/KeyValueDB.h
+++ b/src/os/KeyValueDB.h
@@ -165,6 +165,8 @@ public:
     );
   }
 
+  virtual uint64_t get_estimated_size(map<string,uint64_t> &extra) = 0;
+
   virtual ~KeyValueDB() {}
 
 protected:
diff --git a/src/os/LevelDBStore.h b/src/os/LevelDBStore.h
index f3809cf3496..356ee59aa27 100644
--- a/src/os/LevelDBStore.h
+++ b/src/os/LevelDBStore.h
@@ -20,6 +20,12 @@
 #include "leveldb/filter_policy.h"
 #endif
 
+#include <errno.h>
+#include "common/errno.h"
+#include "common/dout.h"
+#include "include/assert.h"
+#include "common/Formatter.h"
+
 #include "common/ceph_context.h"
 
 class PerfCounters;
@@ -300,6 +306,64 @@ public:
     return limit;
   }
 
+  virtual uint64_t get_estimated_size(map<string,uint64_t> &extra) {
+    DIR *store_dir = opendir(path.c_str());
+    if (!store_dir) {
+      lderr(cct) << __func__ << " something happened opening the store: "
+                 << cpp_strerror(errno) << dendl;
+      return 0;
+    }
+
+    uint64_t total_size = 0;
+    uint64_t sst_size = 0;
+    uint64_t log_size = 0;
+    uint64_t misc_size = 0;
+
+    struct dirent *entry = NULL;
+    while ((entry = readdir(store_dir)) != NULL) {
+      string n(entry->d_name);
+
+      if (n == "." || n == "..")
+        continue;
+
+      string fpath = path + '/' + n;
+      struct stat s;
+      int err = stat(fpath.c_str(), &s);
+      if (err < 0) {
+        lderr(cct) << __func__ << " error obtaining stats for " << fpath
+                   << ": " << cpp_strerror(errno) << dendl;
+        goto err;
+      }
+
+      size_t pos = n.find_last_of('.');
+      if (pos == string::npos) {
+        misc_size += s.st_size;
+        continue;
+      }
+
+      string ext = n.substr(pos+1);
+      if (ext == "sst") {
+        sst_size += s.st_size;
+      } else if (ext == "log") {
+        log_size += s.st_size;
+      } else {
+        misc_size += s.st_size;
+      }
+    }
+
+    total_size = sst_size + log_size + misc_size;
+
+    extra["sst"] = sst_size;
+    extra["log"] = log_size;
+    extra["misc"] = misc_size;
+    extra["total"] = total_size;
+
+err:
+    closedir(store_dir);
+    return total_size;
+  }
+
+
 protected:
   WholeSpaceIterator _get_iterator() {
     return std::tr1::shared_ptr<KeyValueDB::WholeSpaceIteratorImpl>(
diff --git a/src/test/ObjectMap/KeyValueDBMemory.h b/src/test/ObjectMap/KeyValueDBMemory.h
index 93d0809d491..5cffce3ef04 100644
--- a/src/test/ObjectMap/KeyValueDBMemory.h
+++ b/src/test/ObjectMap/KeyValueDBMemory.h
@@ -126,6 +126,24 @@ public:
     return static_cast<TransactionImpl_*>(trans.get())->complete();
   }
 
+  uint64_t get_estimated_size(map<string,uint64_t> &extras) {
+    uint64_t total_size = 0;
+
+    for (map<pair<string,string>,bufferlist>::iterator p = db.begin();
+         p != db.end(); ++p) {
+      string prefix = p->first.first;
+      bufferlist &bl = p->second;
+
+      uint64_t sz = bl.length();
+      total_size += sz;
+      if (extras.count(prefix) == 0)
+        extras[prefix] = 0;
+      extras[prefix] += sz;
+    }
+
+    return total_size;
+  }
+
 private:
   bool exists_prefix(const string &prefix) {
     std::map<std::pair<string,string>,bufferlist>::iterator it;
diff --git a/src/test/ObjectMap/test_store_tool/test_store_tool.cc b/src/test/ObjectMap/test_store_tool/test_store_tool.cc
index ace91220df6..f81598ccfb8 100644
--- a/src/test/ObjectMap/test_store_tool/test_store_tool.cc
+++ b/src/test/ObjectMap/test_store_tool/test_store_tool.cc
@@ -90,6 +90,17 @@ class StoreTool
     exists = false;
     return bufferlist();
   }
+
+  uint64_t get_size() {
+    map<string,uint64_t> extras;
+    uint64_t s = db->get_estimated_size(extras);
+    for (map<string,uint64_t>::iterator p = extras.begin();
+         p != extras.end(); ++p) {
+      std::cout << p->first << " - " << p->second << std::endl;
+    }
+    std::cout << "total: " << s << std::endl;
+    return s;
+  }
 };
 
 void usage(const char *pname)
@@ -101,6 +112,7 @@ void usage(const char *pname)
     << "  exists <prefix> [key]\n"
     << "  get <prefix> <key>\n"
     << "  verify <store path>\n"
+    << "  get-size\n"
     << std::endl;
 }
 
@@ -173,6 +185,8 @@ int main(int argc, const char *argv[])
 
   } else if (cmd == "verify") {
     assert(0);
+  } else if (cmd == "get-size") {
+    std::cout << "estimated store size: " << st.get_size() << std::endl;
   } else {
     std::cerr << "Unrecognized command: " << cmd << std::endl;
     return 1;