os: KeyValueDB: expose interface to obtain estimated store size

On LevelDBStore, instead of using leveldb's GetApproximateSizes() function,
we will instead assess what's the store's raw size from the contents of
the store dir (this means .sst's, .log's, etc).  The reason behind this
approach is that GetApproximateSizes() would expect us to provide a range
of keys for which to obtain an approximate size; on the other hand, what we
really want is to obtain the size of the store -- not the size of the
data (besides, with the compaction issues we've been seeing, we wonder
how reliable such approximation would be).

Signed-off-by: Joao Eduardo Luis <jecluis@gmail.com>

3 files changed, 84 insertions, 0 deletions

diff --git a/src/os/KeyValueDB.h b/src/os/KeyValueDB.h
index f62bca996a5..e98463aa763 100644
--- a/src/os/KeyValueDB.h
+++ b/src/os/KeyValueDB.h
@@ -165,6 +165,8 @@ public:
     );
   }
 
+  virtual uint64_t get_estimated_size(map<string,uint64_t> &extra) = 0;
+
   virtual ~KeyValueDB() {}
 
 protected:
diff --git a/src/os/LevelDBStore.h b/src/os/LevelDBStore.h
index f3809cf3496..356ee59aa27 100644
--- a/src/os/LevelDBStore.h
+++ b/src/os/LevelDBStore.h
@@ -20,6 +20,12 @@
 #include "leveldb/filter_policy.h"
 #endif
 
+#include <errno.h>
+#include "common/errno.h"
+#include "common/dout.h"
+#include "include/assert.h"
+#include "common/Formatter.h"
+
 #include "common/ceph_context.h"
 
 class PerfCounters;
@@ -300,6 +306,64 @@ public:
     return limit;
   }
 
+  virtual uint64_t get_estimated_size(map<string,uint64_t> &extra) {
+    DIR *store_dir = opendir(path.c_str());
+    if (!store_dir) {
+      lderr(cct) << __func__ << " something happened opening the store: "
+                 << cpp_strerror(errno) << dendl;
+      return 0;
+    }
+
+    uint64_t total_size = 0;
+    uint64_t sst_size = 0;
+    uint64_t log_size = 0;
+    uint64_t misc_size = 0;
+
+    struct dirent *entry = NULL;
+    while ((entry = readdir(store_dir)) != NULL) {
+      string n(entry->d_name);
+
+      if (n == "." || n == "..")
+        continue;
+
+      string fpath = path + '/' + n;
+      struct stat s;
+      int err = stat(fpath.c_str(), &s);
+      if (err < 0) {
+        lderr(cct) << __func__ << " error obtaining stats for " << fpath
+                   << ": " << cpp_strerror(errno) << dendl;
+        goto err;
+      }
+
+      size_t pos = n.find_last_of('.');
+      if (pos == string::npos) {
+        misc_size += s.st_size;
+        continue;
+      }
+
+      string ext = n.substr(pos+1);
+      if (ext == "sst") {
+        sst_size += s.st_size;
+      } else if (ext == "log") {
+        log_size += s.st_size;
+      } else {
+        misc_size += s.st_size;
+      }
+    }
+
+    total_size = sst_size + log_size + misc_size;
+
+    extra["sst"] = sst_size;
+    extra["log"] = log_size;
+    extra["misc"] = misc_size;
+    extra["total"] = total_size;
+
+err:
+    closedir(store_dir);
+    return total_size;
+  }
+
+
 protected:
   WholeSpaceIterator _get_iterator() {
     return std::tr1::shared_ptr<KeyValueDB::WholeSpaceIteratorImpl>(
diff --git a/src/test/ObjectMap/KeyValueDBMemory.h b/src/test/ObjectMap/KeyValueDBMemory.h
index 93d0809d491..5cffce3ef04 100644
--- a/src/test/ObjectMap/KeyValueDBMemory.h
+++ b/src/test/ObjectMap/KeyValueDBMemory.h
@@ -126,6 +126,24 @@ public:
     return static_cast<TransactionImpl_*>(trans.get())->complete();
   }
 
+  uint64_t get_estimated_size(map<string,uint64_t> &extras) {
+    uint64_t total_size = 0;
+
+    for (map<pair<string,string>,bufferlist>::iterator p = db.begin();
+         p != db.end(); ++p) {
+      string prefix = p->first.first;
+      bufferlist &bl = p->second;
+
+      uint64_t sz = bl.length();
+      total_size += sz;
+      if (extras.count(prefix) == 0)
+        extras[prefix] = 0;
+      extras[prefix] += sz;
+    }
+
+    return total_size;
+  }
+
 private:
   bool exists_prefix(const string &prefix) {
     std::map<std::pair<string,string>,bufferlist>::iterator it;