SERVER-17161 Fix Rocks build broken with getSnapshotId()

Also piggyback some configuration improvements.

Signed-off-by: Benety Goh <benety@mongodb.com>
(cherry picked from commit 5e128ddad3f15639c1719008bb041c21b2600769)

5 files changed, 46 insertions, 15 deletions

diff --git a/src/mongo/db/storage/rocks/SConscript b/src/mongo/db/storage/rocks/SConscript
index 76714dbff14..c171bd238d2 100644
--- a/src/mongo/db/storage/rocks/SConscript
+++ b/src/mongo/db/storage/rocks/SConscript
@@ -22,6 +22,7 @@ if has_option("rocksdb"):
             '$BUILD_DIR/mongo/db/storage/key_string',
             '$BUILD_DIR/mongo/db/storage/oplog_hack',
             '$BUILD_DIR/mongo/foundation',
+            '$BUILD_DIR/mongo/processinfo',
             '$BUILD_DIR/third_party/shim_snappy',
             ],
         SYSLIBDEPS=["rocksdb",
diff --git a/src/mongo/db/storage/rocks/rocks_engine.cpp b/src/mongo/db/storage/rocks/rocks_engine.cpp
index 1781b7d89ee..604798cb3ea 100644
--- a/src/mongo/db/storage/rocks/rocks_engine.cpp
+++ b/src/mongo/db/storage/rocks/rocks_engine.cpp
@@ -37,10 +37,13 @@
 #include <boost/make_shared.hpp>
 #include <boost/shared_ptr.hpp>
 
+#include <rocksdb/cache.h>
 #include <rocksdb/comparator.h>
 #include <rocksdb/db.h>
 #include <rocksdb/slice.h>
 #include <rocksdb/options.h>
+#include <rocksdb/table.h>
+#include <rocksdb/filter_policy.h>
 #include <rocksdb/utilities/write_batch_with_index.h>
 
 #include "mongo/db/catalog/collection_options.h"
@@ -50,6 +53,7 @@
 #include "mongo/db/storage/rocks/rocks_recovery_unit.h"
 #include "mongo/db/storage/rocks/rocks_index.h"
 #include "mongo/util/log.h"
+#include "mongo/util/processinfo.h"
 
 #define ROCKS_TRACE log()
 
@@ -67,6 +71,20 @@ namespace mongo {
         : _path(path),
           _durable(durable) {
 
+        { // create block cache
+            uint64_t cacheSizeGB = 0;
+            ProcessInfo pi;
+            unsigned long long memSizeMB = pi.getMemSizeMB();
+            if (memSizeMB > 0) {
+                double cacheMB = memSizeMB / 2;
+                cacheSizeGB = static_cast<uint64_t>(cacheMB / 1024);
+            }
+            if (cacheSizeGB < 1) {
+                cacheSizeGB = 1;
+            }
+            _block_cache = rocksdb::NewLRUCache(cacheSizeGB * 1024 * 1024 * 1024LL);
+        }
+
         auto columnFamilyNames = _loadColumnFamilies();       // vector of column family names
         std::unordered_map<std::string, Ordering> orderings;  // column family name -> Ordering
         std::set<std::string> collections;                    // set of collection names
@@ -295,12 +313,11 @@ namespace mongo {
         return names;
     }
 
-    rocksdb::Options RocksEngine::_dbOptions() const {
+    rocksdb::Options RocksEngine::_dbOptions() {
         rocksdb::Options options(rocksdb::DBOptions(), _defaultCFOptions());
 
-        // Optimize RocksDB. This is the easiest way to get RocksDB to perform well
-        options.IncreaseParallelism();
-        options.OptimizeLevelStyleCompaction();
+        options.max_background_compactions = 4;
+        options.max_background_flushes = 4;
 
         // create the DB if it's not already present
         options.create_if_missing = true;
@@ -312,18 +329,22 @@ namespace mongo {
     }
 
     rocksdb::ColumnFamilyOptions RocksEngine::_defaultCFOptions() {
-        rocksdb::ColumnFamilyOptions options;
         // TODO pass or set appropriate options for default CF.
+        rocksdb::ColumnFamilyOptions options;
+        rocksdb::BlockBasedTableOptions table_options;
+        table_options.block_cache = _block_cache;
+        table_options.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10));
+        options.table_factory.reset(rocksdb::NewBlockBasedTableFactory(table_options));
+        options.max_write_buffer_number = 4;
         return options;
     }
 
-    rocksdb::ColumnFamilyOptions RocksEngine::_collectionOptions() const {
-        rocksdb::ColumnFamilyOptions options;
-        return options;
+    rocksdb::ColumnFamilyOptions RocksEngine::_collectionOptions() {
+        return _defaultCFOptions();
     }
 
-    rocksdb::ColumnFamilyOptions RocksEngine::_indexOptions(const Ordering& order) const {
-        return rocksdb::ColumnFamilyOptions();
+    rocksdb::ColumnFamilyOptions RocksEngine::_indexOptions(const Ordering& order) {
+        return _defaultCFOptions();
     }
 
     Status toMongoStatus( rocksdb::Status s ) {
diff --git a/src/mongo/db/storage/rocks/rocks_engine.h b/src/mongo/db/storage/rocks/rocks_engine.h
index 8f344eea60a..12ee7eaf355 100644
--- a/src/mongo/db/storage/rocks/rocks_engine.h
+++ b/src/mongo/db/storage/rocks/rocks_engine.h
@@ -39,6 +39,7 @@
 #include <boost/shared_ptr.hpp>
 #include <boost/thread/mutex.hpp>
 
+#include <rocksdb/cache.h>
 #include <rocksdb/status.h>
 
 #include "mongo/base/disallow_copying.h"
@@ -132,15 +133,16 @@ namespace mongo {
         std::set<std::string> _loadCollections(rocksdb::Iterator* itr);
         std::vector<std::string> _loadColumnFamilies();
 
-        rocksdb::ColumnFamilyOptions _collectionOptions() const;
-        rocksdb::ColumnFamilyOptions _indexOptions(const Ordering& order) const;
+        rocksdb::ColumnFamilyOptions _collectionOptions();
+        rocksdb::ColumnFamilyOptions _indexOptions(const Ordering& order);
 
-        rocksdb::Options _dbOptions() const;
+        rocksdb::Options _dbOptions();
 
-        static rocksdb::ColumnFamilyOptions _defaultCFOptions();
+        rocksdb::ColumnFamilyOptions _defaultCFOptions();
 
         std::string _path;
         boost::scoped_ptr<rocksdb::DB> _db;
+        std::shared_ptr<rocksdb::Cache> _block_cache;
 
         const bool _durable;
 
diff --git a/src/mongo/db/storage/rocks/rocks_recovery_unit.cpp b/src/mongo/db/storage/rocks/rocks_recovery_unit.cpp
index eecf0da7957..05237f5941e 100644
--- a/src/mongo/db/storage/rocks/rocks_recovery_unit.cpp
+++ b/src/mongo/db/storage/rocks/rocks_recovery_unit.cpp
@@ -56,7 +56,8 @@ namespace mongo {
           _transaction(transactionEngine),
           _writeBatch(),
           _snapshot(NULL),
-          _depth(0) {}
+          _depth(0),
+          _myTransactionCount(1) {}
 
     RocksRecoveryUnit::~RocksRecoveryUnit() {
         _abort();
@@ -117,11 +118,14 @@ namespace mongo {
 
     void RocksRecoveryUnit::registerChange(Change* change) { _changes.push_back(change); }
 
+    SnapshotId RocksRecoveryUnit::getSnapshotId() const { return SnapshotId(_myTransactionCount); }
+
     void RocksRecoveryUnit::_releaseSnapshot() {
         if (_snapshot) {
             _db->ReleaseSnapshot(_snapshot);
             _snapshot = nullptr;
         }
+        _myTransactionCount++;
     }
 
     void RocksRecoveryUnit::_commit() {
diff --git a/src/mongo/db/storage/rocks/rocks_recovery_unit.h b/src/mongo/db/storage/rocks/rocks_recovery_unit.h
index 5dfdc98214a..e1f95e1b0a7 100644
--- a/src/mongo/db/storage/rocks/rocks_recovery_unit.h
+++ b/src/mongo/db/storage/rocks/rocks_recovery_unit.h
@@ -80,6 +80,8 @@ namespace mongo {
 
         virtual void setRollbackWritesDisabled() {}
 
+        virtual SnapshotId getSnapshotId() const;
+
         // local api
 
         rocksdb::WriteBatchWithIndex* writeBatch();
@@ -141,6 +143,7 @@ namespace mongo {
         Changes _changes;
 
         int _depth;
+        uint64_t _myTransactionCount;
 
         RecordId _oplogReadTill;
     };