SERVER-50756 Add unoccupied storage space to dbStats

The dbStats command will return how much unoccupied storage space is being used for collections,
indexes, and in total: freeStorageSize, indexFreeStorageSize, and totalFreeStorageSize.

17 files changed, 91 insertions, 8 deletions

diff --git a/jstests/core/dbstats.js b/jstests/core/dbstats.js
index f3670c12552..7de7003b770 100644
--- a/jstests/core/dbstats.js
+++ b/jstests/core/dbstats.js
@@ -2,6 +2,7 @@
 //
 // @tags: [
 //   requires_dbstats,
+//   requires_fcv_47,
 // ]
 
 (function() {
@@ -53,6 +54,11 @@ assert(dbStats.hasOwnProperty("totalSize"), tojson(dbStats));
 assert(dbStats.hasOwnProperty("indexSize"), tojson(dbStats));
 
 if (isUsingPersistentStorage) {
+    assert(dbStats.hasOwnProperty("freeStorageSize"), tojson(dbStats));
+    assert(dbStats.hasOwnProperty("indexFreeStorageSize"), tojson(dbStats));
+    assert(dbStats.hasOwnProperty("totalFreeStorageSize"), tojson(dbStats));
+    assert.eq(dbStats.freeStorageSize + dbStats.indexFreeStorageSize, dbStats.totalFreeStorageSize);
+
     assert(dbStats.hasOwnProperty("fsUsedSize"), tojson(dbStats));
     assert(dbStats.hasOwnProperty("fsTotalSize"), tojson(dbStats));
 }
diff --git a/jstests/noPassthroughWithMongod/free_storage_size.js b/jstests/noPassthroughWithMongod/free_storage_size.js
index fe5cf834293..4c19cd13cf6 100644
--- a/jstests/noPassthroughWithMongod/free_storage_size.js
+++ b/jstests/noPassthroughWithMongod/free_storage_size.js
@@ -1,6 +1,6 @@
 /**
- * Tests that db.collection.stats().freeStorageSize is a non-zero value after records are
- * inserted and then some deleted.
+ * Tests that dbStats and collStats report a non-zero freeStorageSize after records are inserted and
+ * then some deleted.
  *
  * @tags: [
  *   # freeStorageSize is currently only supported by WT.
@@ -42,4 +42,9 @@ forceCheckpoint();
 let collStats = assert.commandWorked(testDB.runCommand({collStats: collName}));
 assert(collStats.hasOwnProperty("freeStorageSize"), tojson(collStats));
 assert.gt(collStats.freeStorageSize, 0);
+
+let dbStats = assert.commandWorked(testDB.runCommand({dbStats: 1}));
+assert(dbStats.hasOwnProperty("freeStorageSize"), tojson(dbStats));
+assert.gt(dbStats.freeStorageSize, 0);
+assert.eq(dbStats.freeStorageSize, collStats.freeStorageSize);
 })();
diff --git a/src/mongo/db/catalog/collection.h b/src/mongo/db/catalog/collection.h
index 756d2f16171..b080647e78d 100644
--- a/src/mongo/db/catalog/collection.h
+++ b/src/mongo/db/catalog/collection.h
@@ -524,6 +524,11 @@ public:
                                   const int scale = 1) const = 0;
 
     /**
+     * Returns the number of unused, free bytes used by all indexes on disk.
+     */
+    virtual uint64_t getIndexFreeStorageBytes(OperationContext* const opCtx) const = 0;
+
+    /**
      * If return value is not boost::none, reads with majority read concern using an older snapshot
      * must error.
      */
diff --git a/src/mongo/db/catalog/collection_impl.cpp b/src/mongo/db/catalog/collection_impl.cpp
index 022735e9110..27e6771e21d 100644
--- a/src/mongo/db/catalog/collection_impl.cpp
+++ b/src/mongo/db/catalog/collection_impl.cpp
@@ -993,6 +993,19 @@ uint64_t CollectionImpl::getIndexSize(OperationContext* opCtx,
     return totalSize;
 }
 
+uint64_t CollectionImpl::getIndexFreeStorageBytes(OperationContext* const opCtx) const {
+    const auto idxCatalog = getIndexCatalog();
+    const bool includeUnfinished = true;
+    auto indexIt = idxCatalog->getIndexIterator(opCtx, includeUnfinished);
+
+    uint64_t totalSize = 0;
+    while (indexIt->more()) {
+        auto entry = indexIt->next();
+        totalSize += entry->accessMethod()->getFreeStorageBytes(opCtx);
+    }
+    return totalSize;
+}
+
 /**
  * order will be:
  * 1) store index specs
diff --git a/src/mongo/db/catalog/collection_impl.h b/src/mongo/db/catalog/collection_impl.h
index e6037e04ebc..ee72e1ddab1 100644
--- a/src/mongo/db/catalog/collection_impl.h
+++ b/src/mongo/db/catalog/collection_impl.h
@@ -320,6 +320,8 @@ public:
                           BSONObjBuilder* details = nullptr,
                           int scale = 1) const final;
 
+    uint64_t getIndexFreeStorageBytes(OperationContext* const opCtx) const final;
+
     /**
      * If return value is not boost::none, reads with majority read concern using an older snapshot
      * must error.
diff --git a/src/mongo/db/catalog/collection_mock.h b/src/mongo/db/catalog/collection_mock.h
index ad5b6bc8580..3a6bd51fe56 100644
--- a/src/mongo/db/catalog/collection_mock.h
+++ b/src/mongo/db/catalog/collection_mock.h
@@ -254,6 +254,10 @@ public:
         std::abort();
     }
 
+    uint64_t getIndexFreeStorageBytes(OperationContext* const opCtx) const {
+        std::abort();
+    }
+
     boost::optional<Timestamp> getMinimumVisibleSnapshot() const {
         std::abort();
     }
diff --git a/src/mongo/db/catalog/database_impl.cpp b/src/mongo/db/catalog/database_impl.cpp
index 83d902c9b2e..b1639f61cb3 100644
--- a/src/mongo/db/catalog/database_impl.cpp
+++ b/src/mongo/db/catalog/database_impl.cpp
@@ -254,8 +254,10 @@ void DatabaseImpl::getStats(OperationContext* opCtx, BSONObjBuilder* output, dou
     long long objects = 0;
     long long size = 0;
     long long storageSize = 0;
+    long long freeStorageSize = 0;
     long long indexes = 0;
     long long indexSize = 0;
+    long long indexFreeStorageSize = 0;
 
     invariant(opCtx->lockState()->isDbLockedForMode(name(), MODE_IS));
 
@@ -267,9 +269,11 @@ void DatabaseImpl::getStats(OperationContext* opCtx, BSONObjBuilder* output, dou
 
             BSONObjBuilder temp;
             storageSize += collection->getRecordStore()->storageSize(opCtx, &temp);
+            freeStorageSize += collection->getRecordStore()->freeStorageSize(opCtx);
 
             indexes += collection->getIndexCatalog()->numIndexesTotal(opCtx);
             indexSize += collection->getIndexSize(opCtx);
+            indexFreeStorageSize += collection->getIndexFreeStorageBytes(opCtx);
 
             return true;
         });
@@ -282,9 +286,12 @@ void DatabaseImpl::getStats(OperationContext* opCtx, BSONObjBuilder* output, dou
     output->append("avgObjSize", objects == 0 ? 0 : double(size) / double(objects));
     output->appendNumber("dataSize", size / scale);
     output->appendNumber("storageSize", storageSize / scale);
+    output->appendNumber("freeStorageSize", freeStorageSize / scale);
     output->appendNumber("indexes", indexes);
     output->appendNumber("indexSize", indexSize / scale);
+    output->appendNumber("indexFreeStorageSize", indexFreeStorageSize / scale);
     output->appendNumber("totalSize", (storageSize + indexSize) / scale);
+    output->appendNumber("totalFreeStorageSize", (freeStorageSize + indexFreeStorageSize) / scale);
     output->appendNumber("scaleFactor", scale);
 
     if (!opCtx->getServiceContext()->getStorageEngine()->isEphemeral()) {
diff --git a/src/mongo/db/index/index_access_method.cpp b/src/mongo/db/index/index_access_method.cpp
index 6080125c723..6bdd126d250 100644
--- a/src/mongo/db/index/index_access_method.cpp
+++ b/src/mongo/db/index/index_access_method.cpp
@@ -332,6 +332,10 @@ long long AbstractIndexAccessMethod::getSpaceUsedBytes(OperationContext* opCtx)
     return _newInterface->getSpaceUsedBytes(opCtx);
 }
 
+long long AbstractIndexAccessMethod::getFreeStorageBytes(OperationContext* opCtx) const {
+    return _newInterface->getFreeStorageBytes(opCtx);
+}
+
 pair<KeyStringSet, KeyStringSet> AbstractIndexAccessMethod::setDifference(
     const KeyStringSet& left, const KeyStringSet& right) {
     // Two iterators to traverse the two sets in sorted order.
diff --git a/src/mongo/db/index/index_access_method.h b/src/mongo/db/index/index_access_method.h
index 15d32be7f1b..013a6663c3a 100644
--- a/src/mongo/db/index/index_access_method.h
+++ b/src/mongo/db/index/index_access_method.h
@@ -203,6 +203,11 @@ public:
      */
     virtual long long getSpaceUsedBytes(OperationContext* opCtx) const = 0;
 
+    /**
+     * The number of unused free bytes consumed by this index on disk.
+     */
+    virtual long long getFreeStorageBytes(OperationContext* opCtx) const = 0;
+
     virtual RecordId findSingle(OperationContext* opCtx, const BSONObj& key) const = 0;
 
     /**
@@ -515,6 +520,8 @@ public:
 
     long long getSpaceUsedBytes(OperationContext* opCtx) const final;
 
+    long long getFreeStorageBytes(OperationContext* opCtx) const final;
+
     RecordId findSingle(OperationContext* opCtx, const BSONObj& key) const final;
 
     Status compact(OperationContext* opCtx) final;
diff --git a/src/mongo/db/storage/devnull/devnull_kv_engine.cpp b/src/mongo/db/storage/devnull/devnull_kv_engine.cpp
index 6c1c4435b7c..c8e6af831c6 100644
--- a/src/mongo/db/storage/devnull/devnull_kv_engine.cpp
+++ b/src/mongo/db/storage/devnull/devnull_kv_engine.cpp
@@ -204,6 +204,10 @@ public:
         return 0;
     }
 
+    virtual long long getFreeStorageBytes(OperationContext* opCtx) const {
+        return 0;
+    }
+
     virtual bool isEmpty(OperationContext* opCtx) {
         return true;
     }
diff --git a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_sorted_impl.h b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_sorted_impl.h
index 629f1bedb66..4dfddae8a15 100644
--- a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_sorted_impl.h
+++ b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_sorted_impl.h
@@ -82,6 +82,9 @@ public:
                            BSONObjBuilder* output,
                            double scale) const override;
     long long getSpaceUsedBytes(OperationContext* opCtx) const override;
+    long long getFreeStorageBytes(OperationContext* opCtx) const override {
+        return 0;
+    }
     bool isEmpty(OperationContext* opCtx) override;
     Status initAsEmpty(OperationContext* opCtx) override;
 
diff --git a/src/mongo/db/storage/sorted_data_interface.h b/src/mongo/db/storage/sorted_data_interface.h
index 186dd20ea20..3203f89fb83 100644
--- a/src/mongo/db/storage/sorted_data_interface.h
+++ b/src/mongo/db/storage/sorted_data_interface.h
@@ -150,6 +150,11 @@ public:
     virtual long long getSpaceUsedBytes(OperationContext* opCtx) const = 0;
 
     /**
+     * The number of unused free bytes consumed by this index on disk.
+     */
+    virtual long long getFreeStorageBytes(OperationContext* opCtx) const = 0;
+
+    /**
      * Return true if 'this' index is empty, and false otherwise.
      */
     virtual bool isEmpty(OperationContext* opCtx) = 0;
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp
index 6046ab6e43e..455a1569fe8 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp
@@ -470,6 +470,14 @@ long long WiredTigerIndex::getSpaceUsedBytes(OperationContext* opCtx) const {
     return static_cast<long long>(WiredTigerUtil::getIdentSize(session->getSession(), _uri));
 }
 
+long long WiredTigerIndex::getFreeStorageBytes(OperationContext* opCtx) const {
+    dassert(opCtx->lockState()->isReadLocked());
+    auto ru = WiredTigerRecoveryUnit::get(opCtx);
+    WiredTigerSession* session = ru->getSession();
+
+    return static_cast<long long>(WiredTigerUtil::getIdentReuseSize(session->getSession(), _uri));
+}
+
 bool WiredTigerIndex::isDup(OperationContext* opCtx, WT_CURSOR* c, const KeyString::Value& key) {
     dassert(opCtx->lockState()->isReadLocked());
     invariant(unique());
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_index.h b/src/mongo/db/storage/wiredtiger/wiredtiger_index.h
index f5dc594d30b..a0268a4181c 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_index.h
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_index.h
@@ -118,6 +118,8 @@ public:
 
     virtual long long getSpaceUsedBytes(OperationContext* opCtx) const;
 
+    virtual long long getFreeStorageBytes(OperationContext* opCtx) const;
+
     virtual Status initAsEmpty(OperationContext* opCtx);
 
     Status compact(OperationContext* opCtx) override;
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp
index a2adafb35ea..d6338be1b1d 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp
@@ -1000,12 +1000,7 @@ int64_t WiredTigerRecordStore::freeStorageSize(OperationContext* opCtx) const {
     invariant(opCtx->lockState()->isReadLocked());
 
     WiredTigerSession* session = WiredTigerRecoveryUnit::get(opCtx)->getSessionNoTxn();
-    auto result = WiredTigerUtil::getStatisticsValue(session->getSession(),
-                                                     "statistics:" + getURI(),
-                                                     "statistics=(fast)",
-                                                     WT_STAT_DSRC_BLOCK_REUSE_BYTES);
-    uassertStatusOK(result.getStatus());
-    return result.getValue();
+    return WiredTigerUtil::getIdentReuseSize(session->getSession(), getURI());
 }
 
 // Retrieve the value from a positioned cursor.
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp
index e94e94dcc10..13bc45e0693 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp
@@ -468,6 +468,13 @@ int64_t WiredTigerUtil::getIdentSize(WT_SESSION* s, const std::string& uri) {
     return result.getValue();
 }
 
+int64_t WiredTigerUtil::getIdentReuseSize(WT_SESSION* s, const std::string& uri) {
+    auto result = WiredTigerUtil::getStatisticsValue(
+        s, "statistics:" + uri, "statistics=(fast)", WT_STAT_DSRC_BLOCK_REUSE_BYTES);
+    uassertStatusOK(result.getStatus());
+    return result.getValue();
+}
+
 size_t WiredTigerUtil::getCacheSizeMB(double requestedCacheSizeGB) {
     double cacheSizeMB;
     const double kMaxSizeCacheMB = 10 * 1000 * 1000;
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_util.h b/src/mongo/db/storage/wiredtiger/wiredtiger_util.h
index 5150c19e153..57a8d7189fc 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_util.h
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_util.h
@@ -220,6 +220,12 @@ public:
 
     static int64_t getIdentSize(WT_SESSION* s, const std::string& uri);
 
+    /**
+     * Returns the bytes available for reuse for an ident. This is the amount of allocated space on
+     * disk that is not storing any data.
+     */
+    static int64_t getIdentReuseSize(WT_SESSION* s, const std::string& uri);
+
 
     /**
      * Return amount of memory to use for the WiredTiger cache based on either the startup