diff options
-rw-r--r-- | jstests/core/dbstats.js | 6 | ||||
-rw-r--r-- | jstests/noPassthroughWithMongod/free_storage_size.js | 9 | ||||
-rw-r--r-- | src/mongo/db/catalog/collection.h | 5 | ||||
-rw-r--r-- | src/mongo/db/catalog/collection_impl.cpp | 13 | ||||
-rw-r--r-- | src/mongo/db/catalog/collection_impl.h | 2 | ||||
-rw-r--r-- | src/mongo/db/catalog/collection_mock.h | 4 | ||||
-rw-r--r-- | src/mongo/db/catalog/database_impl.cpp | 7 | ||||
-rw-r--r-- | src/mongo/db/index/index_access_method.cpp | 4 | ||||
-rw-r--r-- | src/mongo/db/index/index_access_method.h | 7 | ||||
-rw-r--r-- | src/mongo/db/storage/devnull/devnull_kv_engine.cpp | 4 | ||||
-rw-r--r-- | src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_sorted_impl.h | 3 | ||||
-rw-r--r-- | src/mongo/db/storage/sorted_data_interface.h | 5 | ||||
-rw-r--r-- | src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp | 8 | ||||
-rw-r--r-- | src/mongo/db/storage/wiredtiger/wiredtiger_index.h | 2 | ||||
-rw-r--r-- | src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp | 7 | ||||
-rw-r--r-- | src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp | 7 | ||||
-rw-r--r-- | src/mongo/db/storage/wiredtiger/wiredtiger_util.h | 6 |
17 files changed, 91 insertions, 8 deletions
diff --git a/jstests/core/dbstats.js b/jstests/core/dbstats.js index f3670c12552..7de7003b770 100644 --- a/jstests/core/dbstats.js +++ b/jstests/core/dbstats.js @@ -2,6 +2,7 @@ // // @tags: [ // requires_dbstats, +// requires_fcv_47, // ] (function() { @@ -53,6 +54,11 @@ assert(dbStats.hasOwnProperty("totalSize"), tojson(dbStats)); assert(dbStats.hasOwnProperty("indexSize"), tojson(dbStats)); if (isUsingPersistentStorage) { + assert(dbStats.hasOwnProperty("freeStorageSize"), tojson(dbStats)); + assert(dbStats.hasOwnProperty("indexFreeStorageSize"), tojson(dbStats)); + assert(dbStats.hasOwnProperty("totalFreeStorageSize"), tojson(dbStats)); + assert.eq(dbStats.freeStorageSize + dbStats.indexFreeStorageSize, dbStats.totalFreeStorageSize); + assert(dbStats.hasOwnProperty("fsUsedSize"), tojson(dbStats)); assert(dbStats.hasOwnProperty("fsTotalSize"), tojson(dbStats)); } diff --git a/jstests/noPassthroughWithMongod/free_storage_size.js b/jstests/noPassthroughWithMongod/free_storage_size.js index fe5cf834293..4c19cd13cf6 100644 --- a/jstests/noPassthroughWithMongod/free_storage_size.js +++ b/jstests/noPassthroughWithMongod/free_storage_size.js @@ -1,6 +1,6 @@ /** - * Tests that db.collection.stats().freeStorageSize is a non-zero value after records are - * inserted and then some deleted. + * Tests that dbStats and collStats report a non-zero freeStorageSize after records are inserted and + * then some deleted. * * @tags: [ * # freeStorageSize is currently only supported by WT. @@ -42,4 +42,9 @@ forceCheckpoint(); let collStats = assert.commandWorked(testDB.runCommand({collStats: collName})); assert(collStats.hasOwnProperty("freeStorageSize"), tojson(collStats)); assert.gt(collStats.freeStorageSize, 0); + +let dbStats = assert.commandWorked(testDB.runCommand({dbStats: 1})); +assert(dbStats.hasOwnProperty("freeStorageSize"), tojson(dbStats)); +assert.gt(dbStats.freeStorageSize, 0); +assert.eq(dbStats.freeStorageSize, collStats.freeStorageSize); })(); diff --git a/src/mongo/db/catalog/collection.h b/src/mongo/db/catalog/collection.h index 756d2f16171..b080647e78d 100644 --- a/src/mongo/db/catalog/collection.h +++ b/src/mongo/db/catalog/collection.h @@ -524,6 +524,11 @@ public: const int scale = 1) const = 0; /** + * Returns the number of unused, free bytes used by all indexes on disk. + */ + virtual uint64_t getIndexFreeStorageBytes(OperationContext* const opCtx) const = 0; + + /** * If return value is not boost::none, reads with majority read concern using an older snapshot * must error. */ diff --git a/src/mongo/db/catalog/collection_impl.cpp b/src/mongo/db/catalog/collection_impl.cpp index 022735e9110..27e6771e21d 100644 --- a/src/mongo/db/catalog/collection_impl.cpp +++ b/src/mongo/db/catalog/collection_impl.cpp @@ -993,6 +993,19 @@ uint64_t CollectionImpl::getIndexSize(OperationContext* opCtx, return totalSize; } +uint64_t CollectionImpl::getIndexFreeStorageBytes(OperationContext* const opCtx) const { + const auto idxCatalog = getIndexCatalog(); + const bool includeUnfinished = true; + auto indexIt = idxCatalog->getIndexIterator(opCtx, includeUnfinished); + + uint64_t totalSize = 0; + while (indexIt->more()) { + auto entry = indexIt->next(); + totalSize += entry->accessMethod()->getFreeStorageBytes(opCtx); + } + return totalSize; +} + /** * order will be: * 1) store index specs diff --git a/src/mongo/db/catalog/collection_impl.h b/src/mongo/db/catalog/collection_impl.h index e6037e04ebc..ee72e1ddab1 100644 --- a/src/mongo/db/catalog/collection_impl.h +++ b/src/mongo/db/catalog/collection_impl.h @@ -320,6 +320,8 @@ public: BSONObjBuilder* details = nullptr, int scale = 1) const final; + uint64_t getIndexFreeStorageBytes(OperationContext* const opCtx) const final; + /** * If return value is not boost::none, reads with majority read concern using an older snapshot * must error. diff --git a/src/mongo/db/catalog/collection_mock.h b/src/mongo/db/catalog/collection_mock.h index ad5b6bc8580..3a6bd51fe56 100644 --- a/src/mongo/db/catalog/collection_mock.h +++ b/src/mongo/db/catalog/collection_mock.h @@ -254,6 +254,10 @@ public: std::abort(); } + uint64_t getIndexFreeStorageBytes(OperationContext* const opCtx) const { + std::abort(); + } + boost::optional<Timestamp> getMinimumVisibleSnapshot() const { std::abort(); } diff --git a/src/mongo/db/catalog/database_impl.cpp b/src/mongo/db/catalog/database_impl.cpp index 83d902c9b2e..b1639f61cb3 100644 --- a/src/mongo/db/catalog/database_impl.cpp +++ b/src/mongo/db/catalog/database_impl.cpp @@ -254,8 +254,10 @@ void DatabaseImpl::getStats(OperationContext* opCtx, BSONObjBuilder* output, dou long long objects = 0; long long size = 0; long long storageSize = 0; + long long freeStorageSize = 0; long long indexes = 0; long long indexSize = 0; + long long indexFreeStorageSize = 0; invariant(opCtx->lockState()->isDbLockedForMode(name(), MODE_IS)); @@ -267,9 +269,11 @@ void DatabaseImpl::getStats(OperationContext* opCtx, BSONObjBuilder* output, dou BSONObjBuilder temp; storageSize += collection->getRecordStore()->storageSize(opCtx, &temp); + freeStorageSize += collection->getRecordStore()->freeStorageSize(opCtx); indexes += collection->getIndexCatalog()->numIndexesTotal(opCtx); indexSize += collection->getIndexSize(opCtx); + indexFreeStorageSize += collection->getIndexFreeStorageBytes(opCtx); return true; }); @@ -282,9 +286,12 @@ void DatabaseImpl::getStats(OperationContext* opCtx, BSONObjBuilder* output, dou output->append("avgObjSize", objects == 0 ? 0 : double(size) / double(objects)); output->appendNumber("dataSize", size / scale); output->appendNumber("storageSize", storageSize / scale); + output->appendNumber("freeStorageSize", freeStorageSize / scale); output->appendNumber("indexes", indexes); output->appendNumber("indexSize", indexSize / scale); + output->appendNumber("indexFreeStorageSize", indexFreeStorageSize / scale); output->appendNumber("totalSize", (storageSize + indexSize) / scale); + output->appendNumber("totalFreeStorageSize", (freeStorageSize + indexFreeStorageSize) / scale); output->appendNumber("scaleFactor", scale); if (!opCtx->getServiceContext()->getStorageEngine()->isEphemeral()) { diff --git a/src/mongo/db/index/index_access_method.cpp b/src/mongo/db/index/index_access_method.cpp index 6080125c723..6bdd126d250 100644 --- a/src/mongo/db/index/index_access_method.cpp +++ b/src/mongo/db/index/index_access_method.cpp @@ -332,6 +332,10 @@ long long AbstractIndexAccessMethod::getSpaceUsedBytes(OperationContext* opCtx) return _newInterface->getSpaceUsedBytes(opCtx); } +long long AbstractIndexAccessMethod::getFreeStorageBytes(OperationContext* opCtx) const { + return _newInterface->getFreeStorageBytes(opCtx); +} + pair<KeyStringSet, KeyStringSet> AbstractIndexAccessMethod::setDifference( const KeyStringSet& left, const KeyStringSet& right) { // Two iterators to traverse the two sets in sorted order. diff --git a/src/mongo/db/index/index_access_method.h b/src/mongo/db/index/index_access_method.h index 15d32be7f1b..013a6663c3a 100644 --- a/src/mongo/db/index/index_access_method.h +++ b/src/mongo/db/index/index_access_method.h @@ -203,6 +203,11 @@ public: */ virtual long long getSpaceUsedBytes(OperationContext* opCtx) const = 0; + /** + * The number of unused free bytes consumed by this index on disk. + */ + virtual long long getFreeStorageBytes(OperationContext* opCtx) const = 0; + virtual RecordId findSingle(OperationContext* opCtx, const BSONObj& key) const = 0; /** @@ -515,6 +520,8 @@ public: long long getSpaceUsedBytes(OperationContext* opCtx) const final; + long long getFreeStorageBytes(OperationContext* opCtx) const final; + RecordId findSingle(OperationContext* opCtx, const BSONObj& key) const final; Status compact(OperationContext* opCtx) final; diff --git a/src/mongo/db/storage/devnull/devnull_kv_engine.cpp b/src/mongo/db/storage/devnull/devnull_kv_engine.cpp index 6c1c4435b7c..c8e6af831c6 100644 --- a/src/mongo/db/storage/devnull/devnull_kv_engine.cpp +++ b/src/mongo/db/storage/devnull/devnull_kv_engine.cpp @@ -204,6 +204,10 @@ public: return 0; } + virtual long long getFreeStorageBytes(OperationContext* opCtx) const { + return 0; + } + virtual bool isEmpty(OperationContext* opCtx) { return true; } diff --git a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_sorted_impl.h b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_sorted_impl.h index 629f1bedb66..4dfddae8a15 100644 --- a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_sorted_impl.h +++ b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_sorted_impl.h @@ -82,6 +82,9 @@ public: BSONObjBuilder* output, double scale) const override; long long getSpaceUsedBytes(OperationContext* opCtx) const override; + long long getFreeStorageBytes(OperationContext* opCtx) const override { + return 0; + } bool isEmpty(OperationContext* opCtx) override; Status initAsEmpty(OperationContext* opCtx) override; diff --git a/src/mongo/db/storage/sorted_data_interface.h b/src/mongo/db/storage/sorted_data_interface.h index 186dd20ea20..3203f89fb83 100644 --- a/src/mongo/db/storage/sorted_data_interface.h +++ b/src/mongo/db/storage/sorted_data_interface.h @@ -150,6 +150,11 @@ public: virtual long long getSpaceUsedBytes(OperationContext* opCtx) const = 0; /** + * The number of unused free bytes consumed by this index on disk. + */ + virtual long long getFreeStorageBytes(OperationContext* opCtx) const = 0; + + /** * Return true if 'this' index is empty, and false otherwise. */ virtual bool isEmpty(OperationContext* opCtx) = 0; diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp index 6046ab6e43e..455a1569fe8 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp @@ -470,6 +470,14 @@ long long WiredTigerIndex::getSpaceUsedBytes(OperationContext* opCtx) const { return static_cast<long long>(WiredTigerUtil::getIdentSize(session->getSession(), _uri)); } +long long WiredTigerIndex::getFreeStorageBytes(OperationContext* opCtx) const { + dassert(opCtx->lockState()->isReadLocked()); + auto ru = WiredTigerRecoveryUnit::get(opCtx); + WiredTigerSession* session = ru->getSession(); + + return static_cast<long long>(WiredTigerUtil::getIdentReuseSize(session->getSession(), _uri)); +} + bool WiredTigerIndex::isDup(OperationContext* opCtx, WT_CURSOR* c, const KeyString::Value& key) { dassert(opCtx->lockState()->isReadLocked()); invariant(unique()); diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_index.h b/src/mongo/db/storage/wiredtiger/wiredtiger_index.h index f5dc594d30b..a0268a4181c 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_index.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_index.h @@ -118,6 +118,8 @@ public: virtual long long getSpaceUsedBytes(OperationContext* opCtx) const; + virtual long long getFreeStorageBytes(OperationContext* opCtx) const; + virtual Status initAsEmpty(OperationContext* opCtx); Status compact(OperationContext* opCtx) override; diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp index a2adafb35ea..d6338be1b1d 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp @@ -1000,12 +1000,7 @@ int64_t WiredTigerRecordStore::freeStorageSize(OperationContext* opCtx) const { invariant(opCtx->lockState()->isReadLocked()); WiredTigerSession* session = WiredTigerRecoveryUnit::get(opCtx)->getSessionNoTxn(); - auto result = WiredTigerUtil::getStatisticsValue(session->getSession(), - "statistics:" + getURI(), - "statistics=(fast)", - WT_STAT_DSRC_BLOCK_REUSE_BYTES); - uassertStatusOK(result.getStatus()); - return result.getValue(); + return WiredTigerUtil::getIdentReuseSize(session->getSession(), getURI()); } // Retrieve the value from a positioned cursor. diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp index e94e94dcc10..13bc45e0693 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp @@ -468,6 +468,13 @@ int64_t WiredTigerUtil::getIdentSize(WT_SESSION* s, const std::string& uri) { return result.getValue(); } +int64_t WiredTigerUtil::getIdentReuseSize(WT_SESSION* s, const std::string& uri) { + auto result = WiredTigerUtil::getStatisticsValue( + s, "statistics:" + uri, "statistics=(fast)", WT_STAT_DSRC_BLOCK_REUSE_BYTES); + uassertStatusOK(result.getStatus()); + return result.getValue(); +} + size_t WiredTigerUtil::getCacheSizeMB(double requestedCacheSizeGB) { double cacheSizeMB; const double kMaxSizeCacheMB = 10 * 1000 * 1000; diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_util.h b/src/mongo/db/storage/wiredtiger/wiredtiger_util.h index 5150c19e153..57a8d7189fc 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_util.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_util.h @@ -220,6 +220,12 @@ public: static int64_t getIdentSize(WT_SESSION* s, const std::string& uri); + /** + * Returns the bytes available for reuse for an ident. This is the amount of allocated space on + * disk that is not storing any data. + */ + static int64_t getIdentReuseSize(WT_SESSION* s, const std::string& uri); + /** * Return amount of memory to use for the WiredTiger cache based on either the startup |