diff options
| -rw-r--r-- | src/mongo/db/catalog/util/partitioned.h | 10 | ||||
| -rw-r--r-- | src/mongo/db/s/resharding/resharding_metrics.cpp | 8 | ||||
| -rw-r--r-- | src/mongo/db/stats/counters.cpp | 12 | ||||
| -rw-r--r-- | src/mongo/db/stats/counters.h | 77 | ||||
| -rw-r--r-- | src/mongo/util/aligned.h | 96 | ||||
| -rw-r--r-- | src/mongo/util/aligned_test.cpp | 6 |
6 files changed, 139 insertions, 70 deletions
diff --git a/src/mongo/db/catalog/util/partitioned.h b/src/mongo/db/catalog/util/partitioned.h index 380ea5e21a9..f7fa6ea5aa8 100644 --- a/src/mongo/db/catalog/util/partitioned.h +++ b/src/mongo/db/catalog/util/partitioned.h @@ -102,7 +102,7 @@ struct Partitioner { namespace partitioned_detail { -using CacheAlignedMutex = CacheAligned<stdx::mutex>; +using CacheExclusiveMutex = CacheExclusive<stdx::mutex>; template <typename Key, typename Value> Key getKey(const std::pair<Key, Value>& pair) { @@ -300,7 +300,7 @@ public: */ Partitioned(std::size_t nPartitions, const AssociativeContainer& container) : _mutexes(nPartitions), - _partitions(nPartitions, CacheAlignedAssociativeContainer(container)) { + _partitions(nPartitions, CacheExclusiveAssociativeContainer(container)) { invariant(nPartitions > 0); } @@ -383,14 +383,14 @@ public: } private: - using CacheAlignedAssociativeContainer = CacheAligned<AssociativeContainer>; + using CacheExclusiveAssociativeContainer = CacheExclusive<AssociativeContainer>; template <typename T> using AlignedVector = std::vector<T, boost::alignment::aligned_allocator<T>>; // These two vectors parallel each other, but we keep them separate so that we can return an // iterator over `_partitions` from within All. - mutable AlignedVector<partitioned_detail::CacheAlignedMutex> _mutexes; - AlignedVector<CacheAlignedAssociativeContainer> _partitions; + mutable AlignedVector<partitioned_detail::CacheExclusiveMutex> _mutexes; + AlignedVector<CacheExclusiveAssociativeContainer> _partitions; }; } // namespace mongo diff --git a/src/mongo/db/s/resharding/resharding_metrics.cpp b/src/mongo/db/s/resharding/resharding_metrics.cpp index 6a11be7e88f..627f36a89e4 100644 --- a/src/mongo/db/s/resharding/resharding_metrics.cpp +++ b/src/mongo/db/s/resharding/resharding_metrics.cpp @@ -124,7 +124,7 @@ public: private: // Increment member `counter` by n, resetting all counters if it was > 2^60. - void _checkWrap(CacheAligned<AtomicWord<long long>> ReshardingOpCounters::*counter, int n) { + void _checkWrap(CacheExclusive<AtomicWord<long long>> ReshardingOpCounters::*counter, int n) { static constexpr auto maxCount = 1LL << 60; auto oldValue = (this->*counter)->fetchAndAddRelaxed(n); if (oldValue > maxCount - n) { @@ -139,9 +139,9 @@ private: } } - CacheAligned<AtomicWord<long long>> _insert; - CacheAligned<AtomicWord<long long>> _update; - CacheAligned<AtomicWord<long long>> _delete; + CacheExclusive<AtomicWord<long long>> _insert; + CacheExclusive<AtomicWord<long long>> _update; + CacheExclusive<AtomicWord<long long>> _delete; }; // Allows tracking elapsed time for the resharding operation and its sub operations (e.g., diff --git a/src/mongo/db/stats/counters.cpp b/src/mongo/db/stats/counters.cpp index f8bead3f388..eb436737a4a 100644 --- a/src/mongo/db/stats/counters.cpp +++ b/src/mongo/db/stats/counters.cpp @@ -47,7 +47,7 @@ namespace { using namespace fmt::literals; } -void OpCounters::_checkWrap(CacheAligned<AtomicWord<long long>> OpCounters::*counter, int n) { +void OpCounters::_checkWrap(CacheExclusive<AtomicWord<long long>> OpCounters::*counter, int n) { static constexpr auto maxCount = 1LL << 60; auto oldValue = (this->*counter)->fetchAndAddRelaxed(n); if (oldValue > maxCount) { @@ -189,7 +189,7 @@ void NetworkCounter::incrementNumSlowSSLOperations() { } void NetworkCounter::acceptedTFOIngress() { - _tfo->accepted.fetchAndAddRelaxed(1); + _tfoAccepted->fetchAndAddRelaxed(1); } void NetworkCounter::append(BSONObjBuilder& b) { @@ -203,11 +203,11 @@ void NetworkCounter::append(BSONObjBuilder& b) { BSONObjBuilder tfo; #ifdef __linux__ - tfo.append("kernelSetting", _tfo->kernelSetting); + tfo.append("kernelSetting", _tfoKernelSetting); #endif - tfo.append("serverSupported", _tfo->kernelSupportServer); - tfo.append("clientSupported", _tfo->kernelSupportClient); - tfo.append("accepted", _tfo->accepted.loadRelaxed()); + tfo.append("serverSupported", _tfoKernelSupportServer); + tfo.append("clientSupported", _tfoKernelSupportClient); + tfo.append("accepted", _tfoAccepted->loadRelaxed()); b.append("tcpFastOpen", tfo.obj()); } diff --git a/src/mongo/db/stats/counters.h b/src/mongo/db/stats/counters.h index 2e07b64fe3c..cb5915b7626 100644 --- a/src/mongo/db/stats/counters.h +++ b/src/mongo/db/stats/counters.h @@ -150,28 +150,28 @@ public: private: // Increment member `counter` by `n`, resetting all counters if it was > 2^60. - void _checkWrap(CacheAligned<AtomicWord<long long>> OpCounters::*counter, int n); + void _checkWrap(CacheExclusive<AtomicWord<long long>> OpCounters::*counter, int n); - CacheAligned<AtomicWord<long long>> _insert; - CacheAligned<AtomicWord<long long>> _query; - CacheAligned<AtomicWord<long long>> _update; - CacheAligned<AtomicWord<long long>> _delete; - CacheAligned<AtomicWord<long long>> _getmore; - CacheAligned<AtomicWord<long long>> _command; + CacheExclusive<AtomicWord<long long>> _insert; + CacheExclusive<AtomicWord<long long>> _query; + CacheExclusive<AtomicWord<long long>> _update; + CacheExclusive<AtomicWord<long long>> _delete; + CacheExclusive<AtomicWord<long long>> _getmore; + CacheExclusive<AtomicWord<long long>> _command; - CacheAligned<AtomicWord<long long>> _insertOnExistingDoc; - CacheAligned<AtomicWord<long long>> _updateOnMissingDoc; - CacheAligned<AtomicWord<long long>> _deleteWasEmpty; - CacheAligned<AtomicWord<long long>> _deleteFromMissingNamespace; - CacheAligned<AtomicWord<long long>> _acceptableErrorInCommand; + CacheExclusive<AtomicWord<long long>> _insertOnExistingDoc; + CacheExclusive<AtomicWord<long long>> _updateOnMissingDoc; + CacheExclusive<AtomicWord<long long>> _deleteWasEmpty; + CacheExclusive<AtomicWord<long long>> _deleteFromMissingNamespace; + CacheExclusive<AtomicWord<long long>> _acceptableErrorInCommand; // Counters for deprecated opcodes. - CacheAligned<AtomicWord<long long>> _insertDeprecated; - CacheAligned<AtomicWord<long long>> _queryDeprecated; - CacheAligned<AtomicWord<long long>> _updateDeprecated; - CacheAligned<AtomicWord<long long>> _deleteDeprecated; - CacheAligned<AtomicWord<long long>> _getmoreDeprecated; - CacheAligned<AtomicWord<long long>> _killcursorsDeprecated; + CacheExclusive<AtomicWord<long long>> _insertDeprecated; + CacheExclusive<AtomicWord<long long>> _queryDeprecated; + CacheExclusive<AtomicWord<long long>> _updateDeprecated; + CacheExclusive<AtomicWord<long long>> _deleteDeprecated; + CacheExclusive<AtomicWord<long long>> _getmoreDeprecated; + CacheExclusive<AtomicWord<long long>> _killcursorsDeprecated; }; extern OpCounters globalOpCounters; @@ -198,48 +198,47 @@ public: void acceptedTFOIngress(); void setTFOKernelSetting(std::int64_t val) { - _tfo->kernelSetting = val; + _tfoKernelSetting = val; } void setTFOServerSupport(bool val) { - _tfo->kernelSupportServer = val; + _tfoKernelSupportServer = val; } void setTFOClientSupport(bool val) { - _tfo->kernelSupportClient = val; + _tfoKernelSupportClient = val; } void append(BSONObjBuilder& b); private: - CacheAligned<AtomicWord<long long>> _physicalBytesIn{0}; - CacheAligned<AtomicWord<long long>> _physicalBytesOut{0}; + CacheExclusive<AtomicWord<long long>> _physicalBytesIn{0}; + CacheExclusive<AtomicWord<long long>> _physicalBytesOut{0}; // These two counters are always incremented at the same time, so - // we place them on the same cache line. + // we place them on the same cache line. We use + // CacheCombinedExclusive to ensure that they are combined within + // the scope of a constructive interference region, and protected + // from false sharing by padding out to destructive interference + // size. struct Together { AtomicWord<long long> logicalBytesIn{0}; AtomicWord<long long> requests{0}; }; - CacheAligned<Together> _together{}; - static_assert(sizeof(decltype(_together)) <= stdx::hardware_constructive_interference_size, - "cache line spill"); + CacheCombinedExclusive<Together> _together{}; - CacheAligned<AtomicWord<long long>> _logicalBytesOut{0}; + CacheExclusive<AtomicWord<long long>> _logicalBytesOut{0}; - CacheAligned<AtomicWord<long long>> _numSlowDNSOperations{0}; - CacheAligned<AtomicWord<long long>> _numSlowSSLOperations{0}; + CacheExclusive<AtomicWord<long long>> _numSlowDNSOperations{0}; + CacheExclusive<AtomicWord<long long>> _numSlowSSLOperations{0}; - struct TFO { - // Counter of inbound connections at runtime. - AtomicWord<std::int64_t> accepted{0}; + // Counter of inbound connections at runtime. + CacheExclusive<AtomicWord<std::int64_t>> _tfoAccepted{0}; - // Info determined at startup. - std::int64_t kernelSetting; - bool kernelSupportServer{false}; - bool kernelSupportClient{false}; - }; - CacheAligned<TFO> _tfo{}; + // TFO info determined at startup. + std::int64_t _tfoKernelSetting; + bool _tfoKernelSupportServer{false}; + bool _tfoKernelSupportClient{false}; }; extern NetworkCounter networkCounter; diff --git a/src/mongo/util/aligned.h b/src/mongo/util/aligned.h index bdab7f8cac1..3521202834e 100644 --- a/src/mongo/util/aligned.h +++ b/src/mongo/util/aligned.h @@ -37,16 +37,32 @@ namespace mongo { +namespace aligned_detail { +template <size_t step> +constexpr size_t roundUpByStep(size_t value) { + return (((value + (step - 1)) / step) * step); +} +} // namespace aligned_detail + /** - * A wrapper holding a `T` value aligned to `alignof(T)` or `MinAlign`, - * whichever is greater (i.e. more strict). The value is accessed with a - * pointer-like syntax. + * A wrapper holding a `T` value aligned to `alignof(T)` or + * `minAlign`, whichever is greater (i.e. more strict). The value is + * accessed with a pointer-like syntax. Additionally the object will + * be placed in a buffer no smaller than minStorageSize, of which the + * contained T object may use no more than maxObjectSize bytes. */ -template <typename T, size_t MinAlign> +template <typename T, + size_t minAlign = alignof(T), + size_t minStorageSize = sizeof(T), + size_t maxObjectSize = minStorageSize> class Aligned { public: using value_type = T; - static constexpr size_t alignment = std::max(alignof(T), MinAlign); + + static_assert(sizeof(value_type) <= maxObjectSize); + + static constexpr size_t kAlignment = std::max(alignof(value_type), minAlign); + static constexpr size_t kStorageSize = std::max(sizeof(value_type), minStorageSize); template <typename... As> explicit Aligned(As&&... args) noexcept(std::is_nothrow_constructible_v<value_type, As&&...>) { @@ -99,7 +115,7 @@ public: } private: - static_assert((MinAlign & (MinAlign - 1)) == 0, "alignments must be a power of two"); + static_assert((minAlign & (minAlign - 1)) == 0, "alignments must be a power of two"); const value_type* _raw() const noexcept { return reinterpret_cast<const value_type*>(&_storage); @@ -117,21 +133,75 @@ private: return *stdx::launder(_raw()); } - std::aligned_storage_t<sizeof(value_type), alignment> _storage; + std::aligned_storage_t<kStorageSize, kAlignment> _storage; }; /** - * Swap the values. Should not necessarily require they agreen on alignment. - * defined out-of-class to work around https://gcc.gnu.org/bugzilla/show_bug.cgi?id=89612 + * Swap the values. Defined out-of-class to work around + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=89612 + * + * TODO: It should be possible to swap Aligned<T> vs Aligned<U> if T + * and U can be swapped, as well to swap instances with varying + * alignment, padding, etc. However, defining that generic swap + * results in ambiguities. */ -template <typename T, size_t MinAlign> -void swap(Aligned<T, MinAlign>& a, - Aligned<T, MinAlign>& b) noexcept(std::is_nothrow_swappable_v<T>) { +template <typename T, auto... Pack> +void swap(Aligned<T, Pack...>& a, Aligned<T, Pack...>& b) noexcept(std::is_nothrow_swappable_v<T>) { using std::swap; swap(*a, *b); } +/** + * A `CacheExclusive` object is Aligned to the destructive + * interference size, ensuring that it will start at an address that + * will not exhibit false sharing with any objects that precede it in + * memory. Additionally, the storage for the object is padded to a + * sufficient multiple of the destructive interference size to ensure + * that it will not exhibit false sharing with any other objects that + * follow it in memory. However, it is not assured that the embedded T + * object will internally exhibit true sharing with all of itself, as + * the contained object is permitted to be larger than the + * constructive interference size. + */ +template <typename T> +using CacheExclusive = + Aligned<T, + stdx::hardware_destructive_interference_size, + aligned_detail::roundUpByStep<stdx::hardware_destructive_interference_size>(sizeof(T))>; + + +/** + * A `CacheCombined` object is Aligned to the constructive + * interference size, ensuring that it will start at an address that + * can exhibit true sharing for some forward extent. Additionally, the + * size of the object is constrained to ensure that all of its state + * is eligible for true sharing within the same region of constructive + * interference. However, there is no guarantee that the object will + * not exhibit false sharing with objects that either precede or + * follow it in memory, unless those objects are themselves protected + * from false sharing. + */ +template <typename T> +using CacheCombined = Aligned<T, + stdx::hardware_constructive_interference_size, + sizeof(T), + stdx::hardware_constructive_interference_size>; + + +/** + * A `CacheCombinedExclusive` object is Aligned to the destructive + * interference size, ensuring that it will start at an address that + * will not exhibit false sharing with objects that precede it in + * memory. Additionally, the storage for the object is padded to the + * destructive interference size, ensuring that it will not exhibit + * false sharing with objects that follow it. Finally, the size of the + * object is constrained to the constructive interference size, + * ensuring that the object will internally exhibit true sharing. + */ template <typename T> -using CacheAligned = Aligned<T, stdx::hardware_destructive_interference_size>; +using CacheCombinedExclusive = Aligned<T, + stdx::hardware_destructive_interference_size, + stdx::hardware_destructive_interference_size, + stdx::hardware_constructive_interference_size>; } // namespace mongo diff --git a/src/mongo/util/aligned_test.cpp b/src/mongo/util/aligned_test.cpp index 2304877150c..f578ba75aee 100644 --- a/src/mongo/util/aligned_test.cpp +++ b/src/mongo/util/aligned_test.cpp @@ -104,10 +104,10 @@ TEST(Aligned, Swap) { ASSERT_EQ(m2->val, 111); } -TEST(CacheAligned, IsAlignedToPlatformCacheLine) { +TEST(CacheExclusive, IsAlignedToPlatformCacheLine) { static constexpr size_t a = stdx::hardware_destructive_interference_size; - MONGO_STATIC_ASSERT(alignof(CacheAligned<char>) == a); - MONGO_STATIC_ASSERT(alignof(CacheAligned<char>) == a); + MONGO_STATIC_ASSERT(alignof(CacheExclusive<char>) == a); + MONGO_STATIC_ASSERT(alignof(CacheExclusive<char>) == a); } } // namespace |