SERVER-15614 Lock statistics for db.serverStatus.locks

14 files changed, 529 insertions, 55 deletions

diff --git a/src/mongo/db/concurrency/SConscript b/src/mongo/db/concurrency/SConscript
index 9ff84ca730d..1687fa6ce04 100644
--- a/src/mongo/db/concurrency/SConscript
+++ b/src/mongo/db/concurrency/SConscript
@@ -18,6 +18,7 @@ env.Library(
         'd_concurrency.cpp',
         'lock_manager.cpp',
         'lock_state.cpp',
+        'lock_stats.cpp',
     ],
     LIBDEPS=[
         '$BUILD_DIR/mongo/base/base',
@@ -36,6 +37,7 @@ env.CppUnitTest(
             'fast_map_noalloc_test.cpp',
             'lock_manager_test.cpp',
             'lock_state_test.cpp',
+            'lock_stats_test.cpp',
     ],
     LIBDEPS=[
         'lock_manager'
diff --git a/src/mongo/db/concurrency/d_concurrency.cpp b/src/mongo/db/concurrency/d_concurrency.cpp
index d5a09dd7193..5898187bca6 100644
--- a/src/mongo/db/concurrency/d_concurrency.cpp
+++ b/src/mongo/db/concurrency/d_concurrency.cpp
@@ -51,9 +51,6 @@ namespace mongo {
     //  SERVER-14668: Remove or invert sense once MMAPv1 CLL can be default
     MONGO_EXPORT_STARTUP_SERVER_PARAMETER(enableCollectionLocking, bool, true);
 
-    // Local Oplog. Used by OplogLock.
-    static const ResourceId resourceIdOplog =
-                    ResourceId(RESOURCE_COLLECTION, StringData("local.oplog.rs"));
 
     DBTryLockTimeoutException::DBTryLockTimeoutException() {}
     DBTryLockTimeoutException::~DBTryLockTimeoutException() throw() { }
diff --git a/src/mongo/db/concurrency/deadlock_detection_test.cpp b/src/mongo/db/concurrency/deadlock_detection_test.cpp
index 9d05f350eb7..87274255635 100644
--- a/src/mongo/db/concurrency/deadlock_detection_test.cpp
+++ b/src/mongo/db/concurrency/deadlock_detection_test.cpp
@@ -34,8 +34,8 @@ namespace mongo {
     TEST(Deadlock, NoDeadlock) {
         const ResourceId resId(RESOURCE_DATABASE, std::string("A"));
 
-        LockerForTests locker1;
-        LockerForTests locker2;
+        LockerForTests locker1(MODE_IS);
+        LockerForTests locker2(MODE_IS);
 
         ASSERT_EQUALS(LOCK_OK, locker1.lockBegin(resId, MODE_S));
         ASSERT_EQUALS(LOCK_OK, locker2.lockBegin(resId, MODE_S));
@@ -51,8 +51,8 @@ namespace mongo {
         const ResourceId resIdA(RESOURCE_DATABASE, std::string("A"));
         const ResourceId resIdB(RESOURCE_DATABASE, std::string("B"));
 
-        LockerForTests locker1;
-        LockerForTests locker2;
+        LockerForTests locker1(MODE_IX);
+        LockerForTests locker2(MODE_IX);
 
         ASSERT_EQUALS(LOCK_OK, locker1.lockBegin(resIdA, MODE_X));
         ASSERT_EQUALS(LOCK_OK, locker2.lockBegin(resIdB, MODE_X));
@@ -77,8 +77,8 @@ namespace mongo {
     TEST(Deadlock, SimpleUpgrade) {
         const ResourceId resId(RESOURCE_DATABASE, std::string("A"));
 
-        LockerForTests locker1;
-        LockerForTests locker2;
+        LockerForTests locker1(MODE_IX);
+        LockerForTests locker2(MODE_IX);
 
         // Both acquire lock in intent mode
         ASSERT_EQUALS(LOCK_OK, locker1.lockBegin(resId, MODE_IX));
@@ -103,9 +103,9 @@ namespace mongo {
         const ResourceId resIdA(RESOURCE_DATABASE, std::string("A"));
         const ResourceId resIdB(RESOURCE_DATABASE, std::string("B"));
 
-        LockerForTests locker1;
-        LockerForTests locker2;
-        LockerForTests lockerIndirect;
+        LockerForTests locker1(MODE_IX);
+        LockerForTests locker2(MODE_IX);
+        LockerForTests lockerIndirect(MODE_IX);
 
         ASSERT_EQUALS(LOCK_OK, locker1.lockBegin(resIdA, MODE_X));
         ASSERT_EQUALS(LOCK_OK, locker2.lockBegin(resIdB, MODE_X));
@@ -138,9 +138,9 @@ namespace mongo {
         const ResourceId resIdFlush(RESOURCE_MMAPV1_FLUSH, 1);
         const ResourceId resIdDb(RESOURCE_DATABASE, 2);
 
-        LockerForTests flush;
-        LockerForTests reader;
-        LockerForTests writer;
+        LockerForTests flush(MODE_IX);
+        LockerForTests reader(MODE_IS);
+        LockerForTests writer(MODE_IX);
 
         // This sequence simulates the deadlock which occurs during flush
         ASSERT_EQUALS(LOCK_OK, writer.lockBegin(resIdFlush, MODE_IX));
diff --git a/src/mongo/db/concurrency/lock_manager_defs.h b/src/mongo/db/concurrency/lock_manager_defs.h
index 7c93ee55dbd..77bec0d433f 100644
--- a/src/mongo/db/concurrency/lock_manager_defs.h
+++ b/src/mongo/db/concurrency/lock_manager_defs.h
@@ -212,6 +212,11 @@ namespace mongo {
     // Type to uniquely identify a given locker object
     typedef uint64_t LockerId;
 
+    // Hardcoded resource id for the oplog collection, which is special-cased both for resource
+    // acquisition purposes and for statistics reporting.
+    extern const ResourceId resourceIdLocalDB;
+    extern const ResourceId resourceIdOplog;
+
 
     /**
      * Interface on which granted lock requests will be notified. See the contract for the notify
diff --git a/src/mongo/db/concurrency/lock_manager_test_help.h b/src/mongo/db/concurrency/lock_manager_test_help.h
index e76b5e51606..1650ca0806a 100644
--- a/src/mongo/db/concurrency/lock_manager_test_help.h
+++ b/src/mongo/db/concurrency/lock_manager_test_help.h
@@ -35,8 +35,8 @@ namespace mongo {
 
     class LockerForTests : public LockerImpl<false> {
     public:
-        explicit LockerForTests() {
-            lockGlobal(MODE_S);
+        explicit LockerForTests(LockMode globalLockMode) {
+            lockGlobal(globalLockMode);
         }
 
         ~LockerForTests() {
diff --git a/src/mongo/db/concurrency/lock_state.cpp b/src/mongo/db/concurrency/lock_state.cpp
index 970b9860ca7..86d8870a039 100644
--- a/src/mongo/db/concurrency/lock_state.cpp
+++ b/src/mongo/db/concurrency/lock_state.cpp
@@ -32,17 +32,76 @@
 
 #include "mongo/db/concurrency/lock_state.h"
 
-#include "mongo/bson/bsonobjbuilder.h"
+#include "mongo/db/concurrency/lock_stats.h"
 #include "mongo/db/global_environment_experiment.h"
 #include "mongo/db/namespace_string.h"
 #include "mongo/util/log.h"
 #include "mongo/util/mongoutils/str.h"
-#include "mongo/util/stacktrace.h"
-#include "mongo/util/timer.h"
 
 namespace mongo {
 namespace {
 
+    /**
+     * Partitioned global lock statistics, so we don't hit the same bucket.
+     */
+    class PartitionedInstanceWideLockStats {
+        MONGO_DISALLOW_COPYING(PartitionedInstanceWideLockStats);
+    public:
+
+        PartitionedInstanceWideLockStats() { }
+
+        void recordAcquisition(LockerId id, ResourceId resId, LockMode mode) {
+            LockStats& stats = _get(id);
+            stats.recordAcquisition(resId, mode);
+        }
+
+        void recordWait(LockerId id, ResourceId resId, LockMode mode) {
+            LockStats& stats = _get(id);
+            stats.recordWait(resId, mode);
+        }
+
+        void recordWaitTime(LockerId id, ResourceId resId, LockMode mode, uint64_t waitMicros) {
+            LockStats& stats = _get(id);
+            stats.recordWaitTime(resId, mode, waitMicros);
+        }
+
+        void report(LockStats* outStats) const {
+            for (int i = 0; i < NumPartitions; i++) {
+                outStats->append(_partitions[i]);
+            }
+        }
+
+        void reset() {
+            for (int i = 0; i < NumPartitions; i++) {
+                _partitions[i].reset();
+            }
+        }
+
+    private:
+
+        enum { NumPartitions = 8 };
+
+        LockStats& _get(LockerId id) {
+            return _partitions[id % NumPartitions];
+        }
+
+
+        LockStats _partitions[NumPartitions];
+    };
+
+
+    /**
+     * Used to sort locks by granularity when snapshotting lock state. We must report and reacquire
+     * locks in the same granularity in which they are acquired (i.e. global, flush, database,
+     * collection, etc).
+     */
+    struct SortByGranularity {
+        inline bool operator()(const Locker::OneLock& lhs, const Locker::OneLock& rhs) const {
+            return lhs.resourceId.getType() < rhs.resourceId.getType();
+        }
+    };
+
+
     // Global lock manager instance.
     LockManager globalLockManager;
 
@@ -61,16 +120,9 @@ namespace {
     // Dispenses unique LockerId identifiers
     AtomicUInt64 idCounter(0);
 
-    /**
-     * Used to sort locks by granularity when snapshotting lock state. We must report and reacquire
-     * locks in the same granularity in which they are acquired (i.e. global, flush, database,
-     * collection, etc).
-     */
-    struct SortByGranularity {
-        inline bool operator()(const Locker::OneLock& lhs, const Locker::OneLock& rhs) const {
-            return lhs.resourceId.getType() < rhs.resourceId.getType();
-        }
-    };
+    // Partitioned global lock statistics, so we don't hit the same bucket
+    PartitionedInstanceWideLockStats globalStats;
+
 
     /**
      * Returns whether the passed in mode is S or IS. Used for validation checks.
@@ -211,6 +263,7 @@ namespace {
     template<bool IsForMMAPV1>
     LockerImpl<IsForMMAPV1>::LockerImpl()
         : _id(idCounter.addAndFetch(1)),
+          _requestStartTime(0),
           _wuowNestingLevel(0),
           _batchWriter(false),
           _lockPendingParallelWriter(false) {
@@ -253,7 +306,7 @@ namespace {
 
     template<bool IsForMMAPV1>
     LockResult LockerImpl<IsForMMAPV1>::lockGlobalComplete(unsigned timeoutMs) {
-        return lockComplete(resourceIdGlobal, timeoutMs, false);
+        return lockComplete(resourceIdGlobal, getLockMode(resourceIdGlobal), timeoutMs, false);
     }
 
     template<bool IsForMMAPV1>
@@ -349,7 +402,7 @@ namespace {
         // unsuccessful result that the lock manager would return is LOCK_WAITING.
         invariant(result == LOCK_WAITING);
 
-        return lockComplete(resId, timeoutMs, checkDeadlock);
+        return lockComplete(resId, mode, timeoutMs, checkDeadlock);
     }
 
     template<bool IsForMMAPV1>
@@ -560,6 +613,9 @@ namespace {
             isNew = false;
         }
 
+        // Making this call here will record lock re-acquisitions and conversions as well.
+        globalStats.recordAcquisition(_id, resId, mode);
+
         // Give priority to the full modes for global and flush lock so we don't stall global
         // operations such as shutdown or flush.
         if (resId == resourceIdGlobal || (IsForMMAPV1 && resId == resourceIdMMAPV1Flush)) {
@@ -573,20 +629,28 @@ namespace {
             invariant(getLockMode(resourceIdGlobal) != MODE_NONE);
         }
 
+        // The notification object must be cleared before we invoke the lock manager, because
+        // otherwise we might reset state if the lock becomes granted very fast.
         _notify.clear();
 
-        return isNew ? globalLockManager.lock(resId, request, mode) :
-                       globalLockManager.convert(resId, request, mode);
+        LockResult result = isNew ? globalLockManager.lock(resId, request, mode) :
+                                    globalLockManager.convert(resId, request, mode);
+
+        if (result == LOCK_WAITING) {
+            // Start counting the wait time so that lockComplete can update that metric
+            _requestStartTime = curTimeMicros64();
+            globalStats.recordWait(_id, resId, mode);
+        }
+
+        return result;
     }
 
     template<bool IsForMMAPV1>
     LockResult LockerImpl<IsForMMAPV1>::lockComplete(ResourceId resId,
+                                                     LockMode mode,
                                                      unsigned timeoutMs,
                                                      bool checkDeadlock) {
 
-        // Tracks the lock acquisition time if we don't obtain the lock immediately
-        Timer timer;
-
         // Under MMAP V1 engine a deadlock can occur if a thread goes to sleep waiting on
         // DB lock, while holding the flush lock, so it has to be released. This is only
         // correct to do if not in a write unit of work.
@@ -606,6 +670,10 @@ namespace {
         while (true) {
             result = _notify.wait(waitTimeMs);
 
+            // Account for the time spent waiting on the notification object
+            const uint64_t elapsedTimeMicros = curTimeMicros64() - _requestStartTime;
+            globalStats.recordWaitTime(_id, resId, mode, elapsedTimeMicros);
+
             if (result == LOCK_OK) break;
 
             if (checkDeadlock) {
@@ -618,7 +686,7 @@ namespace {
                 }
             }
 
-            const unsigned elapsedTimeMs = timer.millis();
+            const unsigned elapsedTimeMs = elapsedTimeMicros / 1000;
             waitTimeMs = (elapsedTimeMs < timeoutMs) ?
                 std::min(timeoutMs - elapsedTimeMs, DeadlockTimeoutMs) : 0;
 
@@ -740,10 +808,23 @@ namespace {
         return &globalLockManager;
     }
 
+    void reportGlobalLockingStats(LockStats* outStats) {
+        globalStats.report(outStats);
+    }
+
+    void resetGlobalLockStats() {
+        globalStats.reset();
+    }
+
     
     // Ensures that there are two instances compiled for LockerImpl for the two values of the
     // template argument.
     template class LockerImpl<true>;
     template class LockerImpl<false>;
 
+    // Definition for the hardcoded localdb and oplog collection info
+    const ResourceId resourceIdLocalDB = ResourceId(RESOURCE_DATABASE, StringData("local"));
+    const ResourceId resourceIdOplog =
+        ResourceId(RESOURCE_COLLECTION, StringData("local.oplog.rs"));
+
 } // namespace mongo
diff --git a/src/mongo/db/concurrency/lock_state.h b/src/mongo/db/concurrency/lock_state.h
index a64dc7bcc0b..b99731c94e4 100644
--- a/src/mongo/db/concurrency/lock_state.h
+++ b/src/mongo/db/concurrency/lock_state.h
@@ -31,8 +31,8 @@
 #include <queue>
 
 #include "mongo/db/concurrency/fast_map_noalloc.h"
-#include "mongo/util/concurrency/spin_lock.h"
 #include "mongo/db/concurrency/locker.h"
+#include "mongo/util/concurrency/spin_lock.h"
 
 namespace mongo {
 
@@ -147,15 +147,25 @@ namespace mongo {
          *
          * In other words for each call to lockBegin, which does not return LOCK_OK, there needs to
          * be a corresponding call to either lockComplete or unlock.
+         *
+         * NOTE: These methods are not public and should only be used inside the class
+         * implementation and for unit-tests and not called directly.
          */
         LockResult lockBegin(ResourceId resId, LockMode mode);
 
         /**
          * Waits for the completion of a lock, previously requested through lockBegin or
-         * lockGlobalBegin. Must only be called, if lockBegin returned LOCK_WAITING. The resId
-         * argument must match what was previously passed to lockBegin.
+         * lockGlobalBegin. Must only be called, if lockBegin returned LOCK_WAITING.
+         *
+         * @param resId Resource id which was passed to an earlier lockBegin call. Must match.
+         * @param mode Mode which was passed to an earlier lockBegin call. Must match.
+         * @param timeoutMs How long to wait for the lock acquisition to complete.
+         * @param checkDeadlock whether to perform deadlock detection while waiting.
          */
-        LockResult lockComplete(ResourceId resId, unsigned timeoutMs, bool checkDeadlock);
+        LockResult lockComplete(ResourceId resId,
+                                LockMode mode,
+                                unsigned timeoutMs,
+                                bool checkDeadlock);
 
     private:
 
@@ -193,6 +203,10 @@ namespace mongo {
         // and condition variable every time.
         CondVarLockGrantNotification _notify;
 
+        // Timer for measuring duration and timeouts. This value is set when lock acquisition is
+        // about to wait and is sampled at grant time.
+        uint64_t _requestStartTime;
+
         // Delays release of exclusive/intent-exclusive locked resources until the write unit of
         // work completes. Value of 0 means we are not inside a write unit of work.
         int _wuowNestingLevel;
diff --git a/src/mongo/db/concurrency/lock_state_test.cpp b/src/mongo/db/concurrency/lock_state_test.cpp
index 6fb79500c42..7c4be4609fa 100644
--- a/src/mongo/db/concurrency/lock_state_test.cpp
+++ b/src/mongo/db/concurrency/lock_state_test.cpp
@@ -43,7 +43,6 @@ namespace {
     const int NUM_PERF_ITERS = 1000*1000; // numeber of iterations to use for lock perf
 }
 
-    using boost::shared_ptr;
 
     TEST(LockerImpl, LockNoConflict) {
         const ResourceId resId(RESOURCE_COLLECTION, std::string("TestDB.collection"));
@@ -289,7 +288,7 @@ namespace {
         for (int numLockers = 1; numLockers <= 64; numLockers = numLockers * 2) {
             std::vector<boost::shared_ptr<LockerForTests> > lockers(numLockers);
             for (int i = 0; i < numLockers; i++) {
-                lockers[i].reset(new LockerForTests());
+                lockers[i].reset(new LockerForTests(MODE_S));
             }
 
             DefaultLockerImpl locker;
diff --git a/src/mongo/db/concurrency/lock_stats.cpp b/src/mongo/db/concurrency/lock_stats.cpp
new file mode 100644
index 00000000000..ec6e597c608
--- /dev/null
+++ b/src/mongo/db/concurrency/lock_stats.cpp
@@ -0,0 +1,168 @@
+/**
+ *    Copyright (C) 2014 MongoDB Inc.
+ *
+ *    This program is free software: you can redistribute it and/or  modify
+ *    it under the terms of the GNU Affero General Public License, version 3,
+ *    as published by the Free Software Foundation.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Affero General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Affero General Public License
+ *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the GNU Affero General Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/bson/bsonobjbuilder.h"
+#include "mongo/db/concurrency/lock_stats.h"
+
+namespace mongo {
+
+    LockStats::LockStats() {
+
+    }
+
+    void LockStats::recordAcquisition(ResourceId resId, LockMode mode) {
+        PerModeAtomicLockStats& stat = get(resId);
+        stat.stats[mode].numAcquisitions.addAndFetch(1);
+    }
+
+    void LockStats::recordWait(ResourceId resId, LockMode mode) {
+        PerModeAtomicLockStats& stat = get(resId);
+        stat.stats[mode].numWaits.addAndFetch(1);
+    }
+
+    void LockStats::recordWaitTime(ResourceId resId, LockMode mode, uint64_t waitMicros) {
+        PerModeAtomicLockStats& stat = get(resId);
+        stat.stats[mode].combinedWaitTimeMicros.addAndFetch(waitMicros);
+    }
+
+    void LockStats::append(const LockStats& other) {
+        // Append all lock stats
+        for (int i = 0; i < ResourceTypesCount; i++) {
+            for (int mode = 0; mode < LockModesCount; mode++) {
+                const AtomicLockStats& otherStats = other._stats[i].stats[mode];
+
+                AtomicLockStats& thisStats = _stats[i].stats[mode];
+                thisStats.append(otherStats);
+            }
+        }
+
+        // Append the oplog stats
+        for (int mode = 0; mode < LockModesCount; mode++) {
+            const AtomicLockStats& otherStats = other._oplogStats.stats[mode];
+
+            AtomicLockStats& thisStats = _oplogStats.stats[mode];
+            thisStats.append(otherStats);
+        }
+    }
+
+    void LockStats::report(BSONObjBuilder* builder) const {
+        // All indexing below starts from offset 1, because we do not want to report/account
+        // position 0, which is a sentinel value for invalid resource/no lock.
+
+        for (int i = 1; i < ResourceTypesCount; i++) {
+            BSONObjBuilder resBuilder(builder->subobjStart(
+                                                resourceTypeName(static_cast<ResourceType>(i))));
+
+            _report(&resBuilder, _stats[i]);
+
+            resBuilder.done();
+        }
+
+        BSONObjBuilder resBuilder(builder->subobjStart("oplog"));
+        _report(&resBuilder, _oplogStats);
+        resBuilder.done();
+    }
+
+    void LockStats::_report(BSONObjBuilder* builder,
+                                        const PerModeAtomicLockStats& stat) const {
+
+        // All indexing below starts from offset 1, because we do not want to report/account
+        // position 0, which is a sentinel value for invalid resource/no lock.
+
+        // Num acquires
+        {
+            BSONObjBuilder numAcquires(builder->subobjStart("acquireCount"));
+            for (int mode = 1; mode < LockModesCount; mode++) {
+                numAcquires.append(legacyModeName(static_cast<LockMode>(mode)),
+                                   stat.stats[mode].numAcquisitions.load());
+            }
+            numAcquires.done();
+        }
+
+        // Num waits
+        {
+            BSONObjBuilder numWaits(builder->subobjStart("acquireWaitCount"));
+            for (int mode = 1; mode < LockModesCount; mode++) {
+                numWaits.append(legacyModeName(static_cast<LockMode>(mode)),
+                                stat.stats[mode].numWaits.load());
+            }
+            numWaits.done();
+        }
+
+        // Total time waiting
+        {
+            BSONObjBuilder timeAcquiring(builder->subobjStart("timeAcquiringMicros"));
+            for (int mode = 1; mode < LockModesCount; mode++) {
+                timeAcquiring.append(legacyModeName(static_cast<LockMode>(mode)),
+                                     stat.stats[mode].combinedWaitTimeMicros.load());
+            }
+            timeAcquiring.done();
+        }
+    }
+
+    void LockStats::reset() {
+        for (int i = 0; i < ResourceTypesCount; i++) {
+            for (int mode = 0; mode < LockModesCount; mode++) {
+                _stats[i].stats[mode].reset();
+            }
+        }
+
+        for (int mode = 0; mode < LockModesCount; mode++) {
+            _oplogStats.stats[mode].reset();
+        }
+    }
+
+    LockStats::PerModeAtomicLockStats& LockStats::get(ResourceId resId) {
+        if (resId == resourceIdOplog) {
+            return _oplogStats;
+        }
+        else {
+            return _stats[resId.getType()];
+        }
+    }
+
+
+    //
+    // AtomicLockStats
+    //
+
+    void LockStats::AtomicLockStats::append(const AtomicLockStats& other) {
+        numAcquisitions.addAndFetch(other.numAcquisitions.load());
+        numWaits.addAndFetch(other.numWaits.load());
+        combinedWaitTimeMicros.addAndFetch(other.combinedWaitTimeMicros.load());
+    }
+
+    void LockStats::AtomicLockStats::reset() {
+        numAcquisitions.store(0);
+        numWaits.store(0);
+        combinedWaitTimeMicros.store(0);
+    }
+
+} // namespace mongo
diff --git a/src/mongo/db/concurrency/lock_stats.h b/src/mongo/db/concurrency/lock_stats.h
new file mode 100644
index 00000000000..ac4f53dda3d
--- /dev/null
+++ b/src/mongo/db/concurrency/lock_stats.h
@@ -0,0 +1,97 @@
+/**
+ *    Copyright (C) 2014 MongoDB Inc.
+ *
+ *    This program is free software: you can redistribute it and/or  modify
+ *    it under the terms of the GNU Affero General Public License, version 3,
+ *    as published by the Free Software Foundation.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Affero General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Affero General Public License
+ *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the GNU Affero General Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#pragma once
+
+#include "mongo/db/concurrency/lock_manager_defs.h"
+#include "mongo/platform/atomic_word.h"
+
+namespace mongo {
+
+    class BSONObjBuilder;
+
+    class LockStats {
+    public:
+
+        /**
+         * Locking statistics for the top level locks.
+         */
+        struct AtomicLockStats {
+            AtomicLockStats() {
+                reset();
+            }
+
+            void append(const AtomicLockStats& other);
+            void reset();
+
+            AtomicInt64 numAcquisitions;
+            AtomicInt64 numWaits;
+            AtomicInt64 combinedWaitTimeMicros;
+        };
+
+        // Keep the per-mode lock stats next to each other in case we want to do fancy operations
+        // such as atomic operations on 128-bit values.
+        struct PerModeAtomicLockStats {
+            AtomicLockStats stats[LockModesCount];
+        };
+
+
+        LockStats();
+
+        void recordAcquisition(ResourceId resId, LockMode mode);
+        void recordWait(ResourceId resId, LockMode mode);
+        void recordWaitTime(ResourceId resId, LockMode mode, uint64_t waitMicros);
+
+        PerModeAtomicLockStats& get(ResourceId resId);
+
+        void append(const LockStats& other);
+        void report(BSONObjBuilder* builder) const;
+        void reset();
+
+    private:
+
+        void _report(BSONObjBuilder* builder, const PerModeAtomicLockStats& stat) const;
+
+        // Split the lock stats per resource type and special-case the oplog so we can collect
+        // more detailed stats for it.
+        PerModeAtomicLockStats _stats[ResourceTypesCount];
+        PerModeAtomicLockStats _oplogStats;
+    };
+
+
+    /**
+     * Reports instance-wide locking statistics, which can then be converted to BSON or logged.
+     */
+    void reportGlobalLockingStats(LockStats* outStats);
+
+    /**
+     * Currently used for testing only.
+     */
+    void resetGlobalLockStats();
+
+} // namespace mongo
diff --git a/src/mongo/db/concurrency/lock_stats_test.cpp b/src/mongo/db/concurrency/lock_stats_test.cpp
new file mode 100644
index 00000000000..7b1a2e0b384
--- /dev/null
+++ b/src/mongo/db/concurrency/lock_stats_test.cpp
@@ -0,0 +1,103 @@
+/**
+ *    Copyright (C) 2014 MongoDB Inc.
+ *
+ *    This program is free software: you can redistribute it and/or  modify
+ *    it under the terms of the GNU Affero General Public License, version 3,
+ *    as published by the Free Software Foundation.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Affero General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Affero General Public License
+ *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the GNU Affero General Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/bson/bsonobjbuilder.h"
+#include "mongo/db/concurrency/lock_manager_test_help.h"
+#include "mongo/db/concurrency/lock_stats.h"
+#include "mongo/unittest/unittest.h"
+
+namespace mongo {
+
+    TEST(LockStats, NoWait) {
+        const ResourceId resId(RESOURCE_COLLECTION, std::string("LockStats.NoWait"));
+
+        resetGlobalLockStats();
+
+        LockerForTests locker(MODE_IX);
+        locker.lock(resId, MODE_X);
+        locker.unlock(resId);
+
+        // Make sure that the waits/blocks are zero
+        LockStats stats;
+        reportGlobalLockingStats(&stats);
+
+        ASSERT_EQUALS(1, stats.get(resId).stats[MODE_X].numAcquisitions.load());
+        ASSERT_EQUALS(0, stats.get(resId).stats[MODE_X].numWaits.load());
+        ASSERT_EQUALS(0, stats.get(resId).stats[MODE_X].combinedWaitTimeMicros.load());
+    }
+
+    TEST(LockStats, Wait) {
+        const ResourceId resId(RESOURCE_COLLECTION, std::string("LockStats.Wait"));
+
+        resetGlobalLockStats();
+
+        LockerForTests locker(MODE_IX);
+        locker.lock(resId, MODE_X);
+
+        {
+            // This will block
+            LockerForTests lockerConflict(MODE_IX);
+            ASSERT_EQUALS(LOCK_WAITING, lockerConflict.lockBegin(resId, MODE_S));
+
+            // Sleep 1 millisecond so the wait time passes
+            ASSERT_EQUALS(LOCK_TIMEOUT, lockerConflict.lockComplete(resId, MODE_S, 1, false));
+        }
+
+        // Make sure that the waits/blocks are non-zero
+        LockStats stats;
+        reportGlobalLockingStats(&stats);
+
+        ASSERT_EQUALS(1, stats.get(resId).stats[MODE_X].numAcquisitions.load());
+        ASSERT_EQUALS(0, stats.get(resId).stats[MODE_X].numWaits.load());
+        ASSERT_EQUALS(0, stats.get(resId).stats[MODE_X].combinedWaitTimeMicros.load());
+
+        ASSERT_EQUALS(1, stats.get(resId).stats[MODE_S].numAcquisitions.load());
+        ASSERT_EQUALS(1, stats.get(resId).stats[MODE_S].numWaits.load());
+        ASSERT_GREATER_THAN(stats.get(resId).stats[MODE_S].combinedWaitTimeMicros.load(), 0);
+    }
+
+    TEST(LockStats, Reporting) {
+        const ResourceId resId(RESOURCE_COLLECTION, std::string("LockStats.Reporting"));
+
+        resetGlobalLockStats();
+
+        LockerForTests locker(MODE_IX);
+        locker.lock(resId, MODE_X);
+        locker.unlock(resId);
+
+        // Make sure that the waits/blocks are zero
+        LockStats stats;
+        reportGlobalLockingStats(&stats);
+
+        BSONObjBuilder builder;
+        stats.report(&builder);
+    }
+
+} // namespace mongo
diff --git a/src/mongo/db/concurrency/locker.h b/src/mongo/db/concurrency/locker.h
index e95f98a6855..edbedfa4ddb 100644
--- a/src/mongo/db/concurrency/locker.h
+++ b/src/mongo/db/concurrency/locker.h
@@ -32,7 +32,6 @@
 #include <vector>
 
 #include "mongo/base/disallow_copying.h"
-#include "mongo/bson/bsonobj.h"
 #include "mongo/db/concurrency/lock_manager.h"
 
 namespace mongo {
diff --git a/src/mongo/db/stats/lock_server_status_section.cpp b/src/mongo/db/stats/lock_server_status_section.cpp
index a61e7e87e35..b2abd4ce89a 100644
--- a/src/mongo/db/stats/lock_server_status_section.cpp
+++ b/src/mongo/db/stats/lock_server_status_section.cpp
@@ -30,13 +30,16 @@
 
 #include "mongo/db/client.h"
 #include "mongo/db/commands/server_status.h"
+#include "mongo/db/concurrency/lock_stats.h"
+#include "mongo/db/jsobj.h"
 #include "mongo/db/operation_context.h"
 
 namespace mongo {
+namespace {
 
     class GlobalLockServerStatusSection : public ServerStatusSection {
     public:
-        GlobalLockServerStatusSection() : ServerStatusSection("globalLock"){
+        GlobalLockServerStatusSection() : ServerStatusSection("globalLock") {
             _started = curTimeMillis64();
         }
 
@@ -121,21 +124,23 @@ namespace mongo {
 
     class LockStatsServerStatusSection : public ServerStatusSection {
     public:
-        LockStatsServerStatusSection() : ServerStatusSection("locks"){}
+        LockStatsServerStatusSection() : ServerStatusSection("locks") { }
+
         virtual bool includeByDefault() const { return true; }
 
-        BSONObj generateSection(OperationContext* txn,
-                                const BSONElement& configElement) const {
+        virtual BSONObj generateSection(OperationContext* txn,
+                                        const BSONElement& configElement) const {
+            BSONObjBuilder ret;
 
-            BSONObjBuilder b;
+            LockStats stats;
+            reportGlobalLockingStats(&stats);
 
-            // SERVER-14978: Need to report the global and per-DB lock stats here
-            //
-            // b.append(".", qlk.stats.report());
+            stats.report(&ret);
 
-            return b.obj();
+            return ret.obj();
         }
 
     } lockStatsServerStatusSection;
 
+} // namespace
 } // namespace mongo
diff --git a/src/mongo/db/storage/mmap_v1/dur.cpp b/src/mongo/db/storage/mmap_v1/dur.cpp
index 4b1eccf67db..56f9f73d7a3 100644
--- a/src/mongo/db/storage/mmap_v1/dur.cpp
+++ b/src/mongo/db/storage/mmap_v1/dur.cpp
@@ -214,7 +214,7 @@ namespace {
            << _commits << '\t'
            << _journaledBytes / 1000000.0 << '\t'
            << _writeToDataFilesBytes / 1000000.0 << '\t'
-           << 0 << '\t'
+           << _commitsInWriteLock << '\t'
            << 0 << '\t'
            << (unsigned) (_prepLogBufferMicros / 1000) << '\t'
            << (unsigned) (_writeToJournalMicros / 1000) << '\t'
@@ -611,6 +611,8 @@ namespace {
                 // The commit logic itself
                 LOG(4) << "groupCommit begin";
 
+                Timer t;
+
                 OperationContextImpl txn;
                 AutoAcquireFlushLockForMMAPV1Commit autoFlushLock(txn.lockState());
 
@@ -688,6 +690,8 @@ namespace {
                 }
 
                 stats.curr()->_commits++;
+                stats.curr()->_commitsInWriteLock++;
+                stats.curr()->_commitsInWriteLockMicros += t.micros();
 
                 LOG(4) << "groupCommit end";
             }