SERVER-27727 Hide idle threads in hang analyzer (core only)

(cherry picked from commit 27ddad2221974798284ef62d3328a3c02a510220)

8 files changed, 143 insertions, 3 deletions

diff --git a/buildscripts/gdb/mongo.py b/buildscripts/gdb/mongo.py
index 22420ef0ec2..a607f55c06e 100644
--- a/buildscripts/gdb/mongo.py
+++ b/buildscripts/gdb/mongo.py
@@ -387,6 +387,25 @@ class MongoDBDumpLocks(gdb.Command):
 # Register command
 MongoDBDumpLocks()
 
+class BtIfActive(gdb.Command):
+    """Print stack trace or a short message if the current thread is idle"""
+
+    def __init__(self):
+        register_mongo_command(self, "mongodb-bt-if-active", gdb.COMMAND_DATA)
+
+    def invoke(self, arg, _from_tty):
+        try:
+            is_idle = gdb.parse_and_eval("mongo::for_debuggers::threadIsIdle")
+        except gdb.error:
+            is_idle = False # If unsure, print a stack trace.
+
+        if is_idle:
+            print("Thread is idle")
+        else:
+            gdb.execute("bt")
+
+# Register command
+BtIfActive()
 
 class MongoDBUniqueStack(gdb.Command):
     """Print unique stack traces of all threads in current process"""
diff --git a/buildscripts/hang_analyzer.py b/buildscripts/hang_analyzer.py
index 1e88a26f833..b85442da742 100755
--- a/buildscripts/hang_analyzer.py
+++ b/buildscripts/hang_analyzer.py
@@ -349,7 +349,7 @@ class GDBDumper(object):
         source_mongo_lock = "source %s" % mongo_lock_script
         mongodb_dump_locks = "mongodb-dump-locks"
         mongodb_show_locks = "mongodb-show-locks"
-        mongodb_uniqstack = "mongodb-uniqstack bt"
+        mongodb_uniqstack = "mongodb-uniqstack mongodb-bt-if-active"
         mongodb_waitsfor_graph = "mongodb-waitsfor-graph debugger_waitsfor_%s_%d.gv" % \
             (process_name, pid)
         mongodb_javascript_stack = "mongodb-javascript-stack"
diff --git a/src/mongo/SConscript b/src/mongo/SConscript
index 62c17a7ecd1..2cfb2060e7f 100644
--- a/src/mongo/SConscript
+++ b/src/mongo/SConscript
@@ -109,6 +109,7 @@ baseSource=[
     'util/allocator.cpp',
     'util/assert_util.cpp',
     'util/base64.cpp',
+    'util/concurrency/idle_thread_block.cpp',
     'util/concurrency/thread_name.cpp',
     'util/duration.cpp',
     'util/exception_filter_win32.cpp',
diff --git a/src/mongo/db/service_entry_point_mongod.cpp b/src/mongo/db/service_entry_point_mongod.cpp
index ac00e12d9ac..4f89b7cdb38 100644
--- a/src/mongo/db/service_entry_point_mongod.cpp
+++ b/src/mongo/db/service_entry_point_mongod.cpp
@@ -42,6 +42,7 @@
 #include "mongo/transport/session.h"
 #include "mongo/transport/ticket.h"
 #include "mongo/transport/transport_layer.h"
+#include "mongo/util/concurrency/idle_thread_block.h"
 #include "mongo/util/exit.h"
 #include "mongo/util/log.h"
 #include "mongo/util/net/message.h"
@@ -113,7 +114,10 @@ void ServiceEntryPointMongod::_sessionLoop(const transport::SessionHandle& sessi
         // 1. Source a Message from the client (unless we are exhausting)
         if (!inExhaust) {
             inMessage.reset();
-            auto status = session->sourceMessage(&inMessage).wait();
+            auto status = [&] {
+                IdleThreadBlock markIdle;
+                return session->sourceMessage(&inMessage).wait();
+            }();
 
             if (ErrorCodes::isInterruption(status.code()) ||
                 ErrorCodes::isNetworkError(status.code())) {
diff --git a/src/mongo/s/service_entry_point_mongos.cpp b/src/mongo/s/service_entry_point_mongos.cpp
index 67ac08e9354..834875bc1b6 100644
--- a/src/mongo/s/service_entry_point_mongos.cpp
+++ b/src/mongo/s/service_entry_point_mongos.cpp
@@ -43,6 +43,7 @@
 #include "mongo/transport/service_entry_point_utils.h"
 #include "mongo/transport/session.h"
 #include "mongo/transport/transport_layer.h"
+#include "mongo/util/concurrency/idle_thread_block.h"
 #include "mongo/util/log.h"
 #include "mongo/util/net/message.h"
 #include "mongo/util/net/thread_idle_callback.h"
@@ -85,7 +86,10 @@ void ServiceEntryPointMongos::_sessionLoop(const transport::SessionHandle& sessi
 
         // Source a Message from the client
         {
-            auto status = session->sourceMessage(&message).wait();
+            auto status = [&] {
+                IdleThreadBlock markIdle;
+                return session->sourceMessage(&message).wait();
+            }();
 
             if (ErrorCodes::isInterruption(status.code()) ||
                 ErrorCodes::isNetworkError(status.code())) {
diff --git a/src/mongo/util/concurrency/idle_thread_block.cpp b/src/mongo/util/concurrency/idle_thread_block.cpp
new file mode 100644
index 00000000000..783ab4184cf
--- /dev/null
+++ b/src/mongo/util/concurrency/idle_thread_block.cpp
@@ -0,0 +1,50 @@
+/*    Copyright 2017 MongoDB Inc.
+ *
+ *    This program is free software: you can redistribute it and/or  modify
+ *    it under the terms of the GNU Affero General Public License, version 3,
+ *    as published by the Free Software Foundation.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Affero General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Affero General Public License
+ *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the GNU Affero General Public License in all respects
+ *    for all of the code used other than as permitted herein. If you modify
+ *    file(s) with this exception, you may extend this exception to your
+ *    version of the file(s), but you are not obligated to do so. If you do not
+ *    wish to do so, delete this exception statement from your version. If you
+ *    delete this exception statement from all source files in the program,
+ *    then also delete it in the license file.
+ */
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/util/assert_util.h"
+#include "mongo/util/concurrency/idle_thread_block.h"
+#include "mongo/util/concurrency/threadlocal.h"
+
+namespace mongo {
+namespace for_debuggers {
+// This needs external linkage to ensure that debuggers can use it.
+MONGO_TRIVIALLY_CONSTRUCTIBLE_THREAD_LOCAL bool threadIsIdle = false;
+}
+using for_debuggers::threadIsIdle;
+
+void IdleThreadBlock::beginIdleThreadBlock() {
+    invariant(!threadIsIdle);
+    threadIsIdle = true;
+}
+
+void IdleThreadBlock::endIdleThreadBlock() {
+    invariant(threadIsIdle);
+    threadIsIdle = false;
+}
+}
diff --git a/src/mongo/util/concurrency/idle_thread_block.h b/src/mongo/util/concurrency/idle_thread_block.h
new file mode 100644
index 00000000000..cfdcc35d277
--- /dev/null
+++ b/src/mongo/util/concurrency/idle_thread_block.h
@@ -0,0 +1,59 @@
+/*    Copyright 2017 MongoDB Inc.
+ *
+ *    This program is free software: you can redistribute it and/or  modify
+ *    it under the terms of the GNU Affero General Public License, version 3,
+ *    as published by the Free Software Foundation.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Affero General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Affero General Public License
+ *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the GNU Affero General Public License in all respects
+ *    for all of the code used other than as permitted herein. If you modify
+ *    file(s) with this exception, you may extend this exception to your
+ *    version of the file(s), but you are not obligated to do so. If you do not
+ *    wish to do so, delete this exception statement from your version. If you
+ *    delete this exception statement from all source files in the program,
+ *    then also delete it in the license file.
+ */
+
+#pragma once
+
+#include "mongo/base/disallow_copying.h"
+
+namespace mongo {
+
+/**
+ * Marks a thread as idle while in scope.
+ *
+ * Our debugger scripts can hide idle threads when dumping all stacks. You should mark threads as
+ * idle when printing the stack would just be unhelpful noise. IdleThreadBlocks are not allowed to
+ * nest. Each thread should generally have at most one possible place where it it is considered
+ * idle.
+ */
+class IdleThreadBlock {
+    MONGO_DISALLOW_COPYING(IdleThreadBlock);
+
+public:
+    IdleThreadBlock() {
+        beginIdleThreadBlock();
+    }
+    ~IdleThreadBlock() {
+        endIdleThreadBlock();
+    }
+
+    // These should not be called by mongo C++ code. They are only public to allow exposing this
+    // functionality to a C api.
+    static void beginIdleThreadBlock();
+    static void endIdleThreadBlock();
+};
+
+}  // namespace mongo
diff --git a/src/mongo/util/concurrency/thread_pool.cpp b/src/mongo/util/concurrency/thread_pool.cpp
index f5687eabf2f..5e0e7419adf 100644
--- a/src/mongo/util/concurrency/thread_pool.cpp
+++ b/src/mongo/util/concurrency/thread_pool.cpp
@@ -35,6 +35,7 @@
 #include "mongo/base/status.h"
 #include "mongo/platform/atomic_word.h"
 #include "mongo/util/assert_util.h"
+#include "mongo/util/concurrency/idle_thread_block.h"
 #include "mongo/util/concurrency/thread_name.h"
 #include "mongo/util/log.h"
 #include "mongo/util/mongoutils/str.h"
@@ -262,6 +263,7 @@ void ThreadPool::_consumeTasks() {
 
                 LOG(3) << "Not reaping because the earliest retirement date is "
                        << nextThreadRetirementDate;
+                IdleThreadBlock markIdle;
                 _workAvailable.wait_until(lk, nextThreadRetirementDate.toSystemTimePoint());
             } else {
                 // Since the number of threads is not more than minThreads, this thread is not
@@ -270,6 +272,7 @@ void ThreadPool::_consumeTasks() {
                 // would be eligible for retirement once they had no work left to do.
                 LOG(3) << "waiting for work; I am one of " << _threads.size() << " thread(s);"
                        << " the minimum number of threads is " << _options.minThreads;
+                IdleThreadBlock markIdle;
                 _workAvailable.wait(lk);
             }
             continue;