diff options
author | Mathias Stearn <mathias@10gen.com> | 2017-03-10 11:57:13 -0500 |
---|---|---|
committer | Eddie Louie <eddie.louie@mongodb.com> | 2017-05-02 15:44:34 -0400 |
commit | b36663f746b5ed671c9fd73388ad0f9f2478c0f6 (patch) | |
tree | 5003541f197fece663255d4d435d024de8f16439 | |
parent | 2f725411ba9f39ab7ddfa76b5b734907133297ac (diff) | |
download | mongo-b36663f746b5ed671c9fd73388ad0f9f2478c0f6.tar.gz |
SERVER-27727 Hide idle threads in hang analyzer (core only)
(cherry picked from commit 27ddad2221974798284ef62d3328a3c02a510220)
-rw-r--r-- | buildscripts/gdb/mongo.py | 19 | ||||
-rwxr-xr-x | buildscripts/hang_analyzer.py | 2 | ||||
-rw-r--r-- | src/mongo/SConscript | 1 | ||||
-rw-r--r-- | src/mongo/db/service_entry_point_mongod.cpp | 6 | ||||
-rw-r--r-- | src/mongo/s/service_entry_point_mongos.cpp | 6 | ||||
-rw-r--r-- | src/mongo/util/concurrency/idle_thread_block.cpp | 50 | ||||
-rw-r--r-- | src/mongo/util/concurrency/idle_thread_block.h | 59 | ||||
-rw-r--r-- | src/mongo/util/concurrency/thread_pool.cpp | 3 |
8 files changed, 143 insertions, 3 deletions
diff --git a/buildscripts/gdb/mongo.py b/buildscripts/gdb/mongo.py index 22420ef0ec2..a607f55c06e 100644 --- a/buildscripts/gdb/mongo.py +++ b/buildscripts/gdb/mongo.py @@ -387,6 +387,25 @@ class MongoDBDumpLocks(gdb.Command): # Register command MongoDBDumpLocks() +class BtIfActive(gdb.Command): + """Print stack trace or a short message if the current thread is idle""" + + def __init__(self): + register_mongo_command(self, "mongodb-bt-if-active", gdb.COMMAND_DATA) + + def invoke(self, arg, _from_tty): + try: + is_idle = gdb.parse_and_eval("mongo::for_debuggers::threadIsIdle") + except gdb.error: + is_idle = False # If unsure, print a stack trace. + + if is_idle: + print("Thread is idle") + else: + gdb.execute("bt") + +# Register command +BtIfActive() class MongoDBUniqueStack(gdb.Command): """Print unique stack traces of all threads in current process""" diff --git a/buildscripts/hang_analyzer.py b/buildscripts/hang_analyzer.py index 1e88a26f833..b85442da742 100755 --- a/buildscripts/hang_analyzer.py +++ b/buildscripts/hang_analyzer.py @@ -349,7 +349,7 @@ class GDBDumper(object): source_mongo_lock = "source %s" % mongo_lock_script mongodb_dump_locks = "mongodb-dump-locks" mongodb_show_locks = "mongodb-show-locks" - mongodb_uniqstack = "mongodb-uniqstack bt" + mongodb_uniqstack = "mongodb-uniqstack mongodb-bt-if-active" mongodb_waitsfor_graph = "mongodb-waitsfor-graph debugger_waitsfor_%s_%d.gv" % \ (process_name, pid) mongodb_javascript_stack = "mongodb-javascript-stack" diff --git a/src/mongo/SConscript b/src/mongo/SConscript index 62c17a7ecd1..2cfb2060e7f 100644 --- a/src/mongo/SConscript +++ b/src/mongo/SConscript @@ -109,6 +109,7 @@ baseSource=[ 'util/allocator.cpp', 'util/assert_util.cpp', 'util/base64.cpp', + 'util/concurrency/idle_thread_block.cpp', 'util/concurrency/thread_name.cpp', 'util/duration.cpp', 'util/exception_filter_win32.cpp', diff --git a/src/mongo/db/service_entry_point_mongod.cpp b/src/mongo/db/service_entry_point_mongod.cpp index ac00e12d9ac..4f89b7cdb38 100644 --- a/src/mongo/db/service_entry_point_mongod.cpp +++ b/src/mongo/db/service_entry_point_mongod.cpp @@ -42,6 +42,7 @@ #include "mongo/transport/session.h" #include "mongo/transport/ticket.h" #include "mongo/transport/transport_layer.h" +#include "mongo/util/concurrency/idle_thread_block.h" #include "mongo/util/exit.h" #include "mongo/util/log.h" #include "mongo/util/net/message.h" @@ -113,7 +114,10 @@ void ServiceEntryPointMongod::_sessionLoop(const transport::SessionHandle& sessi // 1. Source a Message from the client (unless we are exhausting) if (!inExhaust) { inMessage.reset(); - auto status = session->sourceMessage(&inMessage).wait(); + auto status = [&] { + IdleThreadBlock markIdle; + return session->sourceMessage(&inMessage).wait(); + }(); if (ErrorCodes::isInterruption(status.code()) || ErrorCodes::isNetworkError(status.code())) { diff --git a/src/mongo/s/service_entry_point_mongos.cpp b/src/mongo/s/service_entry_point_mongos.cpp index 67ac08e9354..834875bc1b6 100644 --- a/src/mongo/s/service_entry_point_mongos.cpp +++ b/src/mongo/s/service_entry_point_mongos.cpp @@ -43,6 +43,7 @@ #include "mongo/transport/service_entry_point_utils.h" #include "mongo/transport/session.h" #include "mongo/transport/transport_layer.h" +#include "mongo/util/concurrency/idle_thread_block.h" #include "mongo/util/log.h" #include "mongo/util/net/message.h" #include "mongo/util/net/thread_idle_callback.h" @@ -85,7 +86,10 @@ void ServiceEntryPointMongos::_sessionLoop(const transport::SessionHandle& sessi // Source a Message from the client { - auto status = session->sourceMessage(&message).wait(); + auto status = [&] { + IdleThreadBlock markIdle; + return session->sourceMessage(&message).wait(); + }(); if (ErrorCodes::isInterruption(status.code()) || ErrorCodes::isNetworkError(status.code())) { diff --git a/src/mongo/util/concurrency/idle_thread_block.cpp b/src/mongo/util/concurrency/idle_thread_block.cpp new file mode 100644 index 00000000000..783ab4184cf --- /dev/null +++ b/src/mongo/util/concurrency/idle_thread_block.cpp @@ -0,0 +1,50 @@ +/* Copyright 2017 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/util/assert_util.h" +#include "mongo/util/concurrency/idle_thread_block.h" +#include "mongo/util/concurrency/threadlocal.h" + +namespace mongo { +namespace for_debuggers { +// This needs external linkage to ensure that debuggers can use it. +MONGO_TRIVIALLY_CONSTRUCTIBLE_THREAD_LOCAL bool threadIsIdle = false; +} +using for_debuggers::threadIsIdle; + +void IdleThreadBlock::beginIdleThreadBlock() { + invariant(!threadIsIdle); + threadIsIdle = true; +} + +void IdleThreadBlock::endIdleThreadBlock() { + invariant(threadIsIdle); + threadIsIdle = false; +} +} diff --git a/src/mongo/util/concurrency/idle_thread_block.h b/src/mongo/util/concurrency/idle_thread_block.h new file mode 100644 index 00000000000..cfdcc35d277 --- /dev/null +++ b/src/mongo/util/concurrency/idle_thread_block.h @@ -0,0 +1,59 @@ +/* Copyright 2017 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#pragma once + +#include "mongo/base/disallow_copying.h" + +namespace mongo { + +/** + * Marks a thread as idle while in scope. + * + * Our debugger scripts can hide idle threads when dumping all stacks. You should mark threads as + * idle when printing the stack would just be unhelpful noise. IdleThreadBlocks are not allowed to + * nest. Each thread should generally have at most one possible place where it it is considered + * idle. + */ +class IdleThreadBlock { + MONGO_DISALLOW_COPYING(IdleThreadBlock); + +public: + IdleThreadBlock() { + beginIdleThreadBlock(); + } + ~IdleThreadBlock() { + endIdleThreadBlock(); + } + + // These should not be called by mongo C++ code. They are only public to allow exposing this + // functionality to a C api. + static void beginIdleThreadBlock(); + static void endIdleThreadBlock(); +}; + +} // namespace mongo diff --git a/src/mongo/util/concurrency/thread_pool.cpp b/src/mongo/util/concurrency/thread_pool.cpp index f5687eabf2f..5e0e7419adf 100644 --- a/src/mongo/util/concurrency/thread_pool.cpp +++ b/src/mongo/util/concurrency/thread_pool.cpp @@ -35,6 +35,7 @@ #include "mongo/base/status.h" #include "mongo/platform/atomic_word.h" #include "mongo/util/assert_util.h" +#include "mongo/util/concurrency/idle_thread_block.h" #include "mongo/util/concurrency/thread_name.h" #include "mongo/util/log.h" #include "mongo/util/mongoutils/str.h" @@ -262,6 +263,7 @@ void ThreadPool::_consumeTasks() { LOG(3) << "Not reaping because the earliest retirement date is " << nextThreadRetirementDate; + IdleThreadBlock markIdle; _workAvailable.wait_until(lk, nextThreadRetirementDate.toSystemTimePoint()); } else { // Since the number of threads is not more than minThreads, this thread is not @@ -270,6 +272,7 @@ void ThreadPool::_consumeTasks() { // would be eligible for retirement once they had no work left to do. LOG(3) << "waiting for work; I am one of " << _threads.size() << " thread(s);" << " the minimum number of threads is " << _options.minThreads; + IdleThreadBlock markIdle; _workAvailable.wait(lk); } continue; |