SERVER-38538 Fix thread safety issues in periodic runners

(cherry picked from commit 2f7ccbaa6ae394d773834ff8bd1fc9aae3a42287)

4 files changed, 15 insertions, 14 deletions

diff --git a/src/mongo/embedded/periodic_runner_embedded.cpp b/src/mongo/embedded/periodic_runner_embedded.cpp
index 62a7818a5e7..b20ea9cec2f 100644
--- a/src/mongo/embedded/periodic_runner_embedded.cpp
+++ b/src/mongo/embedded/periodic_runner_embedded.cpp
@@ -111,8 +111,9 @@ void PeriodicRunnerEmbedded::shutdown() {
         _running = false;
 
         for (auto& job : _jobs) {
-            if (job->isAlive(lk)) {
-                job->stop();
+            stdx::lock_guard<stdx::mutex> jobLock(job->_mutex);
+            if (job->isAlive(jobLock)) {
+                job->_stopWithMasterAndJobLock(lk, jobLock);
             }
         }
         _jobs.clear();
@@ -225,8 +226,12 @@ void PeriodicRunnerEmbedded::PeriodicJobImpl::stop() {
     // sure the user can invalidate it after this call.
     stdx::lock_guard<stdx::mutex> masterLock(_periodicRunner->_mutex);
     stdx::lock_guard<stdx::mutex> lk(_mutex);
-    invariant(isAlive(lk));
+    _stopWithMasterAndJobLock(masterLock, lk);
+}
 
+void PeriodicRunnerEmbedded::PeriodicJobImpl::_stopWithMasterAndJobLock(WithLock masterLock,
+                                                                        WithLock jobLock) {
+    invariant(isAlive(jobLock));
     _execStatus = PeriodicJobImpl::ExecutionStatus::kCanceled;
 }
 
diff --git a/src/mongo/embedded/periodic_runner_embedded.h b/src/mongo/embedded/periodic_runner_embedded.h
index dedc0175dea..20dc7a410ba 100644
--- a/src/mongo/embedded/periodic_runner_embedded.h
+++ b/src/mongo/embedded/periodic_runner_embedded.h
@@ -83,6 +83,8 @@ private:
         enum class ExecutionStatus { kNotScheduled, kRunning, kPaused, kCanceled };
 
     private:
+        void _stopWithMasterAndJobLock(WithLock masterLock, WithLock jobLock);
+
         PeriodicJob _job;
         ClockSource* _clockSource;
         PeriodicRunnerEmbedded* _periodicRunner;
diff --git a/src/mongo/util/periodic_runner_impl.cpp b/src/mongo/util/periodic_runner_impl.cpp
index 43ef3317872..aad8d19c34f 100644
--- a/src/mongo/util/periodic_runner_impl.cpp
+++ b/src/mongo/util/periodic_runner_impl.cpp
@@ -89,9 +89,7 @@ void PeriodicRunnerImpl::shutdown() {
         _running = false;
 
         for (auto& job : _jobs) {
-            if (job->isAlive()) {
-                job->stop();
-            }
+            job->stop();
         }
         _jobs.clear();
     }
@@ -156,20 +154,17 @@ void PeriodicRunnerImpl::PeriodicJobImpl::resume() {
 void PeriodicRunnerImpl::PeriodicJobImpl::stop() {
     {
         stdx::lock_guard<stdx::mutex> lk(_mutex);
-        invariant(isAlive());
+        if (_execStatus != ExecutionStatus::RUNNING && _execStatus != ExecutionStatus::PAUSED)
+            return;
+
         invariant(_thread.joinable());
 
         _execStatus = PeriodicJobImpl::ExecutionStatus::CANCELED;
     }
-    invariant(_thread.joinable());
     _condvar.notify_one();
     _thread.join();
 }
 
-bool PeriodicRunnerImpl::PeriodicJobImpl::isAlive() {
-    return _execStatus == ExecutionStatus::RUNNING || _execStatus == ExecutionStatus::PAUSED;
-}
-
 namespace {
 
 template <typename T>
diff --git a/src/mongo/util/periodic_runner_impl.h b/src/mongo/util/periodic_runner_impl.h
index 7c0d0d1bd8f..8c59a4f3e84 100644
--- a/src/mongo/util/periodic_runner_impl.h
+++ b/src/mongo/util/periodic_runner_impl.h
@@ -65,6 +65,7 @@ private:
         MONGO_DISALLOW_COPYING(PeriodicJobImpl);
 
     public:
+        friend class PeriodicRunnerImpl;
         PeriodicJobImpl(PeriodicJob job, ClockSource* source, ServiceContext* svc);
 
         void start();
@@ -72,8 +73,6 @@ private:
         void resume();
         void stop();
 
-        bool isAlive();
-
         enum class ExecutionStatus { NOT_SCHEDULED, RUNNING, PAUSED, CANCELED };
 
     private: