SERVER-46648 Make _killOperations command kill cursors

(cherry picked from commit 29798df747882fcb02fe98178d42ade0b9a0da52)

8 files changed, 157 insertions, 33 deletions

diff --git a/jstests/noPassthrough/kill_operations.js b/jstests/noPassthrough/kill_operations.js
index ae66eb45cc9..bc13e7bbe33 100644
--- a/jstests/noPassthrough/kill_operations.js
+++ b/jstests/noPassthrough/kill_operations.js
@@ -13,6 +13,7 @@ const kOpKey1 = "57710eee-37cf-4c68-a3ac-0b0b900c15d2";
 const kOpKey2 = "488f6050-e331-4483-b356-230a41ec477e";
 const kOpKey3 = "c3eb12fc-4638-4464-8f51-312724ad1710";
 const kOpKey4 = "c7148048-fcf8-4caa-9756-59728052d6a7";
+const kOpKey5 = "c7148048-fcf8-4caa-9756-59728052d6a8";
 
 const st = new ShardingTest({shards: 1, rs: {nodes: 1}, mongos: 1});
 const shardConn = st.rs0.getPrimary();
@@ -150,18 +151,29 @@ function runTest(conn) {
         unblockFinds();
     }
 
-    // Test that _killOperations does not kill the cursors associated with each operationKey.
-    // TODO (SERVER-46648): test that _killOperations does kill the cursors.
-    const res = assert.commandWorked(db.runCommand({
+    // Test that _killOperations kills the cursors that are associated with the given
+    // operationKeys, and does not fail if any of the cursors are already closed.
+    const res4 = assert.commandWorked(db.runCommand({
         find: kCollName,
         filter: {x: {$gte: 0}},
         batchSize: kBatchSize,
         clientOperationKey: UUID(kOpKey4),
     }));
-    const cursorId = res.cursor.id;
-    assert.neq(0, cursorId);
-    killOpKey(conn, [kOpKey4]);
-    assert.commandWorked(db.runCommand({getMore: cursorId, collection: kCollName}));
+    const cursorId4 = res4.cursor.id;
+    assert.neq(0, cursorId4);
+
+    const res5 = assert.commandWorked(db.runCommand({
+        find: kCollName,
+        filter: {x: {$lt: 0}},
+        batchSize: kBatchSize,
+        clientOperationKey: UUID(kOpKey5),
+    }));
+    const cursorId5 = res5.cursor.id;
+    assert.eq(0, cursorId5);
+
+    killOpKey(conn, [kOpKey4, kOpKey5]);
+    assert.commandFailedWithCode(db.runCommand({getMore: cursorId4, collection: kCollName}),
+                                 ErrorCodes.CursorNotFound);
 }
 
 // Test killOp against mongod.
diff --git a/src/mongo/db/clientcursor.cpp b/src/mongo/db/clientcursor.cpp
index 9834c386577..81da894e302 100644
--- a/src/mongo/db/clientcursor.cpp
+++ b/src/mongo/db/clientcursor.cpp
@@ -74,10 +74,6 @@ static ServerStatusMetricField<Counter64> dCursorStatsOpenNoTimeout("cursor.open
 static ServerStatusMetricField<Counter64> dCursorStatusTimedout("cursor.timedOut",
                                                                 &cursorStatsTimedOut);
 
-long long ClientCursor::totalOpen() {
-    return cursorStatsOpen.get();
-}
-
 ClientCursor::ClientCursor(ClientCursorParams params,
                            CursorId cursorId,
                            OperationContext* operationUsingCursor,
diff --git a/src/mongo/db/clientcursor.h b/src/mongo/db/clientcursor.h
index 582c7a72502..5979ed4521c 100644
--- a/src/mongo/db/clientcursor.h
+++ b/src/mongo/db/clientcursor.h
@@ -316,12 +316,6 @@ public:
         _lastKnownCommittedOpTime = std::move(lastCommittedOpTime);
     }
 
-    /**
-     * Returns the server-wide the count of living cursors. Such a cursor is called an "open
-     * cursor".
-     */
-    static long long totalOpen();
-
     friend std::size_t partitionOf(const ClientCursor* cursor) {
         return cursor->cursorid();
     }
diff --git a/src/mongo/db/commands/SConscript b/src/mongo/db/commands/SConscript
index 4fcc84fe67c..51e390c8274 100644
--- a/src/mongo/db/commands/SConscript
+++ b/src/mongo/db/commands/SConscript
@@ -101,7 +101,6 @@ env.Library(
         'kill_all_sessions_by_pattern_command.cpp',
         'kill_all_sessions_command.cpp',
         'kill_sessions_command.cpp',
-        'kill_operations_command.cpp',
         'parameters.cpp',
         'refresh_logical_session_cache_now.cpp',
         'refresh_sessions_command.cpp',
@@ -109,7 +108,6 @@ env.Library(
         'start_session_command.cpp',
         env.Idlc('parameters.idl')[0],
         env.Idlc('sessions_commands.idl')[0],
-        env.Idlc('kill_operations.idl')[0],
     ],
     LIBDEPS_PRIVATE=[
         '$BUILD_DIR/mongo/bson/mutable/mutable_bson',
@@ -123,7 +121,6 @@ env.Library(
         '$BUILD_DIR/mongo/db/logical_session_id_helpers',
         '$BUILD_DIR/mongo/db/logical_session_id',
         '$BUILD_DIR/mongo/db/mongohasher',
-        '$BUILD_DIR/mongo/db/operation_killer',
         '$BUILD_DIR/mongo/db/server_options_core',
         '$BUILD_DIR/mongo/idl/server_parameter',
         '$BUILD_DIR/mongo/logger/parse_log_component_settings',
@@ -270,6 +267,7 @@ env.Library(
         "index_filter_commands.cpp",
         "kill_op.cpp",
         "killcursors_cmd.cpp",
+        "killoperations_cmd.cpp",
         "lock_info.cpp",
         "list_collections.cpp",
         "list_databases.cpp",
@@ -449,7 +447,8 @@ env.Library(
     target='kill_common',
     source=[
         'killcursors_common.cpp',
-        'kill_op_cmd_base.cpp'
+        'kill_op_cmd_base.cpp',
+        env.Idlc('kill_operations.idl')[0],
     ],
     LIBDEPS=[
         '$BUILD_DIR/mongo/db/auth/authprivilege',
diff --git a/src/mongo/db/commands/killoperations_cmd.cpp b/src/mongo/db/commands/killoperations_cmd.cpp
new file mode 100644
index 00000000000..fd01fd1b4b5
--- /dev/null
+++ b/src/mongo/db/commands/killoperations_cmd.cpp
@@ -0,0 +1,56 @@
+/**
+ *    Copyright (C) 2020-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/db/commands/killoperations_common.h"
+#include "mongo/db/cursor_manager.h"
+
+namespace mongo {
+
+class KillOperationsCmd : public KillOperationsCmdBase<KillOperationsCmd> {
+public:
+    static void killCursors(OperationContext* opCtx, const std::vector<OperationKey>& opKeys) {
+        auto cursorManager = CursorManager::get(opCtx);
+        for (auto& cursorId : cursorManager->getCursorsForOpKeys(opKeys)) {
+            LOGV2(4664802, "Attempting to kill cursor", "cursorId"_attr = cursorId);
+            auto status = cursorManager->killCursor(opCtx, cursorId, true /* shouldAudit */);
+
+            if (!status.isOK()) {
+                LOGV2(4664803,
+                      "Failed to kill the cursor ",
+                      "status"_attr = redact(status.toString()));
+            } else {
+                LOGV2(4664804, "Killed cursor", "cursorId"_attr = cursorId);
+            }
+        }
+    }
+} KillOperationsCmd;
+
+}  // namespace mongo
diff --git a/src/mongo/db/commands/kill_operations_command.cpp b/src/mongo/db/commands/killoperations_common.h
index 037eeb554e5..faf0f7bea5c 100644
--- a/src/mongo/db/commands/kill_operations_command.cpp
+++ b/src/mongo/db/commands/killoperations_common.h
@@ -1,5 +1,5 @@
 /**
- *    Copyright (C) 2019-present MongoDB, Inc.
+ *    Copyright (C) 2020-present MongoDB, Inc.
  *
  *    This program is free software: you can redistribute it and/or modify
  *    it under the terms of the Server Side Public License, version 1,
@@ -49,27 +49,30 @@
 
 namespace mongo {
 
-class KillOperationsCommand final : public TypedCommand<KillOperationsCommand> {
+template <typename Derived>
+class KillOperationsCmdBase : public TypedCommand<Derived> {
 public:
     using Request = KillOperationsRequest;
 
-    class Invocation final : public InvocationBase {
+    class Invocation final : public TypedCommand<Derived>::InvocationBase {
     public:
-        using InvocationBase::InvocationBase;
+        using Base = typename TypedCommand<Derived>::InvocationBase;
+        using Base::Base;
 
         void typedRun(OperationContext* opCtx) {
             auto opKiller = OperationKiller(opCtx->getClient());
-            for (auto& opKey : request().getOperationKeys()) {
-                LOGV2(46156011,
-                      "Attempting to kill operation with OperationKey '{operationKey}'",
-                      "operationKey"_attr = opKey);
+            auto opKeys = Base::request().getOperationKeys();
+
+            for (auto& opKey : opKeys) {
+                LOGV2(46156011, "Attempting to kill operation", "operationKey"_attr = opKey);
                 opKiller.killOperation(OperationKey(opKey));
             }
+            Derived::killCursors(opCtx, opKeys);
         }
 
     private:
         NamespaceString ns() const override {
-            return NamespaceString(request().getDbName(), "");
+            return NamespaceString(Base::request().getDbName(), "");
         }
 
         bool supportsWriteConcern() const override {
@@ -97,8 +100,8 @@ public:
     };
 
 private:
-    AllowedOnSecondary secondaryAllowed(ServiceContext*) const override {
-        return AllowedOnSecondary::kAlways;
+    Command::AllowedOnSecondary secondaryAllowed(ServiceContext*) const override {
+        return Command::AllowedOnSecondary::kAlways;
     }
 
     bool adminOnly() const override {
@@ -108,6 +111,6 @@ private:
     std::string help() const override {
         return "Internal command -- Kill operations on the target server by OperationKey.";
     }
-} killOperationsCmd;
+};
 
 }  // namespace mongo
diff --git a/src/mongo/s/commands/SConscript b/src/mongo/s/commands/SConscript
index 51590af5ad8..323d81106a2 100644
--- a/src/mongo/s/commands/SConscript
+++ b/src/mongo/s/commands/SConscript
@@ -60,6 +60,7 @@ env.Library(
         'cluster_is_master_cmd.cpp',
         'cluster_kill_op.cpp',
         'cluster_killcursors_cmd.cpp',
+        'cluster_killoperations_cmd.cpp',
         'cluster_list_databases_cmd.cpp',
         'cluster_list_shards_cmd.cpp',
         'cluster_map_reduce_agg.cpp',
diff --git a/src/mongo/s/commands/cluster_killoperations_cmd.cpp b/src/mongo/s/commands/cluster_killoperations_cmd.cpp
new file mode 100644
index 00000000000..2457958d351
--- /dev/null
+++ b/src/mongo/s/commands/cluster_killoperations_cmd.cpp
@@ -0,0 +1,63 @@
+/**
+ *    Copyright (C) 2020-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/db/commands/killoperations_common.h"
+#include "mongo/s/grid.h"
+#include "mongo/s/query/cluster_cursor_manager.h"
+
+namespace mongo {
+
+class ClusterKillOperationsCmd : public KillOperationsCmdBase<ClusterKillOperationsCmd> {
+public:
+    static void killCursors(OperationContext* opCtx, const std::vector<OperationKey>& opKeys) {
+        auto clusterCursorManager = Grid::get(opCtx)->getCursorManager();
+        for (auto& cursorId : clusterCursorManager->getCursorsForOpKeys(opKeys)) {
+            LOGV2(4664805, "Attempting to kill cursor", "cursorId"_attr = cursorId);
+            auto cursorNss = clusterCursorManager->getNamespaceForCursorId(cursorId);
+            if (!cursorNss) {
+                // The cursor must have already been killed.
+                continue;
+            }
+
+            auto status = clusterCursorManager->killCursor(opCtx, cursorNss.get(), cursorId);
+
+            if (!status.isOK()) {
+                LOGV2(4664806,
+                      "Failed to kill the cursor ",
+                      "status"_attr = redact(status.toString()));
+            } else {
+                LOGV2(4664807, "Killed cursor", "cursorId"_attr = cursorId);
+            }
+        }
+    }
+} ClusterKillOperationsCmd;
+
+}  // namespace mongo