SERVER-67613 Implement Mock for Async RPC API

4 files changed, 559 insertions, 42 deletions

diff --git a/src/mongo/executor/mock_remote_command_runner.h b/src/mongo/executor/mock_remote_command_runner.h
index 85c2d1881b1..2441b166d35 100644
--- a/src/mongo/executor/mock_remote_command_runner.h
+++ b/src/mongo/executor/mock_remote_command_runner.h
@@ -27,19 +27,180 @@
  *    it in the license file.
  */
 
+#include <deque>
+
 #include "mongo/executor/remote_command_runner.h"
+#include "mongo/executor/remote_command_runner_error_info.h"
 #include "mongo/executor/remote_command_targeter.h"
 #include "mongo/executor/task_executor.h"
+#include "mongo/util/future_util.h"
+#include "mongo/util/producer_consumer_queue.h"
 
 namespace mongo::executor::remote_command_runner {
 
 /**
- * A mock implementation of the RemoteCommandRunner. In unit tests, swap out the RemoteCommandRunner
- * decoration on the service context with an instance of this mock to inject your own behavior and
- * check invariants.
+ * This header provides two mock implementations of the remote_command_runner::doRequest API. In
+ * unit tests, swap out the RemoteCommandRunner decoration on the ServiceContext with an instance
+ * of one of these mocks to inject your own behavior and check invariants.
+ *
+ *  See the header comments for each mock for details.
+ */
+
+/**
+ * The SyncMockRemoteCommandRunner's representation of a RPC request. Exposed to users of the mock
+ * via the onCommand/getNextRequest APIs, which can be used to examine what requests have been
+ * scheduled on the mock and to schedule responsess to them.
+ */
+class RequestInfo {
+public:
+    RequestInfo(BSONObj cmd, StringData dbName, HostAndPort target, Promise<BSONObj>&& promise)
+        : _cmd{cmd}, _dbName{dbName}, _target{target}, _responsePromise{std::move(promise)} {}
+
+    bool isFinished() const {
+        return _finished.load();
+    }
+
+    /**
+     * Use this function to respond to the request. Errors in the 'Status' portion of the
+     * 'StatusWith' correspond to 'local errors' (i.e. those on the sending node), while the BSONObj
+     * corresponds to the raw BSON received as a response to the request over-the-wire.
+     */
+    void respondWith(StatusWith<BSONObj> response) {
+        bool expected = false;
+        if (_finished.compareAndSwap(&expected, true))
+            _responsePromise.setFrom(response);
+    }
+
+    BSONObj _cmd;
+    StringData _dbName;
+    HostAndPort _target;
+
+private:
+    AtomicWord<bool> _finished{false};
+    Promise<BSONObj> _responsePromise;
+};
+
+/**
+ * The SyncMockRemoteCommandRunner is a mock implementation that can be interacted with in a
+ * synchronous/ blocking manner. It allows you to:
+ *      -> Synchronously introspect onto what requests have been scheduled.
+ *      -> Synchronously respond to those requests.
+ *      -> Wait (blocking) for a request to be scheduled and respond to it.
+ * See the member funtions below for details.
+ */
+class SyncMockRemoteCommandRunner : public detail::RemoteCommandRunner {
+public:
+    /**
+     * Mock implementation of the core functionality of the RCR. Records the provided request, and
+     * notifies waiters that a new request has been scheduled.
+     */
+    ExecutorFuture<detail::RemoteCommandInternalResponse> _doRequest(
+        StringData dbName,
+        BSONObj cmdBSON,
+        std::unique_ptr<RemoteCommandHostTargeter> targeter,
+        OperationContext* opCtx,
+        std::shared_ptr<TaskExecutor> exec,
+        CancellationToken token) final {
+        auto [p, f] = makePromiseFuture<BSONObj>();
+        return targeter->resolve(token)
+            .thenRunOn(exec)
+            .onError([](Status s) -> StatusWith<std::vector<HostAndPort>> {
+                return Status{RemoteCommandExecutionErrorInfo(s),
+                              "Remote command execution failed"};
+            })
+            .then([=, f = std::move(f), p = std::move(p)](auto&& targets) mutable {
+                stdx::lock_guard lg{_m};
+                _requests.emplace_back(cmdBSON, dbName, targets[0], std::move(p));
+                _hasRequestsCV.notify_one();
+                return std::move(f).then([targetUsed = targets[0]](StatusWith<BSONObj> resp) {
+                    if (!resp.isOK()) {
+                        uassertStatusOK(Status{RemoteCommandExecutionErrorInfo(resp.getStatus()),
+                                               "Remote command execution failed"});
+                    }
+                    Status maybeError(detail::makeErrorIfNeeded(
+                        RemoteCommandOnAnyResponse(targetUsed, resp.getValue(), Microseconds(1))));
+                    uassertStatusOK(maybeError);
+                    return detail::RemoteCommandInternalResponse{resp.getValue(), targetUsed};
+                });
+            });
+    }
+
+    /**
+     * Gets the oldest request the RCR has recieved that has not yet been responded to. Blocking
+     * call - if no unresponded requests are currently queued in the mock, this call blocks until
+     * one arrives.
+     */
+    RequestInfo& getNextRequest() {
+        stdx::unique_lock lk(_m);
+        auto pred = [this] {
+            auto nextUnprocessedRequest =
+                std::find_if(_requests.begin(), _requests.end(), [](const RequestInfo& r) {
+                    return !r.isFinished();
+                });
+            return nextUnprocessedRequest != _requests.end();
+        };
+        _hasRequestsCV.wait(lk, pred);
+        return *std::find_if(_requests.begin(), _requests.end(), [](const RequestInfo& r) {
+            return !r.isFinished();
+        });
+    }
+
+    /**
+     * Responds to the first unprocessed request and then responds to it with the result of calling
+     * `responseFn` with the request as argument. If there are no unprocessed requests, blocks until
+     * there is.
+     */
+    void onCommand(std::function<StatusWith<BSONObj>(const RequestInfo&)> responseFn) {
+        auto& request = getNextRequest();
+        request.respondWith(responseFn(request));
+    }
+
+private:
+    // All requests that have been scheduled on the mock.
+    std::deque<RequestInfo> _requests;
+    // Protects the above _requests queue.
+    Mutex _m;
+    stdx::condition_variable _hasRequestsCV;
+};
+
+/**
+ * The AsyncMockRemoteCommandRunner allows you to asynchrously register expectations about what
+ * requests will be registered; you can:
+ *  -> Create an 'expectation' that a request will be scheduled some time in the future,
+ *     asynchronously.
+ *  -> Provide responses to those requests.
+ *  -> Ensure that any expectations were met/requests meeting all expectations were
+ *     received and responded to.
+ *  -> Examine if any unexpected requests were received and, if so, what they were.
+ * See the member funtions below for details.
  */
-class MockRemoteCommandRunner : public detail::RemoteCommandRunner {
+class AsyncMockRemoteCommandRunner : public detail::RemoteCommandRunner {
 public:
+    struct Request {
+        BSONObj toBSON() const {
+            BSONObjBuilder bob;
+            BSONObjBuilder subBob(bob.subobjStart("AsyncMockRemoteCommandRunner::Request"));
+            subBob.append("dbName: ", dbName);
+            subBob.append("command: ", cmdBSON);
+            subBob.append("target: ", target.toString());
+            subBob.done();
+            return bob.obj();
+        }
+        std::string dbName;
+        BSONObj cmdBSON;
+        HostAndPort target;
+    };
+
+    /**
+     * Mock implementation of the core functionality of the RCR. Checks that the request is expected
+     * by ensuring that we have an unmet expectation registered that matches it. Then responds to
+     * the request with the response provided by the matching expectation. If more the one
+     * expectation matches the request, the first unmet one registered with `expect` will be used.
+     * See `expect` below for details.  If no expectation matches the request, the request is
+     * responded to with a generic error, and the request is recorded. Users that want to make sure
+     * that all requests are expected can use the hadUnexpectedRequests/unexpectedRequests member
+     * functions to inspect that state.
+     */
     ExecutorFuture<detail::RemoteCommandInternalResponse> _doRequest(
         StringData dbName,
         BSONObj cmdBSON,
@@ -47,15 +208,139 @@ public:
         OperationContext* opCtx,
         std::shared_ptr<TaskExecutor> exec,
         CancellationToken token) final {
-        return ExecutorFuture(
-            exec,
-            detail::RemoteCommandInternalResponse{_mockResult, targeter->resolve(token).get()[0]});
+        auto [p, f] = makePromiseFuture<BSONObj>();
+        return targeter->resolve(token).thenRunOn(exec).then(
+            [this, p = std::move(p), cmdBSON, dbName = dbName.toString()](auto&& targets) {
+                stdx::lock_guard lg(_m);
+                Request req{dbName, cmdBSON, targets[0]};
+                auto expectation =
+                    std::find_if(_expectations.begin(),
+                                 _expectations.end(),
+                                 [&](const Expectation& e) { return e.matcher(req) && !e.met; });
+                if (expectation == _expectations.end()) {
+                    // This request was not expected. We record it and return a generic error
+                    // response.
+                    _unexpectedRequests.push_back(req);
+                    uassertStatusOK(
+                        Status{RemoteCommandExecutionErrorInfo(Status(
+                                   ErrorCodes::InternalErrorNotSupported, "Unexpected request")),
+                               "Remote command execution failed"});
+                }
+                auto ans = expectation->response;
+                expectation->promise.emplaceValue();
+                expectation->met = true;
+                if (!ans.isOK()) {
+                    uassertStatusOK(Status{RemoteCommandExecutionErrorInfo(ans.getStatus()),
+                                           "Remote command execution failed"});
+                }
+                Status maybeError(detail::makeErrorIfNeeded(
+                    RemoteCommandOnAnyResponse(targets[0], ans.getValue(), Microseconds(1))));
+                uassertStatusOK(maybeError);
+                return detail::RemoteCommandInternalResponse{ans.getValue(), targets[0]};
+            });
     }
-    void setMockResult(BSONObj obj) {
-        _mockResult = obj;
+
+    using RequestMatcher = std::function<bool(const Request&)>;
+    struct Expectation {
+        Expectation(RequestMatcher m,
+                    StatusWith<BSONObj> response,
+                    Promise<void>&& p,
+                    std::string name)
+            : matcher{m}, response{response}, promise{std::move(p)}, name{name} {}
+        RequestMatcher matcher;
+        StatusWith<BSONObj> response;
+        Promise<void> promise;
+        bool met{false};
+        std::string name;
+    };
+
+    /**
+     * Register an expectation with the mock that a request matching `matcher` will be scheduled on
+     * the mock. The provided RequestMatcher can inspect the cmdObj and target HostAndPort of a
+     * request, and should return 'true' if the request is the expected one. Once the expectation is
+     * 'met' (i.e. the mock has received a matching request), the request is responded to with the
+     * provided `response`. Note that expectations are "one-shot" - each expectation can match
+     * exactly one request; once an expectation has been matched to a scheduled request, it is
+     * considered 'met' and will not be considered in the future.  Additionally, each expectation is
+     * given a name, which allows it to be serialized and identified.
+     *
+     * Errors in the 'Status' portion of the 'StatusWith' response correspond to 'local errors'
+     * (i.e. those on the sending node), while the BSONObj corresponds to the raw BSON received as a
+     * response to the request over-the-wire.
+     */
+    SemiFuture<void> expect(RequestMatcher matcher,
+                            StatusWith<BSONObj> response,
+                            std::string name) {
+        auto [p, f] = makePromiseFuture<void>();
+        stdx::lock_guard lg(_m);
+        _expectations.emplace_back(matcher, response, std::move(p), std::move(name));
+        return std::move(f).semi();
+    }
+
+    bool hasUnmetExpectations() {
+        stdx::lock_guard lg(_m);
+        auto it = std::find_if(_expectations.begin(),
+                               _expectations.end(),
+                               [&](const Expectation& e) { return !e.met; });
+        return it != _expectations.end();
+    }
+
+    // Return the name of the first expectation registered that isn't met. If all have been met,
+    // return boost::none.
+    boost::optional<std::string> getFirstUnmetExpectation() {
+        stdx::lock_guard lg(_m);
+        auto it = std::find_if(_expectations.begin(),
+                               _expectations.end(),
+                               [&](const Expectation& e) { return !e.met; });
+        if (it == _expectations.end()) {
+            return boost::none;
+        }
+        return it->name;
+    }
+
+    // Return the set of names of all unmet expectations.
+    StringSet getUnmetExpectations() {
+        StringSet set;
+        stdx::lock_guard lg(_m);
+        for (auto&& expectation : _expectations) {
+            if (!expectation.met) {
+                set.insert(expectation.name);
+            }
+        }
+        return set;
+    }
+
+    bool hadUnexpectedRequests() {
+        stdx::lock_guard lg(_m);
+        return !_unexpectedRequests.empty();
+    }
+
+    boost::optional<Request> getFirstUnexpectedRequest() {
+        stdx::lock_guard lg(_m);
+        if (!_unexpectedRequests.empty()) {
+            return _unexpectedRequests[0];
+        }
+        return {};
+    }
+
+    std::vector<Request> getUnexpectedRequests() {
+        stdx::lock_guard lg(_m);
+        return _unexpectedRequests;
     }
 
 private:
-    BSONObj _mockResult;
+    std::vector<Expectation> _expectations;
+    std::vector<Request> _unexpectedRequests;
+    // Protects the above _expectations queue.
+    Mutex _m;
 };
+
+std::ostream& operator<<(std::ostream& s, const AsyncMockRemoteCommandRunner::Request& o) {
+    return s << o.toBSON();
+}
+
+std::ostream& operator<<(std::ostream& s, const AsyncMockRemoteCommandRunner::Expectation& o) {
+    return s << o.name;
+}
+
 }  // namespace mongo::executor::remote_command_runner
diff --git a/src/mongo/executor/mock_remote_command_runner_test.cpp b/src/mongo/executor/mock_remote_command_runner_test.cpp
index def6e7a421c..38b8dd7aa7a 100644
--- a/src/mongo/executor/mock_remote_command_runner_test.cpp
+++ b/src/mongo/executor/mock_remote_command_runner_test.cpp
@@ -37,19 +37,22 @@
 #include "mongo/executor/remote_command_runner_test_fixture.h"
 #include "mongo/executor/remote_command_targeter.h"
 #include "mongo/unittest/unittest.h"
+#include "mongo/util/debugger.h"
+#include "mongo/util/optional_util.h"
 
 namespace mongo::executor::remote_command_runner {
 namespace {
 
 /**
- * This test fixture is used to test the functionality of the mock, rather than test any facilities
+ * This test fixture is used to test the functionality of the mocks, rather than test any facilities
  * or usage of the RemoteCommandRunner implementation.
  */
+template <typename MockType>
 class MockRemoteCommandRunnerTestFixture : public RemoteCommandRunnerTestFixture {
 public:
     void setUp() override {
         RemoteCommandRunnerTestFixture::setUp();
-        auto uniqueMock = std::make_unique<MockRemoteCommandRunner>();
+        auto uniqueMock = std::make_unique<MockType>();
         _mock = uniqueMock.get();
         detail::RemoteCommandRunner::set(getServiceContext(), std::move(uniqueMock));
     }
@@ -59,40 +62,264 @@ public:
         RemoteCommandRunnerTestFixture::tearDown();
     }
 
-    MockRemoteCommandRunner& getMockRunner() {
+    MockType& getMockRunner() {
         return *_mock;
     }
 
+
+    SemiFuture<RemoteCommandRunnerResponse<HelloCommandReply>> sendHelloCommandToHostAndPort(
+        HostAndPort target) {
+        HelloCommand hello;
+        initializeCommand(hello);
+        auto opCtxHolder = makeOperationContext();
+        return doRequest(hello,
+                         opCtxHolder.get(),
+                         std::make_unique<RemoteCommandFixedTargeter>(target),
+                         getExecutorPtr(),
+                         _cancellationToken);
+    }
+
+    SemiFuture<RemoteCommandRunnerResponse<HelloCommandReply>> sendHelloCommandToLocalHost() {
+        return sendHelloCommandToHostAndPort({"localhost", serverGlobalParams.port});
+    }
+
 private:
-    MockRemoteCommandRunner* _mock;
+    MockType* _mock;
 };
 
+using SyncMockRemoteCommandRunnerTestFixture =
+    MockRemoteCommandRunnerTestFixture<SyncMockRemoteCommandRunner>;
+using AsyncMockRemoteCommandRunnerTestFixture =
+    MockRemoteCommandRunnerTestFixture<AsyncMockRemoteCommandRunner>;
+
 // A simple test showing that an arbitrary mock result can be set for a command scheduled through
 // the RemoteCommandRunner.
-TEST_F(MockRemoteCommandRunnerTestFixture, Example) {
-    HelloCommand hello;
-    initializeCommand(hello);
-    // Doc that shouldn't be parseable as a HelloCommandReply.
-    auto invalidResult = BSON("An"
-                              << "arbitrary"
-                              << "bogus"
-                              << "document");
-    getMockRunner().setMockResult(invalidResult);
-
-    auto opCtxHolder = makeOperationContext();
-    auto res = doRequest(hello,
-                         opCtxHolder.get(),
-                         std::make_unique<RemoteCommandLocalHostTargeter>(),
-                         getExecutorPtr(),
-                         _cancellationToken);
+TEST_F(SyncMockRemoteCommandRunnerTestFixture, RemoteSuccess) {
+    auto responseFuture = sendHelloCommandToLocalHost();
+
+    HelloCommandReply helloReply = HelloCommandReply(TopologyVersion(OID::gen(), 0));
+    BSONObjBuilder result(helloReply.toBSON());
+    CommandHelpers::appendCommandStatusNoThrow(result, Status::OK());
+    auto expectedResultObj = result.obj();
+
+    auto& request = getMockRunner().getNextRequest();
+    ASSERT_FALSE(responseFuture.isReady());
+
+    request.respondWith(expectedResultObj);
+    auto actualResult = responseFuture.get();
+    HostAndPort localhost = HostAndPort("localhost", serverGlobalParams.port);
+    ASSERT_EQ(actualResult.targetUsed, localhost);
+    ASSERT_BSONOBJ_EQ(actualResult.response.toBSON(), helloReply.toBSON());
+}
+
+TEST_F(SyncMockRemoteCommandRunnerTestFixture, RemoteError) {
+    StringData exampleErrMsg{"example error message"};
+    auto exampleErrCode = ErrorCodes::ShutdownInProgress;
+    ErrorReply errorReply;
+    errorReply.setOk(0);
+    errorReply.setCode(exampleErrCode);
+    errorReply.setCodeName(ErrorCodes::errorString(exampleErrCode));
+    errorReply.setErrmsg(exampleErrMsg);
+
+    auto responseFuture = sendHelloCommandToLocalHost();
+
+    auto& request = getMockRunner().getNextRequest();
+    request.respondWith(errorReply.toBSON());
 
     auto check = [&](const DBException& ex) {
-        ASSERT_EQ(ex.code(), 40415) << ex.toString();
-        ASSERT_STRING_CONTAINS(ex.reason(), "is an unknown field");
+        ASSERT_EQ(ex.code(), ErrorCodes::RemoteCommandExecutionError) << ex.toString();
+        auto extraInfo = ex.extraInfo<RemoteCommandExecutionErrorInfo>();
+        ASSERT(extraInfo);
+
+        ASSERT(extraInfo->isRemote());
+        auto remoteError = extraInfo->asRemote();
+        ASSERT_EQ(remoteError.getRemoteCommandResult().code(), exampleErrCode);
+        ASSERT_EQ(remoteError.getRemoteCommandResult().reason(), exampleErrMsg);
     };
     // Ensure we fail to parse the reply due to the unknown fields.
-    ASSERT_THROWS_WITH_CHECK(res.get(), DBException, check);
+    ASSERT_THROWS_WITH_CHECK(responseFuture.get(), DBException, check);
+}
+
+TEST_F(SyncMockRemoteCommandRunnerTestFixture, LocalError) {
+    auto responseFuture = sendHelloCommandToLocalHost();
+    auto& request = getMockRunner().getNextRequest();
+    ASSERT_FALSE(responseFuture.isReady());
+    auto exampleLocalErr = Status{ErrorCodes::InterruptedAtShutdown, "example local error"};
+    request.respondWith(exampleLocalErr);
+    ASSERT_EQ(responseFuture.getNoThrow().getStatus(), exampleLocalErr);
+}
+
+TEST_F(SyncMockRemoteCommandRunnerTestFixture, MultipleResponses) {
+    auto responseOneFut = sendHelloCommandToLocalHost();
+    ASSERT_FALSE(responseOneFut.isReady());
+    auto& request = getMockRunner().getNextRequest();
+    auto responseTwoFut = sendHelloCommandToLocalHost();
+    ASSERT_FALSE(responseTwoFut.isReady());
+
+    HelloCommandReply helloReply = HelloCommandReply(TopologyVersion(OID::gen(), 0));
+    BSONObjBuilder result(helloReply.toBSON());
+    CommandHelpers::appendCommandStatusNoThrow(result, Status::OK());
+    auto expectedResultObj = result.obj();
+
+    request.respondWith(expectedResultObj);
+    auto responseOne = responseOneFut.get();
+    HostAndPort localhost = HostAndPort("localhost", serverGlobalParams.port);
+    ASSERT_EQ(responseOne.targetUsed, localhost);
+    ASSERT_BSONOBJ_EQ(responseOne.response.toBSON(), helloReply.toBSON());
+
+    auto& requestTwo = getMockRunner().getNextRequest();
+    ASSERT_FALSE(responseTwoFut.isReady());
+    auto exampleLocalErr = Status{ErrorCodes::InterruptedAtShutdown, "example local error"};
+    requestTwo.respondWith(exampleLocalErr);
+    ASSERT_EQ(responseTwoFut.getNoThrow().getStatus(), exampleLocalErr);
+}
+
+TEST_F(SyncMockRemoteCommandRunnerTestFixture, OnCommand) {
+    auto responseFut = sendHelloCommandToLocalHost();
+
+    HelloCommandReply helloReply = HelloCommandReply(TopologyVersion(OID::gen(), 0));
+    BSONObjBuilder result(helloReply.toBSON());
+    CommandHelpers::appendCommandStatusNoThrow(result, Status::OK());
+    auto expectedResultObj = result.obj();
+
+    HelloCommand hello;
+    initializeCommand(hello);
+
+    getMockRunner().onCommand([&](const RequestInfo& ri) {
+        ASSERT_BSONOBJ_EQ(hello.toBSON({}), ri._cmd);
+        return expectedResultObj;
+    });
+    ASSERT_BSONOBJ_EQ(responseFut.get().response.toBSON(), helloReply.toBSON());
+}
+
+// A simple test showing that we can asynchronously register an expectation
+// that a request will eventually be scheduled with the mock before a request
+// actually arrives. Then, once the request is scheduled, we are asynchronously
+// notified of the request and can schedule a response to it.
+TEST_F(AsyncMockRemoteCommandRunnerTestFixture, Expectation) {
+    // We expect that some code will use the runner to send a hello
+    // to localhost on "testdb"
+    auto matcher = [](const AsyncMockRemoteCommandRunner::Request& req) {
+        bool isHello = req.cmdBSON.firstElementFieldName() == "hello"_sd;
+        bool isRightTarget = req.target == HostAndPort("localhost", serverGlobalParams.port);
+        return isHello && isRightTarget;
+    };
+    // Register our expectation and ensure it isn't yet met.
+    HelloCommandReply helloReply = HelloCommandReply(TopologyVersion(OID::gen(), 0));
+    BSONObjBuilder result(helloReply.toBSON());
+    CommandHelpers::appendCommandStatusNoThrow(result, Status::OK());
+
+    auto expectation = getMockRunner().expect(matcher, result.obj(), "example expectation");
+    ASSERT_FALSE(expectation.isReady());
+
+    // Allow a request to be scheduled on the mock.
+    auto response = sendHelloCommandToLocalHost();
+
+    // Now, our expectation should be met, and the response to it provided.
+    auto reply = response.get();
+    expectation.get();
+    ASSERT_BSONOBJ_EQ(reply.response.toBSON(), helloReply.toBSON());
+    ASSERT_EQ(HostAndPort("localhost", serverGlobalParams.port), reply.targetUsed);
+}
+
+// A more complicated test that registers several expectations, and then
+// schedules the requests that match them and their responses out-of-order.
+// Demonstrates how we can register expectations on the mock for events in an
+// unordered way.
+TEST_F(AsyncMockRemoteCommandRunnerTestFixture, SeveralExpectations) {
+    HostAndPort targetOne("FakeHost1", 12345);
+    HostAndPort targetTwo("FakeHost2", 12345);
+    HostAndPort targetThree("FakeHost3", 12345);
+
+    auto matcherOne = [&](const AsyncMockRemoteCommandRunner::Request& req) {
+        return (req.cmdBSON.firstElementFieldName() == "hello"_sd) && (req.target == targetOne);
+    };
+    auto matcherTwo = [&](const AsyncMockRemoteCommandRunner::Request& req) {
+        return (req.cmdBSON.firstElementFieldName() == "hello"_sd) && (req.target == targetTwo);
+    };
+    auto matcherThree = [&](const AsyncMockRemoteCommandRunner::Request& req) {
+        return (req.cmdBSON.firstElementFieldName() == "hello"_sd) && (req.target == targetThree);
+    };
+
+    // Create three expectations
+    HelloCommandReply helloReply = HelloCommandReply(TopologyVersion(OID::gen(), 0));
+    BSONObjBuilder result(helloReply.toBSON());
+    CommandHelpers::appendCommandStatusNoThrow(result, Status::OK());
+    auto resultObj = result.obj();
+    auto e1 = getMockRunner().expect(matcherOne, resultObj, "expectation one");
+    auto e2 = getMockRunner().expect(matcherTwo, resultObj, "expectation two");
+    auto e3 = getMockRunner().expect(matcherThree, resultObj, "expectation three");
+
+    ASSERT_FALSE(e1.isReady());
+    ASSERT_FALSE(e2.isReady());
+    ASSERT_FALSE(e3.isReady());
+
+    // Send requests corresponding to expectations `e3` and `e2`, but not as`e1`.
+    auto r3 = sendHelloCommandToHostAndPort(targetThree);
+    auto r2 = sendHelloCommandToHostAndPort(targetTwo);
+    e3.get();
+    e2.get();
+    ASSERT_FALSE(e1.isReady());
+    // Make sure the correct responses were sent.
+    auto assertResponseMatches = [&](RemoteCommandRunnerResponse<HelloCommandReply> reply,
+                                     const HostAndPort& correctTarget) {
+        ASSERT_EQ(correctTarget, reply.targetUsed);
+        ASSERT_BSONOBJ_EQ(reply.response.toBSON(), helloReply.toBSON());
+    };
+    assertResponseMatches(r3.get(), targetThree);
+    assertResponseMatches(r2.get(), targetTwo);
+
+    // Now, send a request matching `e1` as well.
+    auto r1 = sendHelloCommandToHostAndPort(targetOne);
+    assertResponseMatches(r1.get(), targetOne);
+    e1.get();
 }
 
+TEST_F(AsyncMockRemoteCommandRunnerTestFixture, UnexpectedRequests) {
+    auto responseFut = sendHelloCommandToLocalHost();
+    ASSERT_EQ(responseFut.getNoThrow()
+                  .getStatus()
+                  .extraInfo<RemoteCommandExecutionErrorInfo>()
+                  ->asLocal(),
+              Status(ErrorCodes::InternalErrorNotSupported, "Unexpected request"));
+    ASSERT(getMockRunner().hadUnexpectedRequests());
+    auto unexpectedRequests = getMockRunner().getUnexpectedRequests();
+    ASSERT_EQ(unexpectedRequests.size(), 1);
+    HelloCommand hello;
+    initializeCommand(hello);
+    ASSERT_BSONOBJ_EQ(unexpectedRequests[0].cmdBSON, hello.toBSON({}));
+    ASSERT_EQ(unexpectedRequests[0].dbName, "testdb"_sd);
+    HostAndPort localhost = HostAndPort("localhost", serverGlobalParams.port);
+    ASSERT_EQ(unexpectedRequests[0].target, localhost);
+    // Note that unexpected requests are BSON-convertable and can be printed as extended JSON.
+    // For example, if you wanted to fail the test if any unexpected requests were found, and
+    // print out the first such offending request, you could simply do:
+    // ASSERT(!getMockRunner().hadUnexpectedRequests())
+    //      << "but found: " << optional_io::Extension{getMockRunner().getFirstUnexpectedRequest()};
+    // (This is a live example, feel free to uncomment and try it).
+}
+
+TEST_F(AsyncMockRemoteCommandRunnerTestFixture, UnmetExpectations) {
+    HostAndPort theTarget("FakeHost1", 12345);
+    auto matcher = [&](const AsyncMockRemoteCommandRunner::Request& req) {
+        return (req.cmdBSON.firstElementFieldName() == "hello"_sd) && (req.target == theTarget);
+    };
+    HelloCommandReply helloReply = HelloCommandReply(TopologyVersion(OID::gen(), 0));
+    BSONObjBuilder result(helloReply.toBSON());
+    CommandHelpers::appendCommandStatusNoThrow(result, Status::OK());
+    auto resultObj = result.obj();
+    auto expectation = getMockRunner().expect(matcher, resultObj, "unmet expectation");
+
+    ASSERT(getMockRunner().hasUnmetExpectations());
+    auto unmetExpectations = getMockRunner().getUnmetExpectations();
+    ASSERT_EQ(unmetExpectations.size(), 1);
+    ASSERT(unmetExpectations.contains("unmet expectation"));
+    // Note that unmet expectations all have string names and can be printed.
+    // For example, if you wanted to fail the test if any unmet expectations were found, and
+    // print out the first such offending expectation , you could simply do:
+    // ASSERT(!getMockRunner().hasUnmetExpectations())
+    //      << optional_io::Extension{getMockRunner().getFirstUnmetExpectation()};
+    // (This is a live example, feel free to uncomment and try it).
+}
 }  // namespace
 }  // namespace mongo::executor::remote_command_runner
diff --git a/src/mongo/executor/remote_command_runner.cpp b/src/mongo/executor/remote_command_runner.cpp
index 6b909689094..c106334ce1a 100644
--- a/src/mongo/executor/remote_command_runner.cpp
+++ b/src/mongo/executor/remote_command_runner.cpp
@@ -30,7 +30,6 @@
 #include "mongo/executor/remote_command_runner.h"
 #include "mongo/base/error_codes.h"
 #include "mongo/executor/remote_command_request.h"
-#include "mongo/executor/remote_command_runner_error_info.h"
 #include "mongo/util/assert_util.h"
 #include "mongo/util/future.h"
 #include "mongo/util/net/hostandport.h"
@@ -39,14 +38,6 @@
 namespace mongo::executor::remote_command_runner {
 namespace detail {
 namespace {
-Status makeErrorIfNeeded(TaskExecutor::ResponseOnAnyStatus r) {
-    if (r.status.isOK() && getStatusFromCommandResult(r.data).isOK() &&
-        getWriteConcernStatusFromCommandResult(r.data).isOK() &&
-        getFirstWriteErrorStatusFromCommandResult(r.data).isOK()) {
-        return Status::OK();
-    }
-    return {RemoteCommandExecutionErrorInfo(r), "Remote command execution failed"};
-}
 const auto getRCRImpl = ServiceContext::declareDecoration<std::unique_ptr<RemoteCommandRunner>>();
 }  // namespace
 
diff --git a/src/mongo/executor/remote_command_runner.h b/src/mongo/executor/remote_command_runner.h
index 2404fdd4152..98ed870b90f 100644
--- a/src/mongo/executor/remote_command_runner.h
+++ b/src/mongo/executor/remote_command_runner.h
@@ -32,6 +32,7 @@
 #include "mongo/bson/bsonobj.h"
 #include "mongo/db/operation_context.h"
 #include "mongo/executor/remote_command_response.h"
+#include "mongo/executor/remote_command_runner_error_info.h"
 #include "mongo/executor/remote_command_targeter.h"
 #include "mongo/executor/task_executor.h"
 #include "mongo/rpc/get_status_from_command_result.h"
@@ -74,6 +75,19 @@ public:
     static RemoteCommandRunner* get(ServiceContext* serviceContext);
     static void set(ServiceContext* serviceContext, std::unique_ptr<RemoteCommandRunner> theRunner);
 };
+
+/**
+ * Returns a RemoteCommandExecutionError with ErrorExtraInfo populated to contain
+ * details about any error, local or remote, contained in `r`.
+ */
+inline Status makeErrorIfNeeded(TaskExecutor::ResponseOnAnyStatus r) {
+    if (r.status.isOK() && getStatusFromCommandResult(r.data).isOK() &&
+        getWriteConcernStatusFromCommandResult(r.data).isOK() &&
+        getFirstWriteErrorStatusFromCommandResult(r.data).isOK()) {
+        return Status::OK();
+    }
+    return {RemoteCommandExecutionErrorInfo(r), "Remote command execution failed"};
+}
 }  // namespace detail
 
 template <typename CommandReplyType>