SERVER-34228 Free Monitoring Replica Set support

19 files changed, 984 insertions, 72 deletions

diff --git a/src/mongo/db/free_mon/SConscript b/src/mongo/db/free_mon/SConscript
index 3bfa66d6f76..72d656dce15 100644
--- a/src/mongo/db/free_mon/SConscript
+++ b/src/mongo/db/free_mon/SConscript
@@ -12,6 +12,7 @@ fmEnv.Library(
     source=[
         'free_mon_processor.cpp',
         'free_mon_queue.cpp',
+        'free_mon_op_observer.cpp',
         'free_mon_storage.cpp',
         'free_mon_controller.cpp',
         env.Idlc('free_mon_protocol.idl')[0],
diff --git a/src/mongo/db/free_mon/free_mon_commands.cpp b/src/mongo/db/free_mon/free_mon_commands.cpp
index c04af62777e..4d6cae49bf4 100644
--- a/src/mongo/db/free_mon/free_mon_commands.cpp
+++ b/src/mongo/db/free_mon/free_mon_commands.cpp
@@ -38,7 +38,7 @@ namespace mongo {
 
 namespace {
 
-const auto kRegisterSyncTimeout = Milliseconds{100};
+const auto kRegisterSyncTimeout = Milliseconds{5000};
 
 /**
  * Indicates the current status of Free Monitoring.
@@ -47,6 +47,10 @@ class GetFreeMonitoringStatusCommand : public BasicCommand {
 public:
     GetFreeMonitoringStatusCommand() : BasicCommand("getFreeMonitoringStatus") {}
 
+    bool adminOnly() const override {
+        return true;
+    }
+
     AllowedOnSecondary secondaryAllowed(ServiceContext*) const final {
         return AllowedOnSecondary::kAlways;
     }
@@ -89,6 +93,10 @@ class SetFreeMonitoringCommand : public BasicCommand {
 public:
     SetFreeMonitoringCommand() : BasicCommand("setFreeMonitoring") {}
 
+    bool adminOnly() const override {
+        return true;
+    }
+
     AllowedOnSecondary secondaryAllowed(ServiceContext*) const final {
         return AllowedOnSecondary::kNever;
     }
@@ -119,6 +127,12 @@ public:
         auto cmd = SetFreeMonitoring::parse(ctx, cmdObj);
 
         auto* controller = FreeMonController::get(opCtx->getServiceContext());
+        if (!controller) {
+            // Pending operation.
+            uasserted(50840,
+                      "Free Monitoring has been disabled via the command-line and/or config file");
+        }
+
         boost::optional<Status> optStatus = boost::none;
         if (cmd.getAction() == SetFreeMonActionEnum::enable) {
             optStatus = controller->registerServerCommand(kRegisterSyncTimeout);
diff --git a/src/mongo/db/free_mon/free_mon_commands_stub.cpp b/src/mongo/db/free_mon/free_mon_commands_stub.cpp
index 77538997889..94f6d80f3ed 100644
--- a/src/mongo/db/free_mon/free_mon_commands_stub.cpp
+++ b/src/mongo/db/free_mon/free_mon_commands_stub.cpp
@@ -46,6 +46,10 @@ class GetFreeMonitoringStatusCommandStub : public BasicCommand {
 public:
     GetFreeMonitoringStatusCommandStub() : BasicCommand("getFreeMonitoringStatus") {}
 
+    bool adminOnly() const override {
+        return true;
+    }
+
     AllowedOnSecondary secondaryAllowed(ServiceContext*) const final {
         return AllowedOnSecondary::kAlways;
     }
diff --git a/src/mongo/db/free_mon/free_mon_controller.cpp b/src/mongo/db/free_mon/free_mon_controller.cpp
index 30e31e5cf99..21200304349 100644
--- a/src/mongo/db/free_mon/free_mon_controller.cpp
+++ b/src/mongo/db/free_mon/free_mon_controller.cpp
@@ -38,6 +38,23 @@
 
 namespace mongo {
 
+namespace {
+
+const auto getFreeMonController =
+    ServiceContext::declareDecoration<std::unique_ptr<FreeMonController>>();
+
+}  // namespace
+
+FreeMonController* FreeMonController::get(ServiceContext* serviceContext) {
+    return getFreeMonController(serviceContext).get();
+}
+
+void FreeMonController::set(ServiceContext* serviceContext,
+                            std::unique_ptr<FreeMonController> controller) {
+    getFreeMonController(serviceContext) = std::move(controller);
+}
+
+
 FreeMonNetworkInterface::~FreeMonNetworkInterface() = default;
 
 void FreeMonController::addRegistrationCollector(
@@ -89,6 +106,24 @@ boost::optional<Status> FreeMonController::unregisterServerCommand(Milliseconds
     return Status::OK();
 }
 
+void FreeMonController::notifyOnUpsert(const BSONObj& doc) {
+    _enqueue(FreeMonMessageWithPayload<FreeMonMessageType::NotifyOnUpsert>::createNow(doc));
+}
+
+
+void FreeMonController::notifyOnDelete() {
+    _enqueue(FreeMonMessage::createNow(FreeMonMessageType::NotifyOnDelete));
+}
+
+
+void FreeMonController::notifyOnTransitionToPrimary() {
+    _enqueue(FreeMonMessage::createNow(FreeMonMessageType::OnTransitionToPrimary));
+}
+
+void FreeMonController::notifyOnRollback() {
+    _enqueue(FreeMonMessage::createNow(FreeMonMessageType::NotifyOnRollback));
+}
+
 void FreeMonController::_enqueue(std::shared_ptr<FreeMonMessage> msg) {
     {
         stdx::lock_guard<stdx::mutex> lock(_mutex);
diff --git a/src/mongo/db/free_mon/free_mon_controller.h b/src/mongo/db/free_mon/free_mon_controller.h
index b2eac2cdda8..e78dea57d34 100644
--- a/src/mongo/db/free_mon/free_mon_controller.h
+++ b/src/mongo/db/free_mon/free_mon_controller.h
@@ -67,7 +67,7 @@ public:
 
     /**
      * Turn the crank of the message queue by ignoring deadlines for N messages.
-    */
+     */
     void turnCrankForTest(size_t countMessagesToIgnore);
 
     /**
@@ -86,6 +86,11 @@ public:
     static FreeMonController* get(ServiceContext* serviceContext);
 
     /**
+     * Set the FreeMonController in the ServiceContext.
+     */
+    static void set(ServiceContext* serviceContext, std::unique_ptr<FreeMonController> controller);
+
+    /**
      * Start registration of mongod with remote service.
      *
      * Only sends one remote registration at a time.
@@ -114,7 +119,28 @@ public:
     // TODO - add these methods
     // void getServerStatus(BSONObjBuilder* builder);
 
-    // void notifyObserver(const BSONObj& doc);
+    /**
+     * Notify on upsert.
+     *
+     * Updates and inserts are treated as the same.
+     */
+    void notifyOnUpsert(const BSONObj& doc);
+
+    /**
+     * Notify on document delete or drop collection.
+     */
+    void notifyOnDelete();
+
+    /**
+     * Notify that we local instance has become a primary.
+     */
+    void notifyOnTransitionToPrimary();
+
+    /**
+     * Notify that storage has rolled back
+     */
+    void notifyOnRollback();
+
 private:
     void _enqueue(std::shared_ptr<FreeMonMessage> msg);
 
diff --git a/src/mongo/db/free_mon/free_mon_controller_test.cpp b/src/mongo/db/free_mon/free_mon_controller_test.cpp
index 9fd66fbf517..54491b1b21c 100644
--- a/src/mongo/db/free_mon/free_mon_controller_test.cpp
+++ b/src/mongo/db/free_mon/free_mon_controller_test.cpp
@@ -45,6 +45,7 @@
 #include "mongo/bson/bsonmisc.h"
 #include "mongo/bson/bsonobjbuilder.h"
 #include "mongo/db/client.h"
+#include "mongo/db/free_mon/free_mon_op_observer.h"
 #include "mongo/db/ftdc/collector.h"
 #include "mongo/db/ftdc/config.h"
 #include "mongo/db/ftdc/constants.h"
@@ -389,9 +390,9 @@ private:
 
 class FreeMonControllerTest : public ServiceContextMongoDTest {
 
-private:
-    void setUp() final;
-    void tearDown() final;
+protected:
+    void setUp() override;
+    void tearDown() override;
 
 protected:
     /**
@@ -434,11 +435,9 @@ void FreeMonControllerTest::setUp() {
     //_storage = stdx::make_unique<repl::StorageInterfaceImpl>();
     repl::StorageInterface::set(service, std::make_unique<repl::StorageInterfaceImpl>());
 
-
     // Transition to PRIMARY so that the server can accept writes.
     ASSERT_OK(_getReplCoord()->setFollowerMode(repl::MemberState::RS_PRIMARY));
 
-
     // Create collection with one document.
     CollectionOptions collectionOptions;
     collectionOptions.uuid = UUID::gen();
@@ -468,6 +467,7 @@ repl::ReplicationCoordinatorMock* FreeMonControllerTest::_getReplCoord() const {
         ASSERT_LTE(__x, target + upper);      \
     }
 
+
 // Positive: Ensure deadlines sort properly
 TEST(FreeMonRetryTest, TestRegistration) {
     PseudoRandom random(0);
@@ -783,6 +783,22 @@ public:
         return inc(2, count);
     }
 
+    Turner& onTransitionToPrimary() {
+        return inc(1, 1);
+    }
+
+    Turner& notifyUpsert() {
+        return inc(1, 1);
+    }
+
+    Turner& notifyDelete() {
+        return inc(1, 1);
+    }
+
+    Turner& notifyOnRollback() {
+        return inc(1, 1);
+    }
+
     operator size_t() {
         return _count;
     }
@@ -1221,12 +1237,289 @@ TEST_F(FreeMonControllerTest, TestMetricBatchingOnErrorRealtime) {
     ASSERT_EQ(controller.network->getLastMetrics().size(), 2UL);
 }
 
-// TODO: Positive: ensure optional fields are rotated
 
-// TODO: Positive: Test Metrics works on command register on primary
-// TODO: Positive: Test Metrics works on startup register on secondary
-// TODO: Positive: Test Metrics works on secondary after opObserver register
-// TODO: Positive: Test Metrics works on secondary after opObserver de-register
+class FreeMonControllerRSTest : public FreeMonControllerTest {
+private:
+    void setUp() final;
+    void tearDown() final;
+};
+
+void FreeMonControllerRSTest::setUp() {
+    FreeMonControllerTest::setUp();
+    auto service = getServiceContext();
+
+    // Set up an OpObserver to exercise repl integration
+    auto opObserver = std::make_unique<FreeMonOpObserver>();
+    auto opObserverRegistry = dynamic_cast<OpObserverRegistry*>(service->getOpObserver());
+    opObserverRegistry->addObserver(std::move(opObserver));
+}
+
+void FreeMonControllerRSTest::tearDown() {
+    FreeMonControllerTest::tearDown();
+}
+
+// Positive: Transition to primary
+TEST_F(FreeMonControllerRSTest, TransitionToPrimary) {
+    ControllerHolder controller(_mockThreadPool.get(), FreeMonNetworkInterfaceMock::Options());
+
+    // Now become a secondary, then primary, and try what happens when we become primary
+    ASSERT_OK(_getReplCoord()->setFollowerMode(repl::MemberState::RS_SECONDARY));
+    ASSERT_OK(_getReplCoord()->setFollowerMode(repl::MemberState::RS_PRIMARY));
+
+    controller.start(RegistrationType::RegisterAfterOnTransitionToPrimary);
+
+    controller->turnCrankForTest(Turner().registerServer().collect(2));
+
+    controller->notifyOnTransitionToPrimary();
+
+    controller->turnCrankForTest(Turner().onTransitionToPrimary().registerCommand());
+
+    ASSERT_TRUE(FreeMonStorage::read(_opCtx.get()).is_initialized());
+
+    ASSERT_EQ(controller.registerCollector->count(), 1UL);
+    ASSERT_GTE(controller.metricsCollector->count(), 2UL);
+}
+
+// Positive: Test metrics works on secondary
+TEST_F(FreeMonControllerRSTest, StartupOnSecondary) {
+    ControllerHolder controller(_mockThreadPool.get(), FreeMonNetworkInterfaceMock::Options());
+
+    FreeMonStorage::replace(_opCtx.get(), initStorage(StorageStateEnum::enabled));
+
+    // Now become a secondary, then primary, and try what happens when we become primary
+    ASSERT_OK(_getReplCoord()->setFollowerMode(repl::MemberState::RS_SECONDARY));
+
+    controller.start(RegistrationType::RegisterAfterOnTransitionToPrimary);
+
+    controller->turnCrankForTest(Turner().registerServer().registerCommand().collect());
+
+    ASSERT_TRUE(FreeMonStorage::read(_opCtx.get()).is_initialized());
+
+    // Validate the new registration id was not written
+    ASSERT_EQ(FreeMonStorage::read(_opCtx.get())->getRegistrationId(), "Foo");
+
+    ASSERT_EQ(controller.registerCollector->count(), 1UL);
+    ASSERT_GTE(controller.metricsCollector->count(), 1UL);
+}
+
+// Positive: Test registration occurs on replicated insert from primary
+TEST_F(FreeMonControllerRSTest, SecondaryStartOnInsert) {
+    ControllerHolder controller(_mockThreadPool.get(), FreeMonNetworkInterfaceMock::Options());
+
+    // Now become a secondary
+    ASSERT_OK(_getReplCoord()->setFollowerMode(repl::MemberState::RS_SECONDARY));
+
+    controller.start(RegistrationType::RegisterAfterOnTransitionToPrimary);
+
+    controller->turnCrankForTest(Turner().registerServer().collect(2));
+
+    controller->notifyOnUpsert(initStorage(StorageStateEnum::enabled).toBSON());
+
+    controller->turnCrankForTest(Turner().notifyUpsert().registerCommand().collect());
+
+    ASSERT_FALSE(FreeMonStorage::read(_opCtx.get()).is_initialized());
+
+    ASSERT_EQ(controller.registerCollector->count(), 1UL);
+    ASSERT_GTE(controller.metricsCollector->count(), 2UL);
+}
+
+// Positive: Test registration occurs on replicated update from primary
+TEST_F(FreeMonControllerRSTest, SecondaryStartOnUpdate) {
+    ControllerHolder controller(_mockThreadPool.get(), FreeMonNetworkInterfaceMock::Options());
+
+    FreeMonStorage::replace(_opCtx.get(), initStorage(StorageStateEnum::pending));
+
+    // Now become a secondary
+    ASSERT_OK(_getReplCoord()->setFollowerMode(repl::MemberState::RS_SECONDARY));
+
+    controller.start(RegistrationType::RegisterAfterOnTransitionToPrimary);
+
+    controller->turnCrankForTest(Turner().registerServer().collect(2));
+
+    controller->notifyOnUpsert(initStorage(StorageStateEnum::enabled).toBSON());
+
+    controller->turnCrankForTest(Turner().notifyUpsert().registerCommand().collect());
+
+    // Since there is no local write, it remains pending
+    ASSERT_TRUE(FreeMonStorage::read(_opCtx.get()).get().getState() == StorageStateEnum::pending);
+
+    ASSERT_EQ(controller.registerCollector->count(), 1UL);
+    ASSERT_GTE(controller.metricsCollector->count(), 2UL);
+}
+
+// Positive: Test Metrics works on secondary after opObserver de-register
+TEST_F(FreeMonControllerRSTest, SecondaryStopOnDeRegister) {
+    ControllerHolder controller(_mockThreadPool.get(), FreeMonNetworkInterfaceMock::Options());
+
+    FreeMonStorage::replace(_opCtx.get(), initStorage(StorageStateEnum::enabled));
+
+    // Now become a secondary
+    ASSERT_OK(_getReplCoord()->setFollowerMode(repl::MemberState::RS_SECONDARY));
+
+    controller.start(RegistrationType::RegisterAfterOnTransitionToPrimary);
+
+    controller->turnCrankForTest(Turner().registerServer().registerCommand().collect(1));
+
+    ASSERT_EQ(controller.metricsCollector->count(), 1UL);
+
+    controller->notifyOnUpsert(initStorage(StorageStateEnum::disabled).toBSON());
+
+    controller->turnCrankForTest(Turner().notifyUpsert().collect().metricsSend());
+
+    ASSERT_TRUE(FreeMonStorage::read(_opCtx.get()).is_initialized());
+
+    // Since there is no local write, it remains enabled
+    ASSERT_TRUE(FreeMonStorage::read(_opCtx.get()).get().getState() == StorageStateEnum::enabled);
+
+    ASSERT_EQ(controller.registerCollector->count(), 1UL);
+    ASSERT_EQ(controller.metricsCollector->count(), 2UL);
+}
+
+// Negative: Tricky: Primary becomes secondary during registration
+TEST_F(FreeMonControllerRSTest, StepdownDuringRegistration) {
+    ControllerHolder controller(_mockThreadPool.get(), FreeMonNetworkInterfaceMock::Options());
+
+    controller.start(RegistrationType::RegisterAfterOnTransitionToPrimary);
+
+    ASSERT_OK(controller->registerServerCommand(Milliseconds::min()));
+
+    controller->turnCrankForTest(Turner().registerServer() + 1);
+
+    ASSERT_TRUE(FreeMonStorage::read(_opCtx.get()).get().getState() == StorageStateEnum::pending);
+
+    // Now become a secondary
+    ASSERT_OK(_getReplCoord()->setFollowerMode(repl::MemberState::RS_SECONDARY));
+
+    // Finish registration
+    controller->turnCrankForTest(1);
+    controller->turnCrankForTest(Turner().metricsSend().collect(1));
+
+    // Registration cannot write back to the local store so remain in pending
+    ASSERT_TRUE(FreeMonStorage::read(_opCtx.get()).get().getState() == StorageStateEnum::pending);
+
+    ASSERT_EQ(controller.registerCollector->count(), 1UL);
+    ASSERT_EQ(controller.metricsCollector->count(), 2UL);
+}
+
+// Negative: Tricky: Primary becomes secondary during metrics send
+TEST_F(FreeMonControllerRSTest, StepdownDuringMetricsSend) {
+    ControllerHolder controller(_mockThreadPool.get(), FreeMonNetworkInterfaceMock::Options());
+
+    controller.start(RegistrationType::RegisterAfterOnTransitionToPrimary);
+
+    ASSERT_OK(controller->registerServerCommand(Milliseconds::min()));
+
+    controller->turnCrankForTest(Turner().registerServer().registerCommand().collect());
+
+    // Finish registration
+    controller->turnCrankForTest(Turner().collect(1) + 1);
+
+    // Now become a secondary
+    ASSERT_OK(_getReplCoord()->setFollowerMode(repl::MemberState::RS_SECONDARY));
+
+    // Finish send
+    controller->turnCrankForTest(1);
+
+    ASSERT_EQ(controller.registerCollector->count(), 1UL);
+    ASSERT_EQ(controller.metricsCollector->count(), 2UL);
+}
+
+// Positive: Test Metrics works on secondary after opObserver delete of document
+TEST_F(FreeMonControllerRSTest, SecondaryStopOnDocumentDrop) {
+    ControllerHolder controller(_mockThreadPool.get(), FreeMonNetworkInterfaceMock::Options());
+
+    FreeMonStorage::replace(_opCtx.get(), initStorage(StorageStateEnum::enabled));
+
+    // Now become a secondary
+    ASSERT_OK(_getReplCoord()->setFollowerMode(repl::MemberState::RS_SECONDARY));
+
+    controller.start(RegistrationType::RegisterAfterOnTransitionToPrimary);
+
+    controller->turnCrankForTest(Turner().registerServer().registerCommand().collect(1));
+
+    ASSERT_EQ(controller.metricsCollector->count(), 1UL);
+
+    controller->notifyOnDelete();
+
+    // There is a race condition where sometimes metrics send sneaks in
+    controller->turnCrankForTest(Turner().notifyDelete().collect(3));
+
+    ASSERT_TRUE(FreeMonStorage::read(_opCtx.get()).is_initialized());
+
+    // Since there is no local write, it remains enabled
+    ASSERT_TRUE(FreeMonStorage::read(_opCtx.get()).get().getState() == StorageStateEnum::enabled);
+
+    ASSERT_EQ(controller.registerCollector->count(), 1UL);
+    ASSERT_GTE(controller.metricsCollector->count(), 2UL);
+}
+
+// Negative: Test nice shutdown on bad update
+TEST_F(FreeMonControllerRSTest, SecondaryStartOnBadUpdate) {
+    ControllerHolder controller(_mockThreadPool.get(), FreeMonNetworkInterfaceMock::Options());
+
+    FreeMonStorage::replace(_opCtx.get(), initStorage(StorageStateEnum::enabled));
+
+    // Now become a secondary
+    ASSERT_OK(_getReplCoord()->setFollowerMode(repl::MemberState::RS_SECONDARY));
+
+    controller.start(RegistrationType::RegisterAfterOnTransitionToPrimary);
+
+    controller->turnCrankForTest(Turner().registerServer().registerCommand().collect(2));
+
+    controller->notifyOnUpsert(BSON("version" << 2LL));
+
+    controller->turnCrankForTest(Turner().notifyUpsert());
+
+    // Since there is no local write, it remains enabled
+    ASSERT_TRUE(FreeMonStorage::read(_opCtx.get()).get().getState() == StorageStateEnum::enabled);
+
+    ASSERT_EQ(controller.registerCollector->count(), 1UL);
+    ASSERT_EQ(controller.metricsCollector->count(), 2UL);
+}
+
+// Positive: On rollback, start registration if needed
+TEST_F(FreeMonControllerRSTest, SecondaryRollbackStopMetrics) {
+    ControllerHolder controller(_mockThreadPool.get(), FreeMonNetworkInterfaceMock::Options());
+
+    FreeMonStorage::replace(_opCtx.get(), initStorage(StorageStateEnum::disabled));
+
+    // Now become a secondary
+    ASSERT_OK(_getReplCoord()->setFollowerMode(repl::MemberState::RS_SECONDARY));
+
+    controller.start(RegistrationType::RegisterAfterOnTransitionToPrimary);
+
+    controller->turnCrankForTest(Turner().registerServer().collect(2));
+
+    ASSERT_EQ(controller.metricsCollector->count(), 2UL);
+
+    // Simulate a rollback by writing out of band
+    // Cheat a little by flipping to primary to allow the write to succeed
+    ASSERT_OK(_getReplCoord()->setFollowerMode(repl::MemberState::RS_PRIMARY));
+    FreeMonStorage::replace(_opCtx.get(), initStorage(StorageStateEnum::enabled));
+    ASSERT_OK(_getReplCoord()->setFollowerMode(repl::MemberState::RS_SECONDARY));
+
+    controller->notifyOnRollback();
+
+    controller->turnCrankForTest(
+        Turner().notifyOnRollback().registerCommand().collect(2).metricsSend());
+
+    // Since there is no local write, it remains enabled
+    ASSERT_TRUE(FreeMonStorage::read(_opCtx.get()).get().getState() == StorageStateEnum::enabled);
+
+    ASSERT_EQ(controller.registerCollector->count(), 1UL);
+    ASSERT_EQ(controller.metricsCollector->count(), 4UL);
+}
+
+// TODO: tricky - OnUpser - disable - OnDelete - make sure registration halts
+// TODO: tricky - OnDelete - make sure registration halts
+
+// TODO: Integration: Tricky - secondary as marked via command line - enableCloudFreeMOnitorig =
+// false but a primary replicates a change to enable it
+
+// TODO: test SSL???
+
+
+// TODO: Positive: ensure optional fields are rotated
 
 }  // namespace
 }  // namespace mongo
diff --git a/src/mongo/db/free_mon/free_mon_message.h b/src/mongo/db/free_mon/free_mon_message.h
index 2247532b3be..070f0957c0a 100644
--- a/src/mongo/db/free_mon/free_mon_message.h
+++ b/src/mongo/db/free_mon/free_mon_message.h
@@ -90,9 +90,25 @@ enum class FreeMonMessageType {
      */
     AsyncMetricsFail,
 
-    // TODO - add replication messages
-    // OnPrimary,
-    // OpObserver,
+    /**
+     * Notify that the node has been made a primary replica.
+     */
+    OnTransitionToPrimary,
+
+    /**
+     * Notify that storage has received an insert or update.
+     */
+    NotifyOnUpsert,
+
+    /**
+     * Notify that storage has received a delete or drop collection.
+     */
+    NotifyOnDelete,
+
+    /**
+     * Notify that storage has been rolled back.
+     */
+    NotifyOnRollback,
 };
 
 /**
@@ -202,6 +218,10 @@ struct FreeMonPayloadForMessage<FreeMonMessageType::AsyncMetricsFail> {
     using payload_type = Status;
 };
 
+template <>
+struct FreeMonPayloadForMessage<FreeMonMessageType::NotifyOnUpsert> {
+    using payload_type = BSONObj;
+};
 
 /**
  * Message with a generic payload based on the type of message.
diff --git a/src/mongo/db/free_mon/free_mon_mongod.cpp b/src/mongo/db/free_mon/free_mon_mongod.cpp
index 186b8d934da..500b8c9f936 100644
--- a/src/mongo/db/free_mon/free_mon_mongod.cpp
+++ b/src/mongo/db/free_mon/free_mon_mongod.cpp
@@ -49,6 +49,7 @@
 #include "mongo/db/free_mon/free_mon_http.h"
 #include "mongo/db/free_mon/free_mon_message.h"
 #include "mongo/db/free_mon/free_mon_network.h"
+#include "mongo/db/free_mon/free_mon_op_observer.h"
 #include "mongo/db/free_mon/free_mon_options.h"
 #include "mongo/db/free_mon/free_mon_protocol_gen.h"
 #include "mongo/db/free_mon/free_mon_storage.h"
@@ -67,19 +68,6 @@
 namespace mongo {
 
 namespace {
-
-const auto getFreeMonController =
-    ServiceContext::declareDecoration<std::unique_ptr<FreeMonController>>();
-
-FreeMonController* getGlobalFreeMonController() {
-    if (!hasGlobalServiceContext()) {
-        return nullptr;
-    }
-
-    return getFreeMonController(getGlobalServiceContext()).get();
-}
-
-
 /**
  * Expose cloudFreeMonitoringEndpointURL set parameter to URL for free monitoring.
  */
@@ -317,12 +305,12 @@ void startFreeMonitoring(ServiceContext* serviceContext) {
 
     auto controller = stdx::make_unique<FreeMonController>(std::move(network));
 
+    auto controllerPtr = controller.get();
+
     registerCollectors(controller.get());
 
     // Install the new controller
-    auto& staticFreeMon = getFreeMonController(serviceContext);
-
-    staticFreeMon = std::move(controller);
+    FreeMonController::set(getGlobalServiceContext(), std::move(controller));
 
     RegistrationType registrationType = RegistrationType::DoNotRegister;
     if (globalFreeMonParams.freeMonitoringState == EnableCloudStateEnum::kOn) {
@@ -333,6 +321,8 @@ void startFreeMonitoring(ServiceContext* serviceContext) {
         } else {
             registrationType = RegistrationType::RegisterOnStart;
         }
+    } else if (globalFreeMonParams.freeMonitoringState == EnableCloudStateEnum::kRuntime) {
+        registrationType = RegistrationType::RegisterAfterOnTransitionToPrimary;
     }
 
     controllerPtr->start(registrationType, globalFreeMonParams.freeMonitoringTags);
@@ -343,15 +333,19 @@ void stopFreeMonitoring() {
         return;
     }
 
-    auto controller = getGlobalFreeMonController();
+    auto controller = FreeMonController::get(getGlobalServiceContext());
 
     if (controller != nullptr) {
         controller->stop();
     }
 }
 
-FreeMonController* FreeMonController::get(ServiceContext* serviceContext) {
-    return getFreeMonController(serviceContext).get();
+void notifyFreeMonitoringOnTransitionToPrimary() {
+    FreeMonController::get(getGlobalServiceContext())->notifyOnTransitionToPrimary();
+}
+
+void setupFreeMonitoringOpObserver(OpObserverRegistry* registry) {
+    registry->addObserver(stdx::make_unique<FreeMonOpObserver>());
 }
 
 FreeMonHttpClientInterface::~FreeMonHttpClientInterface() = default;
diff --git a/src/mongo/db/free_mon/free_mon_mongod.h b/src/mongo/db/free_mon/free_mon_mongod.h
index b4638930cf6..345c5145a24 100644
--- a/src/mongo/db/free_mon/free_mon_mongod.h
+++ b/src/mongo/db/free_mon/free_mon_mongod.h
@@ -28,6 +28,7 @@
 
 #pragma once
 
+#include "mongo/db/op_observer_registry.h"
 #include "mongo/db/service_context.h"
 
 namespace mongo {
@@ -43,4 +44,16 @@ void startFreeMonitoring(ServiceContext* serviceContext);
  */
 void stopFreeMonitoring();
 
+/**
+ * Notify free monitoring about a replica set member becoming primary
+ */
+void notifyFreeMonitoringOnTransitionToPrimary();
+
+/**
+ * Setup Free Monitoring OpObserver.
+ *
+ * Called before free monitoring is started.
+ */
+void setupFreeMonitoringOpObserver(OpObserverRegistry* registry);
+
 }  // namespace mongo
diff --git a/src/mongo/db/free_mon/free_mon_op_observer.cpp b/src/mongo/db/free_mon/free_mon_op_observer.cpp
new file mode 100644
index 00000000000..b44e74f4765
--- /dev/null
+++ b/src/mongo/db/free_mon/free_mon_op_observer.cpp
@@ -0,0 +1,145 @@
+/**
+*    Copyright (C) 2018 MongoDB Inc.
+*
+*    This program is free software: you can redistribute it and/or  modify
+*    it under the terms of the GNU Affero General Public License, version 3,
+*    as published by the Free Software Foundation.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU Affero General Public License for more details.
+*
+*    You should have received a copy of the GNU Affero General Public License
+*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*
+*    As a special exception, the copyright holders give permission to link the
+*    code of portions of this program with the OpenSSL library under certain
+*    conditions as described in each individual source file and distribute
+*    linked combinations including the program with the OpenSSL library. You
+*    must comply with the GNU Affero General Public License in all respects for
+*    all of the code used other than as permitted herein. If you modify file(s)
+*    with this exception, you may extend this exception to your version of the
+*    file(s), but you are not obligated to do so. If you do not wish to do so,
+*    delete this exception statement from your version. If you delete this
+*    exception statement from all source files in the program, then also delete
+*    it in the license file.
+*/
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/db/free_mon/free_mon_op_observer.h"
+
+#include "mongo/db/free_mon/free_mon_controller.h"
+#include "mongo/db/free_mon/free_mon_storage.h"
+
+namespace mongo {
+namespace {
+
+bool isStandaloneOrPrimary(OperationContext* opCtx) {
+    auto replCoord = repl::ReplicationCoordinator::get(opCtx);
+    const bool isReplSet =
+        replCoord->getReplicationMode() == repl::ReplicationCoordinator::modeReplSet;
+    return !isReplSet || (repl::ReplicationCoordinator::get(opCtx)->getMemberState() ==
+                          repl::MemberState::RS_PRIMARY);
+}
+
+}  // namespace
+
+FreeMonOpObserver::FreeMonOpObserver() = default;
+
+FreeMonOpObserver::~FreeMonOpObserver() = default;
+
+repl::OpTime FreeMonOpObserver::onDropCollection(OperationContext* opCtx,
+                                                 const NamespaceString& collectionName,
+                                                 OptionalCollectionUUID uuid) {
+    if (collectionName == NamespaceString::kServerConfigurationNamespace) {
+        auto controller = FreeMonController::get(opCtx->getServiceContext());
+
+        if (controller != nullptr) {
+            controller->notifyOnDelete();
+        }
+    }
+
+    return {};
+}
+
+void FreeMonOpObserver::onInserts(OperationContext* opCtx,
+                                  const NamespaceString& nss,
+                                  OptionalCollectionUUID uuid,
+                                  std::vector<InsertStatement>::const_iterator begin,
+                                  std::vector<InsertStatement>::const_iterator end,
+                                  bool fromMigrate) {
+    if (isStandaloneOrPrimary(opCtx)) {
+        return;
+    }
+
+    for (auto it = begin; it != end; ++it) {
+        const auto& insertedDoc = it->doc;
+
+        if (nss == NamespaceString::kServerConfigurationNamespace) {
+            if (auto idElem = insertedDoc["_id"]) {
+                if (idElem.str() == FreeMonStorage::kFreeMonDocIdKey) {
+                    auto controller = FreeMonController::get(opCtx->getServiceContext());
+
+                    if (controller != nullptr) {
+                        controller->notifyOnUpsert(insertedDoc);
+                    }
+                }
+            }
+        }
+    }
+}
+
+void FreeMonOpObserver::onUpdate(OperationContext* opCtx, const OplogUpdateEntryArgs& args) {
+    if (isStandaloneOrPrimary(opCtx)) {
+        return;
+    }
+
+    if (args.nss == NamespaceString::kServerConfigurationNamespace) {
+        if (args.updatedDoc["_id"].str() == FreeMonStorage::kFreeMonDocIdKey) {
+            auto controller = FreeMonController::get(opCtx->getServiceContext());
+
+            if (controller != nullptr) {
+                controller->notifyOnUpsert(args.updatedDoc);
+            }
+        }
+    }
+}
+
+void FreeMonOpObserver::onDelete(OperationContext* opCtx,
+                                 const NamespaceString& nss,
+                                 OptionalCollectionUUID uuid,
+                                 StmtId stmtId,
+                                 bool fromMigrate,
+                                 const boost::optional<BSONObj>& deletedDoc) {
+    if (isStandaloneOrPrimary(opCtx)) {
+        return;
+    }
+
+    if (nss == NamespaceString::kServerConfigurationNamespace) {
+        if (deletedDoc.get()["_id"].str() == FreeMonStorage::kFreeMonDocIdKey) {
+            auto controller = FreeMonController::get(opCtx->getServiceContext());
+
+            if (controller != nullptr) {
+                controller->notifyOnDelete();
+            }
+        }
+    }
+}
+
+void FreeMonOpObserver::onReplicationRollback(OperationContext* opCtx,
+                                              const RollbackObserverInfo& rbInfo) {
+    // Invalidate any in-memory auth data if necessary.
+    const auto& rollbackNamespaces = rbInfo.rollbackNamespaces;
+    if (rollbackNamespaces.count(NamespaceString::kServerConfigurationNamespace) == 1) {
+        auto controller = FreeMonController::get(opCtx->getServiceContext());
+
+        if (controller != nullptr) {
+            controller->notifyOnRollback();
+        }
+    }
+}
+
+
+}  // namespace mongo
diff --git a/src/mongo/db/free_mon/free_mon_op_observer.h b/src/mongo/db/free_mon/free_mon_op_observer.h
new file mode 100644
index 00000000000..acdb03d4890
--- /dev/null
+++ b/src/mongo/db/free_mon/free_mon_op_observer.h
@@ -0,0 +1,130 @@
+/**
+ *    Copyright (C) 2018 MongoDB Inc.
+ *
+ *    This program is free software: you can redistribute it and/or  modify
+ *    it under the terms of the GNU Affero General Public License, version 3,
+ *    as published by the Free Software Foundation.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Affero General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Affero General Public License
+ *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the GNU Affero General Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#pragma once
+
+#include "mongo/base/disallow_copying.h"
+#include "mongo/db/op_observer.h"
+
+namespace mongo {
+
+/**
+ * OpObserver for Free Monitoring. Observes all secondary replication traffic and filters down to
+ * relevant entries for free monitoring.
+ */
+class FreeMonOpObserver final : public OpObserver {
+    MONGO_DISALLOW_COPYING(FreeMonOpObserver);
+
+public:
+    FreeMonOpObserver();
+    ~FreeMonOpObserver();
+
+    void onCreateIndex(OperationContext* opCtx,
+                       const NamespaceString& nss,
+                       OptionalCollectionUUID uuid,
+                       BSONObj indexDoc,
+                       bool fromMigrate) final {}
+
+    void onInserts(OperationContext* opCtx,
+                   const NamespaceString& nss,
+                   OptionalCollectionUUID uuid,
+                   std::vector<InsertStatement>::const_iterator begin,
+                   std::vector<InsertStatement>::const_iterator end,
+                   bool fromMigrate) final;
+
+    void onUpdate(OperationContext* opCtx, const OplogUpdateEntryArgs& args) final;
+
+    void aboutToDelete(OperationContext* opCtx,
+                       const NamespaceString& nss,
+                       const BSONObj& doc) final {}
+
+    void onDelete(OperationContext* opCtx,
+                  const NamespaceString& nss,
+                  OptionalCollectionUUID uuid,
+                  StmtId stmtId,
+                  bool fromMigrate,
+                  const boost::optional<BSONObj>& deletedDoc) final;
+
+    void onInternalOpMessage(OperationContext* opCtx,
+                             const NamespaceString& nss,
+                             const boost::optional<UUID> uuid,
+                             const BSONObj& msgObj,
+                             const boost::optional<BSONObj> o2MsgObj) final {}
+
+    void onCreateCollection(OperationContext* opCtx,
+                            Collection* coll,
+                            const NamespaceString& collectionName,
+                            const CollectionOptions& options,
+                            const BSONObj& idIndex) final {}
+
+    void onCollMod(OperationContext* opCtx,
+                   const NamespaceString& nss,
+                   OptionalCollectionUUID uuid,
+                   const BSONObj& collModCmd,
+                   const CollectionOptions& oldCollOptions,
+                   boost::optional<TTLCollModInfo> ttlInfo) final {}
+
+    void onDropDatabase(OperationContext* opCtx, const std::string& dbName) final {}
+
+    repl::OpTime onDropCollection(OperationContext* opCtx,
+                                  const NamespaceString& collectionName,
+                                  OptionalCollectionUUID uuid) final;
+
+    void onDropIndex(OperationContext* opCtx,
+                     const NamespaceString& nss,
+                     OptionalCollectionUUID uuid,
+                     const std::string& indexName,
+                     const BSONObj& indexInfo) final {}
+
+    repl::OpTime onRenameCollection(OperationContext* opCtx,
+                                    const NamespaceString& fromCollection,
+                                    const NamespaceString& toCollection,
+                                    OptionalCollectionUUID uuid,
+                                    OptionalCollectionUUID dropTargetUUID,
+                                    bool stayTemp) final {
+        return repl::OpTime();
+    }
+
+    void onApplyOps(OperationContext* opCtx,
+                    const std::string& dbName,
+                    const BSONObj& applyOpCmd) final {}
+
+    void onEmptyCapped(OperationContext* opCtx,
+                       const NamespaceString& collectionName,
+                       OptionalCollectionUUID uuid) final {}
+
+    void onTransactionCommit(OperationContext* opCtx) final {}
+
+    void onTransactionPrepare(OperationContext* opCtx) final {}
+
+    void onTransactionAbort(OperationContext* opCtx) final {}
+
+    void onReplicationRollback(OperationContext* opCtx, const RollbackObserverInfo& rbInfo);
+};
+
+}  // namespace mongo
diff --git a/src/mongo/db/free_mon/free_mon_processor.cpp b/src/mongo/db/free_mon/free_mon_processor.cpp
index e5db7630cb8..04620d1cf23 100644
--- a/src/mongo/db/free_mon/free_mon_processor.cpp
+++ b/src/mongo/db/free_mon/free_mon_processor.cpp
@@ -53,6 +53,7 @@ namespace mongo {
 namespace {
 
 constexpr auto kProtocolVersion = 1;
+constexpr auto kStorageVersion = 1;
 
 constexpr auto kRegistrationIdMaxLength = 4096;
 constexpr auto kInformationalURLMaxLength = 4096;
@@ -224,6 +225,26 @@ void FreeMonProcessor::run() {
                             msg.get()));
                     break;
                 }
+                case FreeMonMessageType::OnTransitionToPrimary: {
+                    doOnTransitionToPrimary(client);
+                    break;
+                }
+                case FreeMonMessageType::NotifyOnUpsert: {
+                    doNotifyOnUpsert(
+                        client,
+                        checked_cast<
+                            FreeMonMessageWithPayload<FreeMonMessageType::NotifyOnUpsert>*>(
+                            msg.get()));
+                    break;
+                }
+                case FreeMonMessageType::NotifyOnDelete: {
+                    doNotifyOnDelete(client);
+                    break;
+                }
+                case FreeMonMessageType::NotifyOnRollback: {
+                    doNotifyOnRollback(client);
+                    break;
+                }
                 default:
                     MONGO_UNREACHABLE;
             }
@@ -256,7 +277,7 @@ void FreeMonProcessor::readState(Client* client) {
     } else if (!state.is_initialized()) {
         // Default the state
         _state.setVersion(kProtocolVersion);
-        _state.setState(StorageStateEnum::enabled);
+        _state.setState(StorageStateEnum::disabled);
         _state.setRegistrationId("");
         _state.setInformationalURL("");
         _state.setMessage("");
@@ -312,7 +333,7 @@ void FreeMonProcessor::doServerRegister(
         // 2. a standalone which has never been registered
         //
         if (!state.is_initialized()) {
-            // TODO: hook OnTransitionToPrimary
+            _registerOnTransitionToPrimary = true;
         } else {
             // We are standalone, if we have a registration id, then send a registration
             // notification, else wait for the user to register us
@@ -324,7 +345,7 @@ void FreeMonProcessor::doServerRegister(
         MONGO_UNREACHABLE;
     }
 
-    // Enqueue the first metrics gather
+    // Enqueue the first metrics gather unless we are not going to register
     enqueue(FreeMonMessage::createNow(FreeMonMessageType::MetricsCollect));
 }
 
@@ -768,7 +789,6 @@ void FreeMonProcessor::doAsyncMetricsComplete(
     // TODO: do we reset only the metrics we send or all pending on success?
 
     _metricsBuffer.reset();
-    _metricsRetry.setMin(Seconds(resp.getReportingInterval()));
 
     if (resp.getId().is_initialized()) {
         _state.setRegistrationId(resp.getId().get());
@@ -790,6 +810,7 @@ void FreeMonProcessor::doAsyncMetricsComplete(
     writeState(client);
 
     // Reset retry counter
+    _metricsRetry.setMin(Seconds(resp.getReportingInterval()));
     _metricsRetry.reset();
 
     // Enqueue next metrics upload
@@ -814,4 +835,86 @@ void FreeMonProcessor::doAsyncMetricsFail(
                                                _metricsRetry.getNextDeadline(client)));
 }
 
+void FreeMonProcessor::doOnTransitionToPrimary(Client* client) {
+    if (_registerOnTransitionToPrimary) {
+        enqueue(FreeMonRegisterCommandMessage::createNow(std::vector<std::string>()));
+
+        // On transition to primary once
+        _registerOnTransitionToPrimary = false;
+    }
+}
+
+void FreeMonProcessor::processInMemoryStateChange(const FreeMonStorageState& originalState,
+                                                  const FreeMonStorageState& newState) {
+    // Are we transition from disabled -> enabled?
+    if (originalState.getState() != newState.getState()) {
+        if (originalState.getState() != StorageStateEnum::enabled &&
+            newState.getState() == StorageStateEnum::enabled) {
+
+            // Secondary needs to start registration
+            enqueue(FreeMonRegisterCommandMessage::createNow(std::vector<std::string>()));
+        }
+    }
+}
+
+
+void FreeMonProcessor::doNotifyOnUpsert(
+    Client* client, const FreeMonMessageWithPayload<FreeMonMessageType::NotifyOnUpsert>* msg) {
+    try {
+        const BSONObj& doc = msg->getPayload();
+        auto newState = FreeMonStorageState::parse(IDLParserErrorContext("free_mon_storage"), doc);
+
+        // Likely, the update changed something
+        if (newState != _state) {
+            uassert(50839,
+                    str::stream() << "Unexpected free monitoring storage version "
+                                  << newState.getVersion(),
+                    newState.getVersion() == kStorageVersion);
+
+            processInMemoryStateChange(_state, newState);
+
+            // Note: enabled -> disabled is handled implicitly by register and send metrics checks
+            // after _state is updated below
+
+            // Copy the fields
+            _state = newState;
+        }
+
+    } catch (...) {
+
+        // Stop the queue
+        _queue.stop();
+
+        warning() << "Uncaught exception in '" << exceptionToStatus()
+                  << "' in free monitoring op observer. Shutting down the "
+                     "free monitoring subsystem.";
+    }
+}
+
+void FreeMonProcessor::doNotifyOnDelete(Client* client) {
+    // The config document was either deleted or the entire collection was dropped, we treat them
+    // the same and stop free monitoring. We continue collecting though.
+
+    // So we mark the internal state as disabled which stop registration and metrics send
+    _state.setState(StorageStateEnum::disabled);
+}
+
+void FreeMonProcessor::doNotifyOnRollback(Client* client) {
+    // We have rolled back, the state on disk reflects our new reality
+    // We should re-read the disk state and proceed.
+
+    // copy the in-memory state
+    auto originalState = _state;
+
+    // Re-read state from disk
+    readState(client);
+
+    auto& newState = _state;
+
+    if (newState != originalState) {
+        processInMemoryStateChange(originalState, newState);
+    }
+}
+
+
 }  // namespace mongo
diff --git a/src/mongo/db/free_mon/free_mon_processor.h b/src/mongo/db/free_mon/free_mon_processor.h
index ade74fc09e6..64734d3a981 100644
--- a/src/mongo/db/free_mon/free_mon_processor.h
+++ b/src/mongo/db/free_mon/free_mon_processor.h
@@ -375,6 +375,34 @@ private:
     void doAsyncMetricsFail(
         Client* client, const FreeMonMessageWithPayload<FreeMonMessageType::AsyncMetricsFail>* msg);
 
+    /**
+     * Process a change to become a replica set primary
+     */
+    void doOnTransitionToPrimary(Client* client);
+
+    /**
+     * Process a notification that storage has received insert or update.
+     */
+    void doNotifyOnUpsert(Client* client,
+                          const FreeMonMessageWithPayload<FreeMonMessageType::NotifyOnUpsert>* msg);
+
+    /**
+     * Process a notification that storage has received delete or drop collection.
+     */
+    void doNotifyOnDelete(Client* client);
+
+
+    /**
+     * Process a notification that storage has rolled back.
+     */
+    void doNotifyOnRollback(Client* client);
+
+    /**
+     * Process a in-memory state transition of state.
+     */
+    void processInMemoryStateChange(const FreeMonStorageState& originalState,
+                                    const FreeMonStorageState& newState);
+
 private:
     // Collection of collectors to send on registration
     FreeMonCollectorCollection& _registration;
@@ -412,6 +440,9 @@ private:
     // Last read storage state
     boost::optional<FreeMonStorageState> _lastReadState;
 
+    // When we change to primary, do we register?
+    bool _registerOnTransitionToPrimary{false};
+
     // Pending update to disk
     FreeMonStorageState _state;
 
diff --git a/src/mongo/db/free_mon/free_mon_storage.cpp b/src/mongo/db/free_mon/free_mon_storage.cpp
index d7ee1499e37..978be046eba 100644
--- a/src/mongo/db/free_mon/free_mon_storage.cpp
+++ b/src/mongo/db/free_mon/free_mon_storage.cpp
@@ -36,6 +36,7 @@
 #include "mongo/db/concurrency/lock_manager_defs.h"
 #include "mongo/db/namespace_string.h"
 #include "mongo/db/operation_context.h"
+#include "mongo/db/repl/replication_coordinator.h"
 #include "mongo/db/repl/storage_interface.h"
 #include "mongo/util/assert_util.h"
 
@@ -43,26 +44,28 @@ namespace mongo {
 
 namespace {
 
-static const NamespaceString adminSystemVersionNss("admin.system.version");
-constexpr auto kFreeMonDocIdKey = "free_monitoring";
-
 // mms-automation stores its document in local.clustermanager
 static const NamespaceString localClusterManagerNss("local.clustermanager");
 
 }  // namespace
 
+constexpr StringData FreeMonStorage::kFreeMonDocIdKey;
+
 boost::optional<FreeMonStorageState> FreeMonStorage::read(OperationContext* opCtx) {
     BSONObj deleteKey = BSON("_id" << kFreeMonDocIdKey);
     BSONElement elementKey = deleteKey.firstElement();
 
     auto storageInterface = repl::StorageInterface::get(opCtx);
 
-    Lock::DBLock dblk(opCtx, adminSystemVersionNss.db(), MODE_IS);
-    Lock::CollectionLock lk(opCtx->lockState(), adminSystemVersionNss.ns(), MODE_IS);
+    Lock::DBLock dblk(opCtx, NamespaceString::kServerConfigurationNamespace.db(), MODE_IS);
+    Lock::CollectionLock lk(
+        opCtx->lockState(), NamespaceString::kServerConfigurationNamespace.ns(), MODE_IS);
 
-    auto swObj = storageInterface->findById(opCtx, adminSystemVersionNss, elementKey);
+    auto swObj = storageInterface->findById(
+        opCtx, NamespaceString::kServerConfigurationNamespace, elementKey);
     if (!swObj.isOK()) {
-        if (swObj.getStatus() == ErrorCodes::NoSuchKey) {
+        if (swObj.getStatus() == ErrorCodes::NoSuchKey ||
+            swObj.getStatus() == ErrorCodes::NamespaceNotFound) {
             return {};
         }
 
@@ -80,12 +83,17 @@ void FreeMonStorage::replace(OperationContext* opCtx, const FreeMonStorageState&
 
     auto storageInterface = repl::StorageInterface::get(opCtx);
     {
-        Lock::DBLock dblk(opCtx, adminSystemVersionNss.db(), MODE_IS);
-        Lock::CollectionLock lk(opCtx->lockState(), adminSystemVersionNss.ns(), MODE_IS);
-
-        auto swObj = storageInterface->upsertById(opCtx, adminSystemVersionNss, elementKey, obj);
-        if (!swObj.isOK()) {
-            uassertStatusOK(swObj);
+        Lock::DBLock dblk(opCtx, NamespaceString::kServerConfigurationNamespace.db(), MODE_IS);
+        Lock::CollectionLock lk(
+            opCtx->lockState(), NamespaceString::kServerConfigurationNamespace.ns(), MODE_IS);
+
+        if (repl::ReplicationCoordinator::get(opCtx)->canAcceptWritesFor(
+                opCtx, NamespaceString::kServerConfigurationNamespace)) {
+            auto swObj = storageInterface->upsertById(
+                opCtx, NamespaceString::kServerConfigurationNamespace, elementKey, obj);
+            if (!swObj.isOK()) {
+                uassertStatusOK(swObj);
+            }
         }
     }
 }
@@ -96,17 +104,23 @@ void FreeMonStorage::deleteState(OperationContext* opCtx) {
 
     auto storageInterface = repl::StorageInterface::get(opCtx);
     {
-        Lock::DBLock dblk(opCtx, adminSystemVersionNss.db(), MODE_IS);
-        Lock::CollectionLock lk(opCtx->lockState(), adminSystemVersionNss.ns(), MODE_IS);
-
-        auto swObj = storageInterface->deleteById(opCtx, adminSystemVersionNss, elementKey);
-        if (!swObj.isOK()) {
-            // Ignore errors about no document
-            if (swObj.getStatus() == ErrorCodes::NoSuchKey) {
-                return;
+        Lock::DBLock dblk(opCtx, NamespaceString::kServerConfigurationNamespace.db(), MODE_IS);
+        Lock::CollectionLock lk(
+            opCtx->lockState(), NamespaceString::kServerConfigurationNamespace.ns(), MODE_IS);
+
+        if (repl::ReplicationCoordinator::get(opCtx)->canAcceptWritesFor(
+                opCtx, NamespaceString::kServerConfigurationNamespace)) {
+
+            auto swObj = storageInterface->deleteById(
+                opCtx, NamespaceString::kServerConfigurationNamespace, elementKey);
+            if (!swObj.isOK()) {
+                // Ignore errors about no document
+                if (swObj.getStatus() == ErrorCodes::NoSuchKey) {
+                    return;
+                }
+
+                uassertStatusOK(swObj);
             }
-
-            uassertStatusOK(swObj);
         }
     }
 }
@@ -114,8 +128,9 @@ void FreeMonStorage::deleteState(OperationContext* opCtx) {
 boost::optional<BSONObj> FreeMonStorage::readClusterManagerState(OperationContext* opCtx) {
     auto storageInterface = repl::StorageInterface::get(opCtx);
 
-    Lock::DBLock dblk(opCtx, adminSystemVersionNss.db(), MODE_IS);
-    Lock::CollectionLock lk(opCtx->lockState(), adminSystemVersionNss.ns(), MODE_IS);
+    Lock::DBLock dblk(opCtx, NamespaceString::kServerConfigurationNamespace.db(), MODE_IS);
+    Lock::CollectionLock lk(
+        opCtx->lockState(), NamespaceString::kServerConfigurationNamespace.ns(), MODE_IS);
 
     auto swObj = storageInterface->findSingleton(opCtx, localClusterManagerNss);
     if (!swObj.isOK()) {
diff --git a/src/mongo/db/free_mon/free_mon_storage.h b/src/mongo/db/free_mon/free_mon_storage.h
index cc2fc47ff6d..ce7066bc56c 100644
--- a/src/mongo/db/free_mon/free_mon_storage.h
+++ b/src/mongo/db/free_mon/free_mon_storage.h
@@ -42,6 +42,11 @@ namespace mongo {
 class FreeMonStorage {
 public:
     /**
+     * The _id value in admin.system.version.
+     */
+    static constexpr auto kFreeMonDocIdKey = "free_monitoring"_sd;
+
+    /**
      * Reads document from disk if it exists.
      */
     static boost::optional<FreeMonStorageState> read(OperationContext* opCtx);
diff --git a/src/mongo/db/free_mon/free_mon_storage.idl b/src/mongo/db/free_mon/free_mon_storage.idl
index 2bd4f15185a..2c4cc424135 100644
--- a/src/mongo/db/free_mon/free_mon_storage.idl
+++ b/src/mongo/db/free_mon/free_mon_storage.idl
@@ -43,6 +43,7 @@ structs:
           state:
             description: "Indicates whether it is disabled or enabled"
             type: StorageState
+            default: disabled
           registrationId:
             description: "Registration Id"
             type: string
diff --git a/src/mongo/db/free_mon/free_mon_storage_test.cpp b/src/mongo/db/free_mon/free_mon_storage_test.cpp
index 550691f2a24..6f110881295 100644
--- a/src/mongo/db/free_mon/free_mon_storage_test.cpp
+++ b/src/mongo/db/free_mon/free_mon_storage_test.cpp
@@ -85,13 +85,6 @@ void FreeMonStorageTest::setUp() {
 
     // Transition to PRIMARY so that the server can accept writes.
     ASSERT_OK(_getReplCoord()->setFollowerMode(repl::MemberState::RS_PRIMARY));
-
-    // Create collection with one document.
-    CollectionOptions collectionOptions;
-    collectionOptions.uuid = UUID::gen();
-    auto statusCC = _storage->createCollection(
-        _opCtx.get(), NamespaceString("admin", "system.version"), collectionOptions);
-    ASSERT_OK(statusCC);
 }
 
 void FreeMonStorageTest::tearDown() {
@@ -110,6 +103,20 @@ repl::ReplicationCoordinatorMock* FreeMonStorageTest::_getReplCoord() const {
 // Positive: Test Storage works
 TEST_F(FreeMonStorageTest, TestStorage) {
 
+    // Validate no collection works
+    {
+        auto emptyDoc = FreeMonStorage::read(_opCtx.get());
+        ASSERT_FALSE(emptyDoc.is_initialized());
+    }
+
+    // Create collection with one document.
+    CollectionOptions collectionOptions;
+    collectionOptions.uuid = UUID::gen();
+    auto statusCC = _storage->createCollection(
+        _opCtx.get(), NamespaceString("admin", "system.version"), collectionOptions);
+    ASSERT_OK(statusCC);
+
+
     FreeMonStorageState initialState =
         FreeMonStorageState::parse(IDLParserErrorContext("foo"),
                                    BSON("version" << 1LL << "state"
@@ -149,6 +156,77 @@ TEST_F(FreeMonStorageTest, TestStorage) {
     FreeMonStorage::deleteState(_opCtx.get());
 }
 
+
+// Positive: Test Storage works on a secondary
+TEST_F(FreeMonStorageTest, TestSecondary) {
+
+    // Create collection with one document.
+    CollectionOptions collectionOptions;
+    collectionOptions.uuid = UUID::gen();
+    auto statusCC = _storage->createCollection(
+        _opCtx.get(), NamespaceString("admin", "system.version"), collectionOptions);
+    ASSERT_OK(statusCC);
+
+
+    FreeMonStorageState initialState =
+        FreeMonStorageState::parse(IDLParserErrorContext("foo"),
+                                   BSON("version" << 1LL << "state"
+                                                  << "enabled"
+                                                  << "registrationId"
+                                                  << "1234"
+                                                  << "informationalURL"
+                                                  << "http://example.com"
+                                                  << "message"
+                                                  << "hello"
+                                                  << "userReminder"
+                                                  << ""));
+
+    FreeMonStorage::replace(_opCtx.get(), initialState);
+
+    {
+        auto persistedDoc = FreeMonStorage::read(_opCtx.get());
+
+        ASSERT_TRUE(persistedDoc.is_initialized());
+
+        ASSERT_TRUE(persistedDoc == initialState);
+    }
+
+    // Now become a secondary
+    ASSERT_OK(_getReplCoord()->setFollowerMode(repl::MemberState::RS_SECONDARY));
+
+    FreeMonStorageState updatedState =
+        FreeMonStorageState::parse(IDLParserErrorContext("foo"),
+                                   BSON("version" << 2LL << "state"
+                                                  << "enabled"
+                                                  << "registrationId"
+                                                  << "1234"
+                                                  << "informationalURL"
+                                                  << "http://example.com"
+                                                  << "message"
+                                                  << "hello"
+                                                  << "userReminder"
+                                                  << ""));
+
+
+    {
+        auto persistedDoc = FreeMonStorage::read(_opCtx.get());
+
+        ASSERT_TRUE(persistedDoc.is_initialized());
+
+        ASSERT_TRUE(persistedDoc == initialState);
+    }
+
+    FreeMonStorage::deleteState(_opCtx.get());
+
+    {
+        auto persistedDoc = FreeMonStorage::read(_opCtx.get());
+        ASSERT_TRUE(persistedDoc.is_initialized());
+    }
+
+    // Verfiy delete of nothing succeeds
+    FreeMonStorage::deleteState(_opCtx.get());
+}
+
 void insertDoc(OperationContext* optCtx, const NamespaceString nss, StringData id) {
     auto storageInterface = repl::StorageInterface::get(optCtx);
 
diff --git a/src/mongo/db/free_mon/free_mon_stub.cpp b/src/mongo/db/free_mon/free_mon_stub.cpp
index bc44e1dd9e4..9a3880ce08b 100644
--- a/src/mongo/db/free_mon/free_mon_stub.cpp
+++ b/src/mongo/db/free_mon/free_mon_stub.cpp
@@ -36,4 +36,8 @@ void startFreeMonitoring(ServiceContext* serviceContext) {}
 
 void stopFreeMonitoring() {}
 
+void notifyFreeMonitoringOnTransitionToPrimary(){};
+
+void setupFreeMonitoringOpObserver(OpObserverRegistry* registry) {}
+
 }  // namespace mongo
diff --git a/src/mongo/db/free_mon/http_client_curl.cpp b/src/mongo/db/free_mon/http_client_curl.cpp
index b275804cdcb..01ace7ed205 100644
--- a/src/mongo/db/free_mon/http_client_curl.cpp
+++ b/src/mongo/db/free_mon/http_client_curl.cpp
@@ -211,7 +211,7 @@ private:
             }
 
             long statusCode;
-            result = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &statusCode);
+            result = curl_easy_getinfo(myHandle.get(), CURLINFO_RESPONSE_CODE, &statusCode);
             if (result != CURLE_OK) {
                 shared_promise.setError({ErrorCodes::OperationFailed,
                                          str::stream() << "Unexpected error retrieving response: "