Merge pull request #34040 from dillaman/wip-44396-partial-fix

rbd-mirror: snapshot-based mirroring should use image sync throttler

Reviewed-by: Mykola Golub <mgolub@suse.com>

12 files changed, 280 insertions, 82 deletions

diff --git a/src/librbd/api/Trash.cc b/src/librbd/api/Trash.cc
index 7d615050a25..720be6f55aa 100644
--- a/src/librbd/api/Trash.cc
+++ b/src/librbd/api/Trash.cc
@@ -158,7 +158,7 @@ int Trash<I>::move(librados::IoCtx &io_ctx, rbd_trash_image_source_t source,
       ictx->exclusive_lock->block_requests(0);
 
       r = ictx->operations->prepare_image_update(
-        exclusive_lock::OPERATION_REQUEST_TYPE_GENERAL, false);
+        exclusive_lock::OPERATION_REQUEST_TYPE_GENERAL, true);
       if (r < 0) {
         lderr(cct) << "cannot obtain exclusive lock - not removing" << dendl;
         ictx->owner_lock.unlock_shared();
diff --git a/src/test/rbd_mirror/image_replayer/snapshot/test_mock_Replayer.cc b/src/test/rbd_mirror/image_replayer/snapshot/test_mock_Replayer.cc
index 47e232e9f06..b2e95bad097 100644
--- a/src/test/rbd_mirror/image_replayer/snapshot/test_mock_Replayer.cc
+++ b/src/test/rbd_mirror/image_replayer/snapshot/test_mock_Replayer.cc
@@ -8,6 +8,7 @@
 #include "librbd/mirror/snapshot/GetImageStateRequest.h"
 #include "librbd/mirror/snapshot/ImageMeta.h"
 #include "librbd/mirror/snapshot/UnlinkPeerRequest.h"
+#include "tools/rbd_mirror/InstanceWatcher.h"
 #include "tools/rbd_mirror/Threads.h"
 #include "tools/rbd_mirror/image_replayer/CloseImageRequest.h"
 #include "tools/rbd_mirror/image_replayer/ReplayerListener.h"
@@ -219,6 +220,14 @@ namespace rbd {
 namespace mirror {
 
 template <>
+struct InstanceWatcher<librbd::MockTestImageCtx> {
+  MOCK_METHOD1(cancel_sync_request, void(const std::string&));
+  MOCK_METHOD2(notify_sync_request, void(const std::string&,
+                                         Context*));
+  MOCK_METHOD1(notify_sync_complete, void(const std::string&));
+};
+
+template <>
 struct Threads<librbd::MockTestImageCtx> {
   MockSafeTimer *timer;
   ceph::mutex &timer_lock;
@@ -349,6 +358,7 @@ public:
   typedef Replayer<librbd::MockTestImageCtx> MockReplayer;
   typedef ApplyImageStateRequest<librbd::MockTestImageCtx> MockApplyImageStateRequest;
   typedef StateBuilder<librbd::MockTestImageCtx> MockStateBuilder;
+  typedef InstanceWatcher<librbd::MockTestImageCtx> MockInstanceWatcher;
   typedef Threads<librbd::MockTestImageCtx> MockThreads;
   typedef CloseImageRequest<librbd::MockTestImageCtx> MockCloseImageRequest;
   typedef librbd::deep_copy::ImageCopyRequest<librbd::MockTestImageCtx> MockImageCopyRequest;
@@ -491,6 +501,19 @@ public:
       }));
   }
 
+  void expect_notify_sync_request(MockInstanceWatcher& mock_instance_watcher,
+                                  const std::string& image_id, int r) {
+    EXPECT_CALL(mock_instance_watcher, notify_sync_request(image_id, _))
+      .WillOnce(WithArg<1>(Invoke([this, r](Context* ctx) {
+        m_threads->work_queue->queue(ctx, r);
+      })));
+  }
+
+  void expect_notify_sync_complete(MockInstanceWatcher& mock_instance_watcher,
+                                   const std::string& image_id) {
+    EXPECT_CALL(mock_instance_watcher, notify_sync_complete(image_id));
+  }
+
   void expect_image_copy(MockImageCopyRequest& mock_image_copy_request,
                          uint64_t src_snap_id_start, uint64_t src_snap_id_end,
                          uint64_t dst_snap_id_start,
@@ -635,13 +658,14 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, InitShutDown) {
 
   InSequence seq;
 
+  MockInstanceWatcher mock_instance_watcher;
   MockImageMeta mock_image_meta;
   MockStateBuilder mock_state_builder(mock_local_image_ctx,
                                       mock_remote_image_ctx,
                                       mock_image_meta);
-  MockReplayer mock_replayer{&mock_threads, "local mirror uuid",
-                             &m_pool_meta_cache, &mock_state_builder,
-                             &mock_replayer_listener};
+  MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
+                             "local mirror uuid", &m_pool_meta_cache,
+                             &mock_state_builder, &mock_replayer_listener};
   m_pool_meta_cache.set_remote_pool_meta(
     m_remote_io_ctx.get_id(),
     {"remote mirror uuid", "remote mirror peer uuid"});
@@ -688,13 +712,14 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, SyncSnapshot) {
 
   InSequence seq;
 
+  MockInstanceWatcher mock_instance_watcher;
   MockImageMeta mock_image_meta;
   MockStateBuilder mock_state_builder(mock_local_image_ctx,
                                       mock_remote_image_ctx,
                                       mock_image_meta);
-  MockReplayer mock_replayer{&mock_threads, "local mirror uuid",
-                             &m_pool_meta_cache, &mock_state_builder,
-                             &mock_replayer_listener};
+  MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
+                             "local mirror uuid", &m_pool_meta_cache,
+                             &mock_state_builder, &mock_replayer_listener};
   m_pool_meta_cache.set_remote_pool_meta(
     m_remote_io_ctx.get_id(),
     {"remote mirror uuid", "remote mirror peer uuid"});
@@ -720,6 +745,7 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, SyncSnapshot) {
   expect_create_non_primary_request(mock_create_non_primary_request,
                                     false, "remote mirror uuid", 1,
                                     {{1, CEPH_NOSNAP}}, 11, 0);
+  expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0);
   MockImageCopyRequest mock_image_copy_request;
   expect_image_copy(mock_image_copy_request, 0, 1, 0, {},
                     {{1, CEPH_NOSNAP}}, 0);
@@ -728,6 +754,7 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, SyncSnapshot) {
   expect_mirror_image_snapshot_set_copy_progress(
     mock_local_image_ctx, 11, true, 0, 0);
   expect_notify_update(mock_local_image_ctx);
+  expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id);
 
   // sync snap4
   expect_load_image_meta(mock_image_meta, false, 0);
@@ -746,6 +773,7 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, SyncSnapshot) {
   expect_create_non_primary_request(mock_create_non_primary_request,
                                     false, "remote mirror uuid", 4,
                                     {{1, 11}, {4, CEPH_NOSNAP}}, 14, 0);
+  expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0);
   expect_image_copy(mock_image_copy_request, 1, 4, 11, {},
                     {{1, 11}, {4, CEPH_NOSNAP}}, 0);
   expect_apply_image_state(mock_apply_state_request, 0);
@@ -755,6 +783,7 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, SyncSnapshot) {
   MockUnlinkPeerRequest mock_unlink_peer_request;
   expect_unlink_peer(mock_unlink_peer_request, 1, "remote mirror peer uuid",
                      0);
+  expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id);
 
   // idle
   expect_load_image_meta(mock_image_meta, false, 0);
@@ -798,13 +827,14 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, InterruptedSync) {
 
   InSequence seq;
 
+  MockInstanceWatcher mock_instance_watcher;
   MockImageMeta mock_image_meta;
   MockStateBuilder mock_state_builder(mock_local_image_ctx,
                                       mock_remote_image_ctx,
                                       mock_image_meta);
-  MockReplayer mock_replayer{&mock_threads, "local mirror uuid",
-                             &m_pool_meta_cache, &mock_state_builder,
-                             &mock_replayer_listener};
+  MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
+                             "local mirror uuid", &m_pool_meta_cache,
+                             &mock_state_builder, &mock_replayer_listener};
   m_pool_meta_cache.set_remote_pool_meta(
     m_remote_io_ctx.get_id(),
     {"remote mirror uuid", "remote mirror peer uuid"});
@@ -835,6 +865,7 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, InterruptedSync) {
   expect_is_refresh_required(mock_remote_image_ctx, false);
   MockGetImageStateRequest mock_get_image_state_request;
   expect_get_image_state(mock_get_image_state_request, 11, 0);
+  expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0);
   MockImageCopyRequest mock_image_copy_request;
   expect_image_copy(mock_image_copy_request, 0, 1, 0,
                     librbd::deep_copy::ObjectNumber{123U},
@@ -880,13 +911,14 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, RemoteImageDemoted) {
 
   InSequence seq;
 
+  MockInstanceWatcher mock_instance_watcher;
   MockImageMeta mock_image_meta;
   MockStateBuilder mock_state_builder(mock_local_image_ctx,
                                       mock_remote_image_ctx,
                                       mock_image_meta);
-  MockReplayer mock_replayer{&mock_threads, "local mirror uuid",
-                             &m_pool_meta_cache, &mock_state_builder,
-                             &mock_replayer_listener};
+  MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
+                             "local mirror uuid", &m_pool_meta_cache,
+                             &mock_state_builder, &mock_replayer_listener};
   m_pool_meta_cache.set_remote_pool_meta(
     m_remote_io_ctx.get_id(),
     {"remote mirror uuid", "remote mirror peer uuid"});
@@ -919,6 +951,7 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, RemoteImageDemoted) {
   expect_create_non_primary_request(mock_create_non_primary_request,
                                     true, "remote mirror uuid", 1,
                                     {{1, CEPH_NOSNAP}}, 11, 0);
+  expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0);
   MockImageCopyRequest mock_image_copy_request;
   expect_image_copy(mock_image_copy_request, 0, 1, 0, {},
                     {{1, CEPH_NOSNAP}}, 0);
@@ -964,13 +997,14 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, LocalImagePromoted) {
 
   InSequence seq;
 
+  MockInstanceWatcher mock_instance_watcher;
   MockImageMeta mock_image_meta;
   MockStateBuilder mock_state_builder(mock_local_image_ctx,
                                       mock_remote_image_ctx,
                                       mock_image_meta);
-  MockReplayer mock_replayer{&mock_threads, "local mirror uuid",
-                             &m_pool_meta_cache, &mock_state_builder,
-                             &mock_replayer_listener};
+  MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
+                             "local mirror uuid", &m_pool_meta_cache,
+                             &mock_state_builder, &mock_replayer_listener};
   m_pool_meta_cache.set_remote_pool_meta(
     m_remote_io_ctx.get_id(),
     {"remote mirror uuid", "remote mirror peer uuid"});
@@ -1019,13 +1053,14 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, ResyncRequested) {
 
   InSequence seq;
 
+  MockInstanceWatcher mock_instance_watcher;
   MockImageMeta mock_image_meta;
   MockStateBuilder mock_state_builder(mock_local_image_ctx,
                                       mock_remote_image_ctx,
                                       mock_image_meta);
-  MockReplayer mock_replayer{&mock_threads, "local mirror uuid",
-                             &m_pool_meta_cache, &mock_state_builder,
-                             &mock_replayer_listener};
+  MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
+                             "local mirror uuid", &m_pool_meta_cache,
+                             &mock_state_builder, &mock_replayer_listener};
   m_pool_meta_cache.set_remote_pool_meta(
     m_remote_io_ctx.get_id(),
     {"remote mirror uuid", "remote mirror peer uuid"});
@@ -1062,14 +1097,15 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, RegisterLocalUpdateWatcherError) {
 
   InSequence seq;
 
+  MockInstanceWatcher mock_instance_watcher;
   MockImageMeta mock_image_meta;
   MockStateBuilder mock_state_builder(mock_local_image_ctx,
                                       mock_remote_image_ctx,
                                       mock_image_meta);
   MockReplayerListener mock_replayer_listener;
-  MockReplayer mock_replayer{&mock_threads, "local mirror uuid",
-                             &m_pool_meta_cache, &mock_state_builder,
-                             &mock_replayer_listener};
+  MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
+                             "local mirror uuid", &m_pool_meta_cache,
+                             &mock_state_builder, &mock_replayer_listener};
   m_pool_meta_cache.set_remote_pool_meta(
     m_remote_io_ctx.get_id(),
     {"remote mirror uuid", "remote mirror peer uuid"});
@@ -1094,14 +1130,15 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, RegisterRemoteUpdateWatcherError)
 
   InSequence seq;
 
+  MockInstanceWatcher mock_instance_watcher;
   MockImageMeta mock_image_meta;
   MockStateBuilder mock_state_builder(mock_local_image_ctx,
                                       mock_remote_image_ctx,
                                       mock_image_meta);
   MockReplayerListener mock_replayer_listener;
-  MockReplayer mock_replayer{&mock_threads, "local mirror uuid",
-                             &m_pool_meta_cache, &mock_state_builder,
-                             &mock_replayer_listener};
+  MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
+                             "local mirror uuid", &m_pool_meta_cache,
+                             &mock_state_builder, &mock_replayer_listener};
   m_pool_meta_cache.set_remote_pool_meta(
     m_remote_io_ctx.get_id(),
     {"remote mirror uuid", "remote mirror peer uuid"});
@@ -1133,13 +1170,14 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, UnregisterRemoteUpdateWatcherError
 
   InSequence seq;
 
+  MockInstanceWatcher mock_instance_watcher;
   MockImageMeta mock_image_meta;
   MockStateBuilder mock_state_builder(mock_local_image_ctx,
                                       mock_remote_image_ctx,
                                       mock_image_meta);
-  MockReplayer mock_replayer{&mock_threads, "local mirror uuid",
-                             &m_pool_meta_cache, &mock_state_builder,
-                             &mock_replayer_listener};
+  MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
+                             "local mirror uuid", &m_pool_meta_cache,
+                             &mock_state_builder, &mock_replayer_listener};
   m_pool_meta_cache.set_remote_pool_meta(
     m_remote_io_ctx.get_id(),
     {"remote mirror uuid", "remote mirror peer uuid"});
@@ -1174,13 +1212,14 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, UnregisterLocalUpdateWatcherError)
 
   InSequence seq;
 
+  MockInstanceWatcher mock_instance_watcher;
   MockImageMeta mock_image_meta;
   MockStateBuilder mock_state_builder(mock_local_image_ctx,
                                       mock_remote_image_ctx,
                                       mock_image_meta);
-  MockReplayer mock_replayer{&mock_threads, "local mirror uuid",
-                             &m_pool_meta_cache, &mock_state_builder,
-                             &mock_replayer_listener};
+  MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
+                             "local mirror uuid", &m_pool_meta_cache,
+                             &mock_state_builder, &mock_replayer_listener};
   m_pool_meta_cache.set_remote_pool_meta(
     m_remote_io_ctx.get_id(),
     {"remote mirror uuid", "remote mirror peer uuid"});
@@ -1215,13 +1254,14 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, LoadImageMetaError) {
 
   InSequence seq;
 
+  MockInstanceWatcher mock_instance_watcher;
   MockImageMeta mock_image_meta;
   MockStateBuilder mock_state_builder(mock_local_image_ctx,
                                       mock_remote_image_ctx,
                                       mock_image_meta);
-  MockReplayer mock_replayer{&mock_threads, "local mirror uuid",
-                             &m_pool_meta_cache, &mock_state_builder,
-                             &mock_replayer_listener};
+  MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
+                             "local mirror uuid", &m_pool_meta_cache,
+                             &mock_state_builder, &mock_replayer_listener};
   m_pool_meta_cache.set_remote_pool_meta(
     m_remote_io_ctx.get_id(),
     {"remote mirror uuid", "remote mirror peer uuid"});
@@ -1262,13 +1302,14 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, RefreshLocalImageError) {
 
   InSequence seq;
 
+  MockInstanceWatcher mock_instance_watcher;
   MockImageMeta mock_image_meta;
   MockStateBuilder mock_state_builder(mock_local_image_ctx,
                                       mock_remote_image_ctx,
                                       mock_image_meta);
-  MockReplayer mock_replayer{&mock_threads, "local mirror uuid",
-                             &m_pool_meta_cache, &mock_state_builder,
-                             &mock_replayer_listener};
+  MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
+                             "local mirror uuid", &m_pool_meta_cache,
+                             &mock_state_builder, &mock_replayer_listener};
   m_pool_meta_cache.set_remote_pool_meta(
     m_remote_io_ctx.get_id(),
     {"remote mirror uuid", "remote mirror peer uuid"});
@@ -1311,13 +1352,14 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, RefreshRemoteImageError) {
 
   InSequence seq;
 
+  MockInstanceWatcher mock_instance_watcher;
   MockImageMeta mock_image_meta;
   MockStateBuilder mock_state_builder(mock_local_image_ctx,
                                       mock_remote_image_ctx,
                                       mock_image_meta);
-  MockReplayer mock_replayer{&mock_threads, "local mirror uuid",
-                             &m_pool_meta_cache, &mock_state_builder,
-                             &mock_replayer_listener};
+  MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
+                             "local mirror uuid", &m_pool_meta_cache,
+                             &mock_state_builder, &mock_replayer_listener};
   m_pool_meta_cache.set_remote_pool_meta(
     m_remote_io_ctx.get_id(),
     {"remote mirror uuid", "remote mirror peer uuid"});
@@ -1361,13 +1403,14 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, CopySnapshotsError) {
 
   InSequence seq;
 
+  MockInstanceWatcher mock_instance_watcher;
   MockImageMeta mock_image_meta;
   MockStateBuilder mock_state_builder(mock_local_image_ctx,
                                       mock_remote_image_ctx,
                                       mock_image_meta);
-  MockReplayer mock_replayer{&mock_threads, "local mirror uuid",
-                             &m_pool_meta_cache, &mock_state_builder,
-                             &mock_replayer_listener};
+  MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
+                             "local mirror uuid", &m_pool_meta_cache,
+                             &mock_state_builder, &mock_replayer_listener};
   m_pool_meta_cache.set_remote_pool_meta(
     m_remote_io_ctx.get_id(),
     {"remote mirror uuid", "remote mirror peer uuid"});
@@ -1420,13 +1463,14 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, GetImageStateError) {
 
   InSequence seq;
 
+  MockInstanceWatcher mock_instance_watcher;
   MockImageMeta mock_image_meta;
   MockStateBuilder mock_state_builder(mock_local_image_ctx,
                                       mock_remote_image_ctx,
                                       mock_image_meta);
-  MockReplayer mock_replayer{&mock_threads, "local mirror uuid",
-                             &m_pool_meta_cache, &mock_state_builder,
-                             &mock_replayer_listener};
+  MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
+                             "local mirror uuid", &m_pool_meta_cache,
+                             &mock_state_builder, &mock_replayer_listener};
   m_pool_meta_cache.set_remote_pool_meta(
     m_remote_io_ctx.get_id(),
     {"remote mirror uuid", "remote mirror peer uuid"});
@@ -1481,13 +1525,14 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, CreateNonPrimarySnapshotError) {
 
   InSequence seq;
 
+  MockInstanceWatcher mock_instance_watcher;
   MockImageMeta mock_image_meta;
   MockStateBuilder mock_state_builder(mock_local_image_ctx,
                                       mock_remote_image_ctx,
                                       mock_image_meta);
-  MockReplayer mock_replayer{&mock_threads, "local mirror uuid",
-                             &m_pool_meta_cache, &mock_state_builder,
-                             &mock_replayer_listener};
+  MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
+                             "local mirror uuid", &m_pool_meta_cache,
+                             &mock_state_builder, &mock_replayer_listener};
   m_pool_meta_cache.set_remote_pool_meta(
     m_remote_io_ctx.get_id(),
     {"remote mirror uuid", "remote mirror peer uuid"});
@@ -1534,6 +1579,75 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, CreateNonPrimarySnapshotError) {
                                         mock_remote_image_ctx));
 }
 
+TEST_F(TestMockImageReplayerSnapshotReplayer, RequestSyncError) {
+  librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
+  librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
+
+  MockThreads mock_threads(m_threads);
+  expect_work_queue_repeatedly(mock_threads);
+
+  MockReplayerListener mock_replayer_listener;
+  expect_notification(mock_threads, mock_replayer_listener);
+
+  InSequence seq;
+
+  MockInstanceWatcher mock_instance_watcher;
+  MockImageMeta mock_image_meta;
+  MockStateBuilder mock_state_builder(mock_local_image_ctx,
+                                      mock_remote_image_ctx,
+                                      mock_image_meta);
+  MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
+                             "local mirror uuid", &m_pool_meta_cache,
+                             &mock_state_builder, &mock_replayer_listener};
+  m_pool_meta_cache.set_remote_pool_meta(
+    m_remote_io_ctx.get_id(),
+    {"remote mirror uuid", "remote mirror peer uuid"});
+
+  librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
+  ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads,
+                                   mock_local_image_ctx,
+                                   mock_remote_image_ctx,
+                                   mock_replayer_listener,
+                                   mock_image_meta,
+                                   &update_watch_ctx));
+
+  // inject snapshot
+  mock_remote_image_ctx.snap_info = {
+    {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
+       cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, "",
+       0U, true, 0, {}},
+     0, {}, 0, 0, {}}}};
+
+  // sync snap1
+  expect_load_image_meta(mock_image_meta, false, 0);
+  expect_is_refresh_required(mock_local_image_ctx, false);
+  expect_is_refresh_required(mock_remote_image_ctx, false);
+  MockSnapshotCopyRequest mock_snapshot_copy_request;
+  expect_snapshot_copy(mock_snapshot_copy_request, 0, 1, 0, {{1, CEPH_NOSNAP}},
+                       0);
+  MockGetImageStateRequest mock_get_image_state_request;
+  expect_get_image_state(mock_get_image_state_request, 1, 0);
+  MockCreateNonPrimaryRequest mock_create_non_primary_request;
+  expect_create_non_primary_request(mock_create_non_primary_request,
+                                    false, "remote mirror uuid", 1,
+                                    {{1, CEPH_NOSNAP}}, 11, 0);
+  expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id,
+                             -ECANCELED);
+
+  // wake-up replayer
+  update_watch_ctx->handle_notify();
+
+  // wait for sync to complete and expect replay complete
+  ASSERT_EQ(0, wait_for_notification(1));
+  ASSERT_FALSE(mock_replayer.is_replaying());
+  ASSERT_EQ(-ECANCELED, mock_replayer.get_error_code());
+
+  ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads,
+                                        mock_local_image_ctx,
+                                        mock_remote_image_ctx));
+
+}
+
 TEST_F(TestMockImageReplayerSnapshotReplayer, CopyImageError) {
   librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
   librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
@@ -1546,13 +1660,14 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, CopyImageError) {
 
   InSequence seq;
 
+  MockInstanceWatcher mock_instance_watcher;
   MockImageMeta mock_image_meta;
   MockStateBuilder mock_state_builder(mock_local_image_ctx,
                                       mock_remote_image_ctx,
                                       mock_image_meta);
-  MockReplayer mock_replayer{&mock_threads, "local mirror uuid",
-                             &m_pool_meta_cache, &mock_state_builder,
-                             &mock_replayer_listener};
+  MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
+                             "local mirror uuid", &m_pool_meta_cache,
+                             &mock_state_builder, &mock_replayer_listener};
   m_pool_meta_cache.set_remote_pool_meta(
     m_remote_io_ctx.get_id(),
     {"remote mirror uuid", "remote mirror peer uuid"});
@@ -1585,6 +1700,7 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, CopyImageError) {
   expect_create_non_primary_request(mock_create_non_primary_request,
                                     false, "remote mirror uuid", 1,
                                     {{1, CEPH_NOSNAP}}, 11, 0);
+  expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0);
   MockImageCopyRequest mock_image_copy_request;
   expect_image_copy(mock_image_copy_request, 0, 1, 0, {},
                     {{1, CEPH_NOSNAP}}, -EINVAL);
@@ -1614,13 +1730,14 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, UpdateNonPrimarySnapshotError) {
 
   InSequence seq;
 
+  MockInstanceWatcher mock_instance_watcher;
   MockImageMeta mock_image_meta;
   MockStateBuilder mock_state_builder(mock_local_image_ctx,
                                       mock_remote_image_ctx,
                                       mock_image_meta);
-  MockReplayer mock_replayer{&mock_threads, "local mirror uuid",
-                             &m_pool_meta_cache, &mock_state_builder,
-                             &mock_replayer_listener};
+  MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
+                             "local mirror uuid", &m_pool_meta_cache,
+                             &mock_state_builder, &mock_replayer_listener};
   m_pool_meta_cache.set_remote_pool_meta(
     m_remote_io_ctx.get_id(),
     {"remote mirror uuid", "remote mirror peer uuid"});
@@ -1653,6 +1770,7 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, UpdateNonPrimarySnapshotError) {
   expect_create_non_primary_request(mock_create_non_primary_request,
                                     false, "remote mirror uuid", 1,
                                     {{1, CEPH_NOSNAP}}, 11, 0);
+  expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0);
   MockImageCopyRequest mock_image_copy_request;
   expect_image_copy(mock_image_copy_request, 0, 1, 0, {},
                     {{1, CEPH_NOSNAP}}, 0);
@@ -1686,13 +1804,14 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, UnlinkPeerError) {
 
   InSequence seq;
 
+  MockInstanceWatcher mock_instance_watcher;
   MockImageMeta mock_image_meta;
   MockStateBuilder mock_state_builder(mock_local_image_ctx,
                                       mock_remote_image_ctx,
                                       mock_image_meta);
-  MockReplayer mock_replayer{&mock_threads, "local mirror uuid",
-                             &m_pool_meta_cache, &mock_state_builder,
-                             &mock_replayer_listener};
+  MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
+                             "local mirror uuid", &m_pool_meta_cache,
+                             &mock_state_builder, &mock_replayer_listener};
   m_pool_meta_cache.set_remote_pool_meta(
     m_remote_io_ctx.get_id(),
     {"remote mirror uuid", "remote mirror peer uuid"});
@@ -1734,6 +1853,7 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, UnlinkPeerError) {
   expect_create_non_primary_request(mock_create_non_primary_request,
                                     false, "remote mirror uuid", 2,
                                     {{2, CEPH_NOSNAP}}, 12, 0);
+  expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0);
   MockImageCopyRequest mock_image_copy_request;
   expect_image_copy(mock_image_copy_request, 1, 2, 11, {},
                     {{2, CEPH_NOSNAP}}, 0);
@@ -1745,6 +1865,7 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, UnlinkPeerError) {
   MockUnlinkPeerRequest mock_unlink_peer_request;
   expect_unlink_peer(mock_unlink_peer_request, 1, "remote mirror peer uuid",
                      -EINVAL);
+  expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id);
 
   // wake-up replayer
   update_watch_ctx->handle_notify();
@@ -1771,13 +1892,14 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, SplitBrain) {
 
   InSequence seq;
 
+  MockInstanceWatcher mock_instance_watcher;
   MockImageMeta mock_image_meta;
   MockStateBuilder mock_state_builder(mock_local_image_ctx,
                                       mock_remote_image_ctx,
                                       mock_image_meta);
-  MockReplayer mock_replayer{&mock_threads, "local mirror uuid",
-                             &m_pool_meta_cache, &mock_state_builder,
-                             &mock_replayer_listener};
+  MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher,
+                             "local mirror uuid", &m_pool_meta_cache,
+                             &mock_state_builder, &mock_replayer_listener};
   m_pool_meta_cache.set_remote_pool_meta(
     m_remote_io_ctx.get_id(),
     {"remote mirror uuid", "remote mirror peer uuid"});
diff --git a/src/test/rbd_mirror/test_mock_ImageReplayer.cc b/src/test/rbd_mirror/test_mock_ImageReplayer.cc
index 653b6e98bb7..20693e3223b 100644
--- a/src/test/rbd_mirror/test_mock_ImageReplayer.cc
+++ b/src/test/rbd_mirror/test_mock_ImageReplayer.cc
@@ -175,7 +175,8 @@ struct StateBuilder<librbd::MockTestImageCtx> {
   }
 
   MOCK_METHOD1(close, void(Context*));
-  MOCK_METHOD4(create_replayer, Replayer*(Threads<librbd::MockTestImageCtx>*,
+  MOCK_METHOD5(create_replayer, Replayer*(Threads<librbd::MockTestImageCtx>*,
+                                          InstanceWatcher<librbd::MockTestImageCtx>*,
                                           const std::string&, PoolMetaCache*,
                                           ReplayerListener*));
 
@@ -301,8 +302,8 @@ public:
 
   void expect_create_replayer(MockStateBuilder& mock_state_builder,
                               MockReplayer& mock_replayer) {
-    EXPECT_CALL(mock_state_builder, create_replayer(_, _, _, _))
-      .WillOnce(WithArg<3>(
+    EXPECT_CALL(mock_state_builder, create_replayer(_, _, _, _, _))
+      .WillOnce(WithArg<4>(
         Invoke([&mock_replayer]
                (image_replayer::ReplayerListener* replayer_listener) {
           mock_replayer.replayer_listener = replayer_listener;
diff --git a/src/tools/rbd_mirror/ImageReplayer.cc b/src/tools/rbd_mirror/ImageReplayer.cc
index 8068ef67871..dd74512dfd1 100644
--- a/src/tools/rbd_mirror/ImageReplayer.cc
+++ b/src/tools/rbd_mirror/ImageReplayer.cc
@@ -408,7 +408,8 @@ void ImageReplayer<I>::start_replay() {
 
   std::unique_lock locker{m_lock};
   ceph_assert(m_replayer == nullptr);
-  m_replayer = m_state_builder->create_replayer(m_threads, m_local_mirror_uuid,
+  m_replayer = m_state_builder->create_replayer(m_threads, m_instance_watcher,
+                                                m_local_mirror_uuid,
                                                 m_pool_meta_cache,
                                                 m_replayer_listener);
 
diff --git a/src/tools/rbd_mirror/LeaderWatcher.cc b/src/tools/rbd_mirror/LeaderWatcher.cc
index 1581319219d..ae705e3c5e2 100644
--- a/src/tools/rbd_mirror/LeaderWatcher.cc
+++ b/src/tools/rbd_mirror/LeaderWatcher.cc
@@ -946,7 +946,7 @@ void LeaderWatcher<I>::handle_heartbeat(Context *on_notify_ack) {
     std::scoped_lock locker{m_threads->timer_lock, m_lock};
     if (is_leader(m_lock)) {
       dout(5) << "got another leader heartbeat, ignoring" << dendl;
-    } else {
+    } else if (!m_locker.cookie.empty()) {
       cancel_timer_task();
       m_acquire_attempts = 0;
       schedule_acquire_leader_lock(1);
diff --git a/src/tools/rbd_mirror/image_replayer/StateBuilder.h b/src/tools/rbd_mirror/image_replayer/StateBuilder.h
index 99f5fce4487..d055c84e020 100644
--- a/src/tools/rbd_mirror/image_replayer/StateBuilder.h
+++ b/src/tools/rbd_mirror/image_replayer/StateBuilder.h
@@ -15,6 +15,7 @@ namespace rbd {
 namespace mirror {
 
 struct BaseRequest;
+template <typename> class InstanceWatcher;
 struct PoolMetaCache;
 struct ProgressContext;
 template <typename> class Threads;
@@ -73,6 +74,7 @@ public:
 
   virtual Replayer* create_replayer(
       Threads<ImageCtxT>* threads,
+      InstanceWatcher<ImageCtxT>* instance_watcher,
       const std::string& local_mirror_uuid,
       PoolMetaCache* pool_meta_cache,
       ReplayerListener* replayer_listener) = 0;
diff --git a/src/tools/rbd_mirror/image_replayer/journal/StateBuilder.cc b/src/tools/rbd_mirror/image_replayer/journal/StateBuilder.cc
index c28e0694823..8d391580ed7 100644
--- a/src/tools/rbd_mirror/image_replayer/journal/StateBuilder.cc
+++ b/src/tools/rbd_mirror/image_replayer/journal/StateBuilder.cc
@@ -104,7 +104,8 @@ BaseRequest* StateBuilder<I>::create_prepare_replay_request(
 
 template <typename I>
 image_replayer::Replayer* StateBuilder<I>::create_replayer(
-   Threads<I>* threads,
+    Threads<I>* threads,
+    InstanceWatcher<I>* instance_watcher,
     const std::string& local_mirror_uuid,
     PoolMetaCache* pool_meta_cache,
     ReplayerListener* replayer_listener) {
diff --git a/src/tools/rbd_mirror/image_replayer/journal/StateBuilder.h b/src/tools/rbd_mirror/image_replayer/journal/StateBuilder.h
index 3f3621727ce..1d4fa83cbb0 100644
--- a/src/tools/rbd_mirror/image_replayer/journal/StateBuilder.h
+++ b/src/tools/rbd_mirror/image_replayer/journal/StateBuilder.h
@@ -60,6 +60,7 @@ public:
 
   image_replayer::Replayer* create_replayer(
       Threads<ImageCtxT>* threads,
+      InstanceWatcher<ImageCtxT>* instance_watcher,
       const std::string& local_mirror_uuid,
       PoolMetaCache* pool_meta_cache,
       ReplayerListener* replayer_listener) override;
diff --git a/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.cc b/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.cc
index 2bb495913a5..69fb3d93af4 100644
--- a/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.cc
+++ b/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.cc
@@ -17,6 +17,7 @@
 #include "librbd/mirror/snapshot/GetImageStateRequest.h"
 #include "librbd/mirror/snapshot/ImageMeta.h"
 #include "librbd/mirror/snapshot/UnlinkPeerRequest.h"
+#include "tools/rbd_mirror/InstanceWatcher.h"
 #include "tools/rbd_mirror/PoolMetaCache.h"
 #include "tools/rbd_mirror/Threads.h"
 #include "tools/rbd_mirror/Types.h"
@@ -110,11 +111,13 @@ struct Replayer<I>::ProgressContext : public librbd::ProgressContext {
 template <typename I>
 Replayer<I>::Replayer(
     Threads<I>* threads,
+    InstanceWatcher<I>* instance_watcher,
     const std::string& local_mirror_uuid,
     PoolMetaCache* pool_meta_cache,
     StateBuilder<I>* state_builder,
     ReplayerListener* replayer_listener)
   : m_threads(threads),
+    m_instance_watcher(instance_watcher),
     m_local_mirror_uuid(local_mirror_uuid),
     m_pool_meta_cache(pool_meta_cache),
     m_state_builder(state_builder),
@@ -172,6 +175,10 @@ void Replayer<I>::shut_down(Context* on_finish) {
   std::swap(m_state, state);
 
   if (state == STATE_REPLAYING) {
+    // if a sync request was pending, request a cancelation
+    m_instance_watcher->cancel_sync_request(
+      m_state_builder->local_image_ctx->id);
+
     // TODO interrupt snapshot copy and image copy state machines even if remote
     // cluster is unreachable
     dout(10) << "shut down pending on completion of snapshot replay" << dendl;
@@ -708,7 +715,7 @@ void Replayer<I>::handle_get_local_image_state(int r) {
     return;
   }
 
-  copy_image();
+  request_sync();
 }
 
 template <typename I>
@@ -737,6 +744,41 @@ void Replayer<I>::handle_create_non_primary_snapshot(int r) {
 
   dout(15) << "local_snap_id_end=" << m_local_snap_id_end << dendl;
 
+  request_sync();
+}
+
+template <typename I>
+void Replayer<I>::request_sync() {
+  dout(10) << dendl;
+
+  std::unique_lock locker{m_lock};
+  auto ctx = create_async_context_callback(
+    m_threads->work_queue, create_context_callback<
+      Replayer<I>, &Replayer<I>::handle_request_sync>(this));
+  m_instance_watcher->notify_sync_request(m_state_builder->local_image_ctx->id,
+                                          ctx);
+}
+
+template <typename I>
+void Replayer<I>::handle_request_sync(int r) {
+  dout(10) << "r=" << r << dendl;
+
+  std::unique_lock locker{m_lock};
+  if (is_replay_interrupted(&locker)) {
+    return;
+  } else if (r == -ECANCELED) {
+    dout(5) << "image-sync canceled" << dendl;
+    handle_replay_complete(&locker, r, "image-sync canceled");
+    return;
+  } else if (r < 0) {
+    derr << "failed to request image-sync: " << cpp_strerror(r) << dendl;
+    handle_replay_complete(&locker, r, "failed to request image-sync");
+    return;
+  }
+
+  m_sync_in_progress = true;
+  locker.unlock();
+
   copy_image();
 }
 
@@ -897,22 +939,13 @@ void Replayer<I>::handle_notify_image_update(int r) {
     derr << "failed to notify local image update: " << cpp_strerror(r) << dendl;
   }
 
-  if (is_replay_interrupted()) {
-    return;
-  }
-
   unlink_peer();
 }
 
 template <typename I>
 void Replayer<I>::unlink_peer() {
   if (m_remote_snap_id_start == 0) {
-    {
-      std::unique_lock locker{m_lock};
-      notify_status_updated();
-    }
-
-    load_local_image_meta();
+    finish_sync();
     return;
   }
 
@@ -939,9 +972,24 @@ void Replayer<I>::handle_unlink_peer(int r) {
     return;
   }
 
+  finish_sync();
+}
+
+template <typename I>
+void Replayer<I>::finish_sync() {
+  dout(10) << dendl;
+
   {
     std::unique_lock locker{m_lock};
     notify_status_updated();
+
+    m_sync_in_progress = false;
+    m_instance_watcher->notify_sync_complete(
+      m_state_builder->local_image_ctx->id);
+  }
+
+  if (is_replay_interrupted()) {
+    return;
   }
 
   load_local_image_meta();
@@ -1135,6 +1183,12 @@ void Replayer<I>::handle_replay_complete(std::unique_lock<ceph::mutex>* locker,
     m_error_description = description;
   }
 
+  if (m_sync_in_progress) {
+    m_sync_in_progress = false;
+    m_instance_watcher->notify_sync_complete(
+      m_state_builder->local_image_ctx->id);
+  }
+
   if (m_state != STATE_REPLAYING && m_state != STATE_IDLE) {
     return;
   }
diff --git a/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.h b/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.h
index e5ae54c6272..e7536f5a6fa 100644
--- a/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.h
+++ b/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.h
@@ -22,6 +22,7 @@ namespace snapshot { template <typename I> class Replay; }
 namespace rbd {
 namespace mirror {
 
+template <typename> struct InstanceWatcher;
 class PoolMetaCache;
 template <typename> struct Threads;
 
@@ -40,16 +41,18 @@ class Replayer : public image_replayer::Replayer {
 public:
   static Replayer* create(
       Threads<ImageCtxT>* threads,
+      InstanceWatcher<ImageCtxT>* instance_watcher,
       const std::string& local_mirror_uuid,
       PoolMetaCache* pool_meta_cache,
       StateBuilder<ImageCtxT>* state_builder,
       ReplayerListener* replayer_listener) {
-    return new Replayer(threads, local_mirror_uuid, pool_meta_cache,
-                        state_builder, replayer_listener);
+    return new Replayer(threads, instance_watcher, local_mirror_uuid,
+                        pool_meta_cache, state_builder, replayer_listener);
   }
 
   Replayer(
       Threads<ImageCtxT>* threads,
+      InstanceWatcher<ImageCtxT>* instance_watcher,
       const std::string& local_mirror_uuid,
       PoolMetaCache* pool_meta_cache,
       StateBuilder<ImageCtxT>* state_builder,
@@ -123,6 +126,9 @@ private:
    *    |                     |/----------------------/ |
    *    |                     |                         |
    *    |                     v                         |
+   *    |                 REQUEST_SYNC                  |
+   *    |                     |                         |
+   *    |                     v                         |
    *    |                 COPY_IMAGE                    |
    *    |                     |                         |
    *    |                     v                         |
@@ -180,6 +186,7 @@ private:
   struct ProgressContext;
 
   Threads<ImageCtxT>* m_threads;
+  InstanceWatcher<ImageCtxT>* m_instance_watcher;
   std::string m_local_mirror_uuid;
   PoolMetaCache* m_pool_meta_cache;
   StateBuilder<ImageCtxT>* m_state_builder;
@@ -217,6 +224,7 @@ private:
 
   bool m_remote_image_updated = false;
   bool m_updating_sync_point = false;
+  bool m_sync_in_progress = false;
 
   void load_local_image_meta();
   void handle_load_local_image_meta(int r);
@@ -242,6 +250,9 @@ private:
   void create_non_primary_snapshot();
   void handle_create_non_primary_snapshot(int r);
 
+  void request_sync();
+  void handle_request_sync(int r);
+
   void copy_image();
   void handle_copy_image(int r);
   void handle_copy_image_progress(uint64_t object_number,
@@ -259,6 +270,8 @@ private:
   void unlink_peer();
   void handle_unlink_peer(int r);
 
+  void finish_sync();
+
   void register_local_update_watcher();
   void handle_register_local_update_watcher(int r);
 
diff --git a/src/tools/rbd_mirror/image_replayer/snapshot/StateBuilder.cc b/src/tools/rbd_mirror/image_replayer/snapshot/StateBuilder.cc
index 27225707f55..2fb2534d642 100644
--- a/src/tools/rbd_mirror/image_replayer/snapshot/StateBuilder.cc
+++ b/src/tools/rbd_mirror/image_replayer/snapshot/StateBuilder.cc
@@ -103,12 +103,14 @@ BaseRequest* StateBuilder<I>::create_prepare_replay_request(
 
 template <typename I>
 image_replayer::Replayer* StateBuilder<I>::create_replayer(
-   Threads<I>* threads,
+    Threads<I>* threads,
+    InstanceWatcher<I>* instance_watcher,
     const std::string& local_mirror_uuid,
     PoolMetaCache* pool_meta_cache,
     ReplayerListener* replayer_listener) {
   return Replayer<I>::create(
-    threads, local_mirror_uuid, pool_meta_cache, this, replayer_listener);
+    threads, instance_watcher, local_mirror_uuid, pool_meta_cache, this,
+    replayer_listener);
 }
 
 } // namespace snapshot
diff --git a/src/tools/rbd_mirror/image_replayer/snapshot/StateBuilder.h b/src/tools/rbd_mirror/image_replayer/snapshot/StateBuilder.h
index 83c55ebafdc..ab326255fdb 100644
--- a/src/tools/rbd_mirror/image_replayer/snapshot/StateBuilder.h
+++ b/src/tools/rbd_mirror/image_replayer/snapshot/StateBuilder.h
@@ -69,6 +69,7 @@ public:
 
   image_replayer::Replayer* create_replayer(
       Threads<ImageCtxT>* threads,
+      InstanceWatcher<ImageCtxT>* instance_watcher,
       const std::string& local_mirror_uuid,
       PoolMetaCache* pool_meta_cache,
       ReplayerListener* replayer_listener) override;