diff options
12 files changed, 280 insertions, 82 deletions
diff --git a/src/librbd/api/Trash.cc b/src/librbd/api/Trash.cc index 7d615050a25..720be6f55aa 100644 --- a/src/librbd/api/Trash.cc +++ b/src/librbd/api/Trash.cc @@ -158,7 +158,7 @@ int Trash<I>::move(librados::IoCtx &io_ctx, rbd_trash_image_source_t source, ictx->exclusive_lock->block_requests(0); r = ictx->operations->prepare_image_update( - exclusive_lock::OPERATION_REQUEST_TYPE_GENERAL, false); + exclusive_lock::OPERATION_REQUEST_TYPE_GENERAL, true); if (r < 0) { lderr(cct) << "cannot obtain exclusive lock - not removing" << dendl; ictx->owner_lock.unlock_shared(); diff --git a/src/test/rbd_mirror/image_replayer/snapshot/test_mock_Replayer.cc b/src/test/rbd_mirror/image_replayer/snapshot/test_mock_Replayer.cc index 47e232e9f06..b2e95bad097 100644 --- a/src/test/rbd_mirror/image_replayer/snapshot/test_mock_Replayer.cc +++ b/src/test/rbd_mirror/image_replayer/snapshot/test_mock_Replayer.cc @@ -8,6 +8,7 @@ #include "librbd/mirror/snapshot/GetImageStateRequest.h" #include "librbd/mirror/snapshot/ImageMeta.h" #include "librbd/mirror/snapshot/UnlinkPeerRequest.h" +#include "tools/rbd_mirror/InstanceWatcher.h" #include "tools/rbd_mirror/Threads.h" #include "tools/rbd_mirror/image_replayer/CloseImageRequest.h" #include "tools/rbd_mirror/image_replayer/ReplayerListener.h" @@ -219,6 +220,14 @@ namespace rbd { namespace mirror { template <> +struct InstanceWatcher<librbd::MockTestImageCtx> { + MOCK_METHOD1(cancel_sync_request, void(const std::string&)); + MOCK_METHOD2(notify_sync_request, void(const std::string&, + Context*)); + MOCK_METHOD1(notify_sync_complete, void(const std::string&)); +}; + +template <> struct Threads<librbd::MockTestImageCtx> { MockSafeTimer *timer; ceph::mutex &timer_lock; @@ -349,6 +358,7 @@ public: typedef Replayer<librbd::MockTestImageCtx> MockReplayer; typedef ApplyImageStateRequest<librbd::MockTestImageCtx> MockApplyImageStateRequest; typedef StateBuilder<librbd::MockTestImageCtx> MockStateBuilder; + typedef InstanceWatcher<librbd::MockTestImageCtx> MockInstanceWatcher; typedef Threads<librbd::MockTestImageCtx> MockThreads; typedef CloseImageRequest<librbd::MockTestImageCtx> MockCloseImageRequest; typedef librbd::deep_copy::ImageCopyRequest<librbd::MockTestImageCtx> MockImageCopyRequest; @@ -491,6 +501,19 @@ public: })); } + void expect_notify_sync_request(MockInstanceWatcher& mock_instance_watcher, + const std::string& image_id, int r) { + EXPECT_CALL(mock_instance_watcher, notify_sync_request(image_id, _)) + .WillOnce(WithArg<1>(Invoke([this, r](Context* ctx) { + m_threads->work_queue->queue(ctx, r); + }))); + } + + void expect_notify_sync_complete(MockInstanceWatcher& mock_instance_watcher, + const std::string& image_id) { + EXPECT_CALL(mock_instance_watcher, notify_sync_complete(image_id)); + } + void expect_image_copy(MockImageCopyRequest& mock_image_copy_request, uint64_t src_snap_id_start, uint64_t src_snap_id_end, uint64_t dst_snap_id_start, @@ -635,13 +658,14 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, InitShutDown) { InSequence seq; + MockInstanceWatcher mock_instance_watcher; MockImageMeta mock_image_meta; MockStateBuilder mock_state_builder(mock_local_image_ctx, mock_remote_image_ctx, mock_image_meta); - MockReplayer mock_replayer{&mock_threads, "local mirror uuid", - &m_pool_meta_cache, &mock_state_builder, - &mock_replayer_listener}; + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; m_pool_meta_cache.set_remote_pool_meta( m_remote_io_ctx.get_id(), {"remote mirror uuid", "remote mirror peer uuid"}); @@ -688,13 +712,14 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, SyncSnapshot) { InSequence seq; + MockInstanceWatcher mock_instance_watcher; MockImageMeta mock_image_meta; MockStateBuilder mock_state_builder(mock_local_image_ctx, mock_remote_image_ctx, mock_image_meta); - MockReplayer mock_replayer{&mock_threads, "local mirror uuid", - &m_pool_meta_cache, &mock_state_builder, - &mock_replayer_listener}; + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; m_pool_meta_cache.set_remote_pool_meta( m_remote_io_ctx.get_id(), {"remote mirror uuid", "remote mirror peer uuid"}); @@ -720,6 +745,7 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, SyncSnapshot) { expect_create_non_primary_request(mock_create_non_primary_request, false, "remote mirror uuid", 1, {{1, CEPH_NOSNAP}}, 11, 0); + expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0); MockImageCopyRequest mock_image_copy_request; expect_image_copy(mock_image_copy_request, 0, 1, 0, {}, {{1, CEPH_NOSNAP}}, 0); @@ -728,6 +754,7 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, SyncSnapshot) { expect_mirror_image_snapshot_set_copy_progress( mock_local_image_ctx, 11, true, 0, 0); expect_notify_update(mock_local_image_ctx); + expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id); // sync snap4 expect_load_image_meta(mock_image_meta, false, 0); @@ -746,6 +773,7 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, SyncSnapshot) { expect_create_non_primary_request(mock_create_non_primary_request, false, "remote mirror uuid", 4, {{1, 11}, {4, CEPH_NOSNAP}}, 14, 0); + expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0); expect_image_copy(mock_image_copy_request, 1, 4, 11, {}, {{1, 11}, {4, CEPH_NOSNAP}}, 0); expect_apply_image_state(mock_apply_state_request, 0); @@ -755,6 +783,7 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, SyncSnapshot) { MockUnlinkPeerRequest mock_unlink_peer_request; expect_unlink_peer(mock_unlink_peer_request, 1, "remote mirror peer uuid", 0); + expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id); // idle expect_load_image_meta(mock_image_meta, false, 0); @@ -798,13 +827,14 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, InterruptedSync) { InSequence seq; + MockInstanceWatcher mock_instance_watcher; MockImageMeta mock_image_meta; MockStateBuilder mock_state_builder(mock_local_image_ctx, mock_remote_image_ctx, mock_image_meta); - MockReplayer mock_replayer{&mock_threads, "local mirror uuid", - &m_pool_meta_cache, &mock_state_builder, - &mock_replayer_listener}; + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; m_pool_meta_cache.set_remote_pool_meta( m_remote_io_ctx.get_id(), {"remote mirror uuid", "remote mirror peer uuid"}); @@ -835,6 +865,7 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, InterruptedSync) { expect_is_refresh_required(mock_remote_image_ctx, false); MockGetImageStateRequest mock_get_image_state_request; expect_get_image_state(mock_get_image_state_request, 11, 0); + expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0); MockImageCopyRequest mock_image_copy_request; expect_image_copy(mock_image_copy_request, 0, 1, 0, librbd::deep_copy::ObjectNumber{123U}, @@ -880,13 +911,14 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, RemoteImageDemoted) { InSequence seq; + MockInstanceWatcher mock_instance_watcher; MockImageMeta mock_image_meta; MockStateBuilder mock_state_builder(mock_local_image_ctx, mock_remote_image_ctx, mock_image_meta); - MockReplayer mock_replayer{&mock_threads, "local mirror uuid", - &m_pool_meta_cache, &mock_state_builder, - &mock_replayer_listener}; + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; m_pool_meta_cache.set_remote_pool_meta( m_remote_io_ctx.get_id(), {"remote mirror uuid", "remote mirror peer uuid"}); @@ -919,6 +951,7 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, RemoteImageDemoted) { expect_create_non_primary_request(mock_create_non_primary_request, true, "remote mirror uuid", 1, {{1, CEPH_NOSNAP}}, 11, 0); + expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0); MockImageCopyRequest mock_image_copy_request; expect_image_copy(mock_image_copy_request, 0, 1, 0, {}, {{1, CEPH_NOSNAP}}, 0); @@ -964,13 +997,14 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, LocalImagePromoted) { InSequence seq; + MockInstanceWatcher mock_instance_watcher; MockImageMeta mock_image_meta; MockStateBuilder mock_state_builder(mock_local_image_ctx, mock_remote_image_ctx, mock_image_meta); - MockReplayer mock_replayer{&mock_threads, "local mirror uuid", - &m_pool_meta_cache, &mock_state_builder, - &mock_replayer_listener}; + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; m_pool_meta_cache.set_remote_pool_meta( m_remote_io_ctx.get_id(), {"remote mirror uuid", "remote mirror peer uuid"}); @@ -1019,13 +1053,14 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, ResyncRequested) { InSequence seq; + MockInstanceWatcher mock_instance_watcher; MockImageMeta mock_image_meta; MockStateBuilder mock_state_builder(mock_local_image_ctx, mock_remote_image_ctx, mock_image_meta); - MockReplayer mock_replayer{&mock_threads, "local mirror uuid", - &m_pool_meta_cache, &mock_state_builder, - &mock_replayer_listener}; + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; m_pool_meta_cache.set_remote_pool_meta( m_remote_io_ctx.get_id(), {"remote mirror uuid", "remote mirror peer uuid"}); @@ -1062,14 +1097,15 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, RegisterLocalUpdateWatcherError) { InSequence seq; + MockInstanceWatcher mock_instance_watcher; MockImageMeta mock_image_meta; MockStateBuilder mock_state_builder(mock_local_image_ctx, mock_remote_image_ctx, mock_image_meta); MockReplayerListener mock_replayer_listener; - MockReplayer mock_replayer{&mock_threads, "local mirror uuid", - &m_pool_meta_cache, &mock_state_builder, - &mock_replayer_listener}; + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; m_pool_meta_cache.set_remote_pool_meta( m_remote_io_ctx.get_id(), {"remote mirror uuid", "remote mirror peer uuid"}); @@ -1094,14 +1130,15 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, RegisterRemoteUpdateWatcherError) InSequence seq; + MockInstanceWatcher mock_instance_watcher; MockImageMeta mock_image_meta; MockStateBuilder mock_state_builder(mock_local_image_ctx, mock_remote_image_ctx, mock_image_meta); MockReplayerListener mock_replayer_listener; - MockReplayer mock_replayer{&mock_threads, "local mirror uuid", - &m_pool_meta_cache, &mock_state_builder, - &mock_replayer_listener}; + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; m_pool_meta_cache.set_remote_pool_meta( m_remote_io_ctx.get_id(), {"remote mirror uuid", "remote mirror peer uuid"}); @@ -1133,13 +1170,14 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, UnregisterRemoteUpdateWatcherError InSequence seq; + MockInstanceWatcher mock_instance_watcher; MockImageMeta mock_image_meta; MockStateBuilder mock_state_builder(mock_local_image_ctx, mock_remote_image_ctx, mock_image_meta); - MockReplayer mock_replayer{&mock_threads, "local mirror uuid", - &m_pool_meta_cache, &mock_state_builder, - &mock_replayer_listener}; + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; m_pool_meta_cache.set_remote_pool_meta( m_remote_io_ctx.get_id(), {"remote mirror uuid", "remote mirror peer uuid"}); @@ -1174,13 +1212,14 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, UnregisterLocalUpdateWatcherError) InSequence seq; + MockInstanceWatcher mock_instance_watcher; MockImageMeta mock_image_meta; MockStateBuilder mock_state_builder(mock_local_image_ctx, mock_remote_image_ctx, mock_image_meta); - MockReplayer mock_replayer{&mock_threads, "local mirror uuid", - &m_pool_meta_cache, &mock_state_builder, - &mock_replayer_listener}; + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; m_pool_meta_cache.set_remote_pool_meta( m_remote_io_ctx.get_id(), {"remote mirror uuid", "remote mirror peer uuid"}); @@ -1215,13 +1254,14 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, LoadImageMetaError) { InSequence seq; + MockInstanceWatcher mock_instance_watcher; MockImageMeta mock_image_meta; MockStateBuilder mock_state_builder(mock_local_image_ctx, mock_remote_image_ctx, mock_image_meta); - MockReplayer mock_replayer{&mock_threads, "local mirror uuid", - &m_pool_meta_cache, &mock_state_builder, - &mock_replayer_listener}; + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; m_pool_meta_cache.set_remote_pool_meta( m_remote_io_ctx.get_id(), {"remote mirror uuid", "remote mirror peer uuid"}); @@ -1262,13 +1302,14 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, RefreshLocalImageError) { InSequence seq; + MockInstanceWatcher mock_instance_watcher; MockImageMeta mock_image_meta; MockStateBuilder mock_state_builder(mock_local_image_ctx, mock_remote_image_ctx, mock_image_meta); - MockReplayer mock_replayer{&mock_threads, "local mirror uuid", - &m_pool_meta_cache, &mock_state_builder, - &mock_replayer_listener}; + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; m_pool_meta_cache.set_remote_pool_meta( m_remote_io_ctx.get_id(), {"remote mirror uuid", "remote mirror peer uuid"}); @@ -1311,13 +1352,14 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, RefreshRemoteImageError) { InSequence seq; + MockInstanceWatcher mock_instance_watcher; MockImageMeta mock_image_meta; MockStateBuilder mock_state_builder(mock_local_image_ctx, mock_remote_image_ctx, mock_image_meta); - MockReplayer mock_replayer{&mock_threads, "local mirror uuid", - &m_pool_meta_cache, &mock_state_builder, - &mock_replayer_listener}; + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; m_pool_meta_cache.set_remote_pool_meta( m_remote_io_ctx.get_id(), {"remote mirror uuid", "remote mirror peer uuid"}); @@ -1361,13 +1403,14 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, CopySnapshotsError) { InSequence seq; + MockInstanceWatcher mock_instance_watcher; MockImageMeta mock_image_meta; MockStateBuilder mock_state_builder(mock_local_image_ctx, mock_remote_image_ctx, mock_image_meta); - MockReplayer mock_replayer{&mock_threads, "local mirror uuid", - &m_pool_meta_cache, &mock_state_builder, - &mock_replayer_listener}; + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; m_pool_meta_cache.set_remote_pool_meta( m_remote_io_ctx.get_id(), {"remote mirror uuid", "remote mirror peer uuid"}); @@ -1420,13 +1463,14 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, GetImageStateError) { InSequence seq; + MockInstanceWatcher mock_instance_watcher; MockImageMeta mock_image_meta; MockStateBuilder mock_state_builder(mock_local_image_ctx, mock_remote_image_ctx, mock_image_meta); - MockReplayer mock_replayer{&mock_threads, "local mirror uuid", - &m_pool_meta_cache, &mock_state_builder, - &mock_replayer_listener}; + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; m_pool_meta_cache.set_remote_pool_meta( m_remote_io_ctx.get_id(), {"remote mirror uuid", "remote mirror peer uuid"}); @@ -1481,13 +1525,14 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, CreateNonPrimarySnapshotError) { InSequence seq; + MockInstanceWatcher mock_instance_watcher; MockImageMeta mock_image_meta; MockStateBuilder mock_state_builder(mock_local_image_ctx, mock_remote_image_ctx, mock_image_meta); - MockReplayer mock_replayer{&mock_threads, "local mirror uuid", - &m_pool_meta_cache, &mock_state_builder, - &mock_replayer_listener}; + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; m_pool_meta_cache.set_remote_pool_meta( m_remote_io_ctx.get_id(), {"remote mirror uuid", "remote mirror peer uuid"}); @@ -1534,6 +1579,75 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, CreateNonPrimarySnapshotError) { mock_remote_image_ctx)); } +TEST_F(TestMockImageReplayerSnapshotReplayer, RequestSyncError) { + librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; + librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + + MockReplayerListener mock_replayer_listener; + expect_notification(mock_threads, mock_replayer_listener); + + InSequence seq; + + MockInstanceWatcher mock_instance_watcher; + MockImageMeta mock_image_meta; + MockStateBuilder mock_state_builder(mock_local_image_ctx, + mock_remote_image_ctx, + mock_image_meta); + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; + m_pool_meta_cache.set_remote_pool_meta( + m_remote_io_ctx.get_id(), + {"remote mirror uuid", "remote mirror peer uuid"}); + + librbd::UpdateWatchCtx* update_watch_ctx = nullptr; + ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx, + mock_replayer_listener, + mock_image_meta, + &update_watch_ctx)); + + // inject snapshot + mock_remote_image_ctx.snap_info = { + {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{ + cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"}, "", + 0U, true, 0, {}}, + 0, {}, 0, 0, {}}}}; + + // sync snap1 + expect_load_image_meta(mock_image_meta, false, 0); + expect_is_refresh_required(mock_local_image_ctx, false); + expect_is_refresh_required(mock_remote_image_ctx, false); + MockSnapshotCopyRequest mock_snapshot_copy_request; + expect_snapshot_copy(mock_snapshot_copy_request, 0, 1, 0, {{1, CEPH_NOSNAP}}, + 0); + MockGetImageStateRequest mock_get_image_state_request; + expect_get_image_state(mock_get_image_state_request, 1, 0); + MockCreateNonPrimaryRequest mock_create_non_primary_request; + expect_create_non_primary_request(mock_create_non_primary_request, + false, "remote mirror uuid", 1, + {{1, CEPH_NOSNAP}}, 11, 0); + expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, + -ECANCELED); + + // wake-up replayer + update_watch_ctx->handle_notify(); + + // wait for sync to complete and expect replay complete + ASSERT_EQ(0, wait_for_notification(1)); + ASSERT_FALSE(mock_replayer.is_replaying()); + ASSERT_EQ(-ECANCELED, mock_replayer.get_error_code()); + + ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads, + mock_local_image_ctx, + mock_remote_image_ctx)); + +} + TEST_F(TestMockImageReplayerSnapshotReplayer, CopyImageError) { librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx}; librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx}; @@ -1546,13 +1660,14 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, CopyImageError) { InSequence seq; + MockInstanceWatcher mock_instance_watcher; MockImageMeta mock_image_meta; MockStateBuilder mock_state_builder(mock_local_image_ctx, mock_remote_image_ctx, mock_image_meta); - MockReplayer mock_replayer{&mock_threads, "local mirror uuid", - &m_pool_meta_cache, &mock_state_builder, - &mock_replayer_listener}; + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; m_pool_meta_cache.set_remote_pool_meta( m_remote_io_ctx.get_id(), {"remote mirror uuid", "remote mirror peer uuid"}); @@ -1585,6 +1700,7 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, CopyImageError) { expect_create_non_primary_request(mock_create_non_primary_request, false, "remote mirror uuid", 1, {{1, CEPH_NOSNAP}}, 11, 0); + expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0); MockImageCopyRequest mock_image_copy_request; expect_image_copy(mock_image_copy_request, 0, 1, 0, {}, {{1, CEPH_NOSNAP}}, -EINVAL); @@ -1614,13 +1730,14 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, UpdateNonPrimarySnapshotError) { InSequence seq; + MockInstanceWatcher mock_instance_watcher; MockImageMeta mock_image_meta; MockStateBuilder mock_state_builder(mock_local_image_ctx, mock_remote_image_ctx, mock_image_meta); - MockReplayer mock_replayer{&mock_threads, "local mirror uuid", - &m_pool_meta_cache, &mock_state_builder, - &mock_replayer_listener}; + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; m_pool_meta_cache.set_remote_pool_meta( m_remote_io_ctx.get_id(), {"remote mirror uuid", "remote mirror peer uuid"}); @@ -1653,6 +1770,7 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, UpdateNonPrimarySnapshotError) { expect_create_non_primary_request(mock_create_non_primary_request, false, "remote mirror uuid", 1, {{1, CEPH_NOSNAP}}, 11, 0); + expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0); MockImageCopyRequest mock_image_copy_request; expect_image_copy(mock_image_copy_request, 0, 1, 0, {}, {{1, CEPH_NOSNAP}}, 0); @@ -1686,13 +1804,14 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, UnlinkPeerError) { InSequence seq; + MockInstanceWatcher mock_instance_watcher; MockImageMeta mock_image_meta; MockStateBuilder mock_state_builder(mock_local_image_ctx, mock_remote_image_ctx, mock_image_meta); - MockReplayer mock_replayer{&mock_threads, "local mirror uuid", - &m_pool_meta_cache, &mock_state_builder, - &mock_replayer_listener}; + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; m_pool_meta_cache.set_remote_pool_meta( m_remote_io_ctx.get_id(), {"remote mirror uuid", "remote mirror peer uuid"}); @@ -1734,6 +1853,7 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, UnlinkPeerError) { expect_create_non_primary_request(mock_create_non_primary_request, false, "remote mirror uuid", 2, {{2, CEPH_NOSNAP}}, 12, 0); + expect_notify_sync_request(mock_instance_watcher, mock_local_image_ctx.id, 0); MockImageCopyRequest mock_image_copy_request; expect_image_copy(mock_image_copy_request, 1, 2, 11, {}, {{2, CEPH_NOSNAP}}, 0); @@ -1745,6 +1865,7 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, UnlinkPeerError) { MockUnlinkPeerRequest mock_unlink_peer_request; expect_unlink_peer(mock_unlink_peer_request, 1, "remote mirror peer uuid", -EINVAL); + expect_notify_sync_complete(mock_instance_watcher, mock_local_image_ctx.id); // wake-up replayer update_watch_ctx->handle_notify(); @@ -1771,13 +1892,14 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, SplitBrain) { InSequence seq; + MockInstanceWatcher mock_instance_watcher; MockImageMeta mock_image_meta; MockStateBuilder mock_state_builder(mock_local_image_ctx, mock_remote_image_ctx, mock_image_meta); - MockReplayer mock_replayer{&mock_threads, "local mirror uuid", - &m_pool_meta_cache, &mock_state_builder, - &mock_replayer_listener}; + MockReplayer mock_replayer{&mock_threads, &mock_instance_watcher, + "local mirror uuid", &m_pool_meta_cache, + &mock_state_builder, &mock_replayer_listener}; m_pool_meta_cache.set_remote_pool_meta( m_remote_io_ctx.get_id(), {"remote mirror uuid", "remote mirror peer uuid"}); diff --git a/src/test/rbd_mirror/test_mock_ImageReplayer.cc b/src/test/rbd_mirror/test_mock_ImageReplayer.cc index 653b6e98bb7..20693e3223b 100644 --- a/src/test/rbd_mirror/test_mock_ImageReplayer.cc +++ b/src/test/rbd_mirror/test_mock_ImageReplayer.cc @@ -175,7 +175,8 @@ struct StateBuilder<librbd::MockTestImageCtx> { } MOCK_METHOD1(close, void(Context*)); - MOCK_METHOD4(create_replayer, Replayer*(Threads<librbd::MockTestImageCtx>*, + MOCK_METHOD5(create_replayer, Replayer*(Threads<librbd::MockTestImageCtx>*, + InstanceWatcher<librbd::MockTestImageCtx>*, const std::string&, PoolMetaCache*, ReplayerListener*)); @@ -301,8 +302,8 @@ public: void expect_create_replayer(MockStateBuilder& mock_state_builder, MockReplayer& mock_replayer) { - EXPECT_CALL(mock_state_builder, create_replayer(_, _, _, _)) - .WillOnce(WithArg<3>( + EXPECT_CALL(mock_state_builder, create_replayer(_, _, _, _, _)) + .WillOnce(WithArg<4>( Invoke([&mock_replayer] (image_replayer::ReplayerListener* replayer_listener) { mock_replayer.replayer_listener = replayer_listener; diff --git a/src/tools/rbd_mirror/ImageReplayer.cc b/src/tools/rbd_mirror/ImageReplayer.cc index 8068ef67871..dd74512dfd1 100644 --- a/src/tools/rbd_mirror/ImageReplayer.cc +++ b/src/tools/rbd_mirror/ImageReplayer.cc @@ -408,7 +408,8 @@ void ImageReplayer<I>::start_replay() { std::unique_lock locker{m_lock}; ceph_assert(m_replayer == nullptr); - m_replayer = m_state_builder->create_replayer(m_threads, m_local_mirror_uuid, + m_replayer = m_state_builder->create_replayer(m_threads, m_instance_watcher, + m_local_mirror_uuid, m_pool_meta_cache, m_replayer_listener); diff --git a/src/tools/rbd_mirror/LeaderWatcher.cc b/src/tools/rbd_mirror/LeaderWatcher.cc index 1581319219d..ae705e3c5e2 100644 --- a/src/tools/rbd_mirror/LeaderWatcher.cc +++ b/src/tools/rbd_mirror/LeaderWatcher.cc @@ -946,7 +946,7 @@ void LeaderWatcher<I>::handle_heartbeat(Context *on_notify_ack) { std::scoped_lock locker{m_threads->timer_lock, m_lock}; if (is_leader(m_lock)) { dout(5) << "got another leader heartbeat, ignoring" << dendl; - } else { + } else if (!m_locker.cookie.empty()) { cancel_timer_task(); m_acquire_attempts = 0; schedule_acquire_leader_lock(1); diff --git a/src/tools/rbd_mirror/image_replayer/StateBuilder.h b/src/tools/rbd_mirror/image_replayer/StateBuilder.h index 99f5fce4487..d055c84e020 100644 --- a/src/tools/rbd_mirror/image_replayer/StateBuilder.h +++ b/src/tools/rbd_mirror/image_replayer/StateBuilder.h @@ -15,6 +15,7 @@ namespace rbd { namespace mirror { struct BaseRequest; +template <typename> class InstanceWatcher; struct PoolMetaCache; struct ProgressContext; template <typename> class Threads; @@ -73,6 +74,7 @@ public: virtual Replayer* create_replayer( Threads<ImageCtxT>* threads, + InstanceWatcher<ImageCtxT>* instance_watcher, const std::string& local_mirror_uuid, PoolMetaCache* pool_meta_cache, ReplayerListener* replayer_listener) = 0; diff --git a/src/tools/rbd_mirror/image_replayer/journal/StateBuilder.cc b/src/tools/rbd_mirror/image_replayer/journal/StateBuilder.cc index c28e0694823..8d391580ed7 100644 --- a/src/tools/rbd_mirror/image_replayer/journal/StateBuilder.cc +++ b/src/tools/rbd_mirror/image_replayer/journal/StateBuilder.cc @@ -104,7 +104,8 @@ BaseRequest* StateBuilder<I>::create_prepare_replay_request( template <typename I> image_replayer::Replayer* StateBuilder<I>::create_replayer( - Threads<I>* threads, + Threads<I>* threads, + InstanceWatcher<I>* instance_watcher, const std::string& local_mirror_uuid, PoolMetaCache* pool_meta_cache, ReplayerListener* replayer_listener) { diff --git a/src/tools/rbd_mirror/image_replayer/journal/StateBuilder.h b/src/tools/rbd_mirror/image_replayer/journal/StateBuilder.h index 3f3621727ce..1d4fa83cbb0 100644 --- a/src/tools/rbd_mirror/image_replayer/journal/StateBuilder.h +++ b/src/tools/rbd_mirror/image_replayer/journal/StateBuilder.h @@ -60,6 +60,7 @@ public: image_replayer::Replayer* create_replayer( Threads<ImageCtxT>* threads, + InstanceWatcher<ImageCtxT>* instance_watcher, const std::string& local_mirror_uuid, PoolMetaCache* pool_meta_cache, ReplayerListener* replayer_listener) override; diff --git a/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.cc b/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.cc index 2bb495913a5..69fb3d93af4 100644 --- a/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.cc +++ b/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.cc @@ -17,6 +17,7 @@ #include "librbd/mirror/snapshot/GetImageStateRequest.h" #include "librbd/mirror/snapshot/ImageMeta.h" #include "librbd/mirror/snapshot/UnlinkPeerRequest.h" +#include "tools/rbd_mirror/InstanceWatcher.h" #include "tools/rbd_mirror/PoolMetaCache.h" #include "tools/rbd_mirror/Threads.h" #include "tools/rbd_mirror/Types.h" @@ -110,11 +111,13 @@ struct Replayer<I>::ProgressContext : public librbd::ProgressContext { template <typename I> Replayer<I>::Replayer( Threads<I>* threads, + InstanceWatcher<I>* instance_watcher, const std::string& local_mirror_uuid, PoolMetaCache* pool_meta_cache, StateBuilder<I>* state_builder, ReplayerListener* replayer_listener) : m_threads(threads), + m_instance_watcher(instance_watcher), m_local_mirror_uuid(local_mirror_uuid), m_pool_meta_cache(pool_meta_cache), m_state_builder(state_builder), @@ -172,6 +175,10 @@ void Replayer<I>::shut_down(Context* on_finish) { std::swap(m_state, state); if (state == STATE_REPLAYING) { + // if a sync request was pending, request a cancelation + m_instance_watcher->cancel_sync_request( + m_state_builder->local_image_ctx->id); + // TODO interrupt snapshot copy and image copy state machines even if remote // cluster is unreachable dout(10) << "shut down pending on completion of snapshot replay" << dendl; @@ -708,7 +715,7 @@ void Replayer<I>::handle_get_local_image_state(int r) { return; } - copy_image(); + request_sync(); } template <typename I> @@ -737,6 +744,41 @@ void Replayer<I>::handle_create_non_primary_snapshot(int r) { dout(15) << "local_snap_id_end=" << m_local_snap_id_end << dendl; + request_sync(); +} + +template <typename I> +void Replayer<I>::request_sync() { + dout(10) << dendl; + + std::unique_lock locker{m_lock}; + auto ctx = create_async_context_callback( + m_threads->work_queue, create_context_callback< + Replayer<I>, &Replayer<I>::handle_request_sync>(this)); + m_instance_watcher->notify_sync_request(m_state_builder->local_image_ctx->id, + ctx); +} + +template <typename I> +void Replayer<I>::handle_request_sync(int r) { + dout(10) << "r=" << r << dendl; + + std::unique_lock locker{m_lock}; + if (is_replay_interrupted(&locker)) { + return; + } else if (r == -ECANCELED) { + dout(5) << "image-sync canceled" << dendl; + handle_replay_complete(&locker, r, "image-sync canceled"); + return; + } else if (r < 0) { + derr << "failed to request image-sync: " << cpp_strerror(r) << dendl; + handle_replay_complete(&locker, r, "failed to request image-sync"); + return; + } + + m_sync_in_progress = true; + locker.unlock(); + copy_image(); } @@ -897,22 +939,13 @@ void Replayer<I>::handle_notify_image_update(int r) { derr << "failed to notify local image update: " << cpp_strerror(r) << dendl; } - if (is_replay_interrupted()) { - return; - } - unlink_peer(); } template <typename I> void Replayer<I>::unlink_peer() { if (m_remote_snap_id_start == 0) { - { - std::unique_lock locker{m_lock}; - notify_status_updated(); - } - - load_local_image_meta(); + finish_sync(); return; } @@ -939,9 +972,24 @@ void Replayer<I>::handle_unlink_peer(int r) { return; } + finish_sync(); +} + +template <typename I> +void Replayer<I>::finish_sync() { + dout(10) << dendl; + { std::unique_lock locker{m_lock}; notify_status_updated(); + + m_sync_in_progress = false; + m_instance_watcher->notify_sync_complete( + m_state_builder->local_image_ctx->id); + } + + if (is_replay_interrupted()) { + return; } load_local_image_meta(); @@ -1135,6 +1183,12 @@ void Replayer<I>::handle_replay_complete(std::unique_lock<ceph::mutex>* locker, m_error_description = description; } + if (m_sync_in_progress) { + m_sync_in_progress = false; + m_instance_watcher->notify_sync_complete( + m_state_builder->local_image_ctx->id); + } + if (m_state != STATE_REPLAYING && m_state != STATE_IDLE) { return; } diff --git a/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.h b/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.h index e5ae54c6272..e7536f5a6fa 100644 --- a/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.h +++ b/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.h @@ -22,6 +22,7 @@ namespace snapshot { template <typename I> class Replay; } namespace rbd { namespace mirror { +template <typename> struct InstanceWatcher; class PoolMetaCache; template <typename> struct Threads; @@ -40,16 +41,18 @@ class Replayer : public image_replayer::Replayer { public: static Replayer* create( Threads<ImageCtxT>* threads, + InstanceWatcher<ImageCtxT>* instance_watcher, const std::string& local_mirror_uuid, PoolMetaCache* pool_meta_cache, StateBuilder<ImageCtxT>* state_builder, ReplayerListener* replayer_listener) { - return new Replayer(threads, local_mirror_uuid, pool_meta_cache, - state_builder, replayer_listener); + return new Replayer(threads, instance_watcher, local_mirror_uuid, + pool_meta_cache, state_builder, replayer_listener); } Replayer( Threads<ImageCtxT>* threads, + InstanceWatcher<ImageCtxT>* instance_watcher, const std::string& local_mirror_uuid, PoolMetaCache* pool_meta_cache, StateBuilder<ImageCtxT>* state_builder, @@ -123,6 +126,9 @@ private: * | |/----------------------/ | * | | | * | v | + * | REQUEST_SYNC | + * | | | + * | v | * | COPY_IMAGE | * | | | * | v | @@ -180,6 +186,7 @@ private: struct ProgressContext; Threads<ImageCtxT>* m_threads; + InstanceWatcher<ImageCtxT>* m_instance_watcher; std::string m_local_mirror_uuid; PoolMetaCache* m_pool_meta_cache; StateBuilder<ImageCtxT>* m_state_builder; @@ -217,6 +224,7 @@ private: bool m_remote_image_updated = false; bool m_updating_sync_point = false; + bool m_sync_in_progress = false; void load_local_image_meta(); void handle_load_local_image_meta(int r); @@ -242,6 +250,9 @@ private: void create_non_primary_snapshot(); void handle_create_non_primary_snapshot(int r); + void request_sync(); + void handle_request_sync(int r); + void copy_image(); void handle_copy_image(int r); void handle_copy_image_progress(uint64_t object_number, @@ -259,6 +270,8 @@ private: void unlink_peer(); void handle_unlink_peer(int r); + void finish_sync(); + void register_local_update_watcher(); void handle_register_local_update_watcher(int r); diff --git a/src/tools/rbd_mirror/image_replayer/snapshot/StateBuilder.cc b/src/tools/rbd_mirror/image_replayer/snapshot/StateBuilder.cc index 27225707f55..2fb2534d642 100644 --- a/src/tools/rbd_mirror/image_replayer/snapshot/StateBuilder.cc +++ b/src/tools/rbd_mirror/image_replayer/snapshot/StateBuilder.cc @@ -103,12 +103,14 @@ BaseRequest* StateBuilder<I>::create_prepare_replay_request( template <typename I> image_replayer::Replayer* StateBuilder<I>::create_replayer( - Threads<I>* threads, + Threads<I>* threads, + InstanceWatcher<I>* instance_watcher, const std::string& local_mirror_uuid, PoolMetaCache* pool_meta_cache, ReplayerListener* replayer_listener) { return Replayer<I>::create( - threads, local_mirror_uuid, pool_meta_cache, this, replayer_listener); + threads, instance_watcher, local_mirror_uuid, pool_meta_cache, this, + replayer_listener); } } // namespace snapshot diff --git a/src/tools/rbd_mirror/image_replayer/snapshot/StateBuilder.h b/src/tools/rbd_mirror/image_replayer/snapshot/StateBuilder.h index 83c55ebafdc..ab326255fdb 100644 --- a/src/tools/rbd_mirror/image_replayer/snapshot/StateBuilder.h +++ b/src/tools/rbd_mirror/image_replayer/snapshot/StateBuilder.h @@ -69,6 +69,7 @@ public: image_replayer::Replayer* create_replayer( Threads<ImageCtxT>* threads, + InstanceWatcher<ImageCtxT>* instance_watcher, const std::string& local_mirror_uuid, PoolMetaCache* pool_meta_cache, ReplayerListener* replayer_listener) override; |