Revert "osd: simplify finalizing scrub on replica"

This reverts commit dd5087fabb2a743741a96ee4610379afa8431f68.

Calling osr.flush() is not quite enough since the onreadable callbacks
may not have been called (thus, last_update_applied may still lag behind
the tail of the log).

Signed-off-by: Samuel Just <samuel.just@dreamhost.com>

3 files changed, 25 insertions, 6 deletions

diff --git a/src/osd/PG.cc b/src/osd/PG.cc
index 384a5eb8521..d8548e029f0 100644
--- a/src/osd/PG.cc
+++ b/src/osd/PG.cc
@@ -2840,11 +2840,15 @@ void PG::repair_object(const hobject_t& soid, ScrubMap::object *po, int bad_peer
  * If msg->scrub_from is not set, replica_scrub calls build_scrubmap to
  * build a complete map (with the pg lock dropped).
  *
- * If msg->scrub_from is set, we build an incremental scrub map with
- * the pg lock held.
+ * If msg->scrub_from is set, replica_scrub sets finalizing_scrub.
+ * Similarly to scrub, if last_update_applied is behind info.last_update
+ * replica_scrub returns to be requeued by sub_op_modify_applied.
+ * replica_scrub then builds an incremental scrub map with the 
+ * pg lock held.
  */
 void PG::replica_scrub(MOSDRepScrub *msg)
 {
+  assert(!active_rep_scrub);
   dout(7) << "replica_scrub" << dendl;
 
   if (msg->map_epoch < info.history.same_interval_since) {
@@ -2862,10 +2866,17 @@ void PG::replica_scrub(MOSDRepScrub *msg)
 
   ScrubMap map;
   if (msg->scrub_from > eversion_t()) {
-    // flush out in-flight writes to disk, so we can scrub the
-    // resulting on-disk state.
-    osr.flush();
+    if (finalizing_scrub) {
+      assert(last_update_applied == info.last_update);
+    } else {
+      finalizing_scrub = 1;
+      if (last_update_applied != info.last_update) {
+	active_rep_scrub = msg;
+	return;
+      }
+    }
     build_inc_scrub_map(map, msg->scrub_from);
+    finalizing_scrub = 0;
   } else {
     build_scrub_map(map);
   }
diff --git a/src/osd/PG.h b/src/osd/PG.h
index 2084f89a24c..5b6353bb2fd 100644
--- a/src/osd/PG.h
+++ b/src/osd/PG.h
@@ -1517,6 +1517,7 @@ public:
   int scrub_waiting_on;
   epoch_t scrub_epoch_start;
   ScrubMap primary_scrubmap;
+  MOSDRepScrub *active_rep_scrub;
 
   void repair_object(const hobject_t& soid, ScrubMap::object *po, int bad_peer, int ok_peer);
   bool _compare_scrub_objects(ScrubMap::object &auth,
@@ -1569,7 +1570,8 @@ public:
     finish_sync_event(NULL),
     finalizing_scrub(false),
     scrub_reserved(false), scrub_reserve_failed(false),
-    scrub_waiting_on(0)
+    scrub_waiting_on(0),
+    active_rep_scrub(0)
   {
     pool->get();
   }
diff --git a/src/osd/ReplicatedPG.cc b/src/osd/ReplicatedPG.cc
index d277016692b..5a356862541 100644
--- a/src/osd/ReplicatedPG.cc
+++ b/src/osd/ReplicatedPG.cc
@@ -3450,6 +3450,12 @@ void ReplicatedPG::sub_op_modify_applied(RepModify *rm)
   bool done = rm->applied && rm->committed;
 
   last_update_applied = rm->op->version;
+  if (last_update_applied == info.last_update && finalizing_scrub) {
+    assert(active_rep_scrub);
+    osd->rep_scrub_wq.queue(active_rep_scrub);
+    active_rep_scrub->put();
+    active_rep_scrub = 0;
+  }
 
   unlock();
   if (done) {