mon: handle -EAGAIN in completion contexts

We can get ECANCELED, EAGAIN, or success out of the completion contexts,
but in the EAGAIN case (meaning there was an election) we were sending
a success to the client.  This resulted in client hangs and all-around
confusion when the monitor cluster was thrashing.

Backport: bobtail
Signed-off-by: Sage Weil <sage@inktank.com>
Reviewed-by: Joao Luis <joao.luis@inktank.com>

1 files changed, 14 insertions, 16 deletions

diff --git a/src/mon/OSDMonitor.h b/src/mon/OSDMonitor.h
index 9529f731c84..f53b6285abb 100644
--- a/src/mon/OSDMonitor.h
+++ b/src/mon/OSDMonitor.h
@@ -209,14 +209,10 @@ private:
     C_Booted(OSDMonitor *cm, MOSDBoot *m_, bool l=true) : 
       cmon(cm), m(m_), logit(l) {}
     void finish(int r) {
-      if (r == -ECANCELED) {
-	if (m)
-	  m->put();
-	return;
-      }
-
       if (r >= 0)
 	cmon->_booted(m, logit);
+      else if (r == -ECANCELED)
+	m->put();
       else
 	cmon->dispatch((PaxosServiceMessage*)m);
     }
@@ -228,12 +224,13 @@ private:
     epoch_t e;
     C_ReplyMap(OSDMonitor *o, PaxosServiceMessage *mm, epoch_t ee) : osdmon(o), m(mm), e(ee) {}
     void finish(int r) {
-      if (r == -ECANCELED) {
-	if (m)
-	  m->put();
-	return;
+      if (r >= 0) {
+	osdmon->_reply_map(m, e);
+      } else if (r == -ECANCELED) {
+	m->put();
+      } else {
+	osdmon->dispatch(m);
       }
-      osdmon->_reply_map(m, e);
     }    
   };
   struct C_PoolOp : public Context {
@@ -248,12 +245,13 @@ private:
 	reply_data = *rd;
     }
     void finish(int r) {
-      if (r == -ECANCELED) {
-	if (m)
-	  m->put();
-	return;
+      if (r >= 0) {
+	osdmon->_pool_op_reply(m, replyCode, epoch, &reply_data);
+      } else if (r == -ECANCELED) {
+	m->put();
+      } else {
+	osdmon->dispatch(m);
       }
-      osdmon->_pool_op_reply(m, replyCode, epoch, &reply_data);
     }
   };