client: remove requests from closed MetaSession

If we get a CLOSED message on a session, remove/kick any requests on
that session before tearing it down.  Otherwise, we get a crash like

2013-09-26 03:51:44.704446 7f4d35a46700 10 client.4111 kick_requests for mds.0
2013-09-26 03:51:45.014156 7f4d35a46700 -1 ./include/xlist.h: In function 'xlist<T>::~xlist() [with T = MetaRequest*]' thread 7f4d35a46700 time 2013-09-26 03:51:44.751908
./include/xlist.h: 69: FAILED assert(_size == 0)

 ceph version 0.61.5 (8ee10dc4bb73bdd918873f29c70eedc3c7ef1979)
 1: (MetaSession::~MetaSession()+0x425) [0x4e0105]
 2: (Client::_closed_mds_session(MetaSession*)+0x116) [0x48a696]
 3: (Client::handle_client_session(MClientSession*)+0x2bb) [0x48bf5b]
 4: (Client::ms_dispatch(Message*)+0x56b) [0x4bfa0b]
 5: (DispatchQueue::entry()+0x3f1) [0x621b31]
 6: (DispatchQueue::DispatchThread::entry()+0xd) [0x6191bd]
 7: (()+0x7851) [0x7f4d3c168851]
 8: (clone()+0x6d) [0x7f4d3b09d90d]

Note that this can happen if we fail to reconnect do an MDS during its
reconnect interval.  If that happens, we probably have inodes in our
cache with no caps and things are generally not going to work very well.
This is but one step in improving the situation.

Separate out the two methods since they share little/no behavior.

Signed-off-by: Sage Weil <sage@inktank.com>

2 files changed, 27 insertions, 16 deletions

diff --git a/src/client/Client.cc b/src/client/Client.cc
index 285b7c543c6..60a5e4550b8 100644
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -1541,7 +1541,7 @@ void Client::_closed_mds_session(MetaSession *s)
   signal_context_list(s->waiting_for_open);
   mount_cond.Signal();
   remove_session_caps(s);
-  kick_requests(s, true);
+  kick_requests_closed(s);
   mds_sessions.erase(s->mds_num);
   delete s;
 }
@@ -1914,7 +1914,7 @@ void Client::handle_mds_map(MMDSMap* m)
 
     if (newstate >= MDSMap::STATE_ACTIVE) {
       if (oldstate < MDSMap::STATE_ACTIVE) {
-	kick_requests(p->second, false);
+	kick_requests(p->second);
 	kick_flushing_caps(p->second);
 	signal_context_list(p->second->waiting_for_open);
 	kick_maxsize_requests(p->second);
@@ -1998,25 +1998,16 @@ void Client::send_reconnect(MetaSession *session)
 }
 
 
-void Client::kick_requests(MetaSession *session, bool signal)
+void Client::kick_requests(MetaSession *session)
 {
   ldout(cct, 10) << "kick_requests for mds." << session->mds_num << dendl;
-
   for (map<tid_t, MetaRequest*>::iterator p = mds_requests.begin();
        p != mds_requests.end();
-       ++p) 
+       ++p) {
     if (p->second->mds == session->mds_num) {
-      if (signal) {
-	// only signal caller if there is a caller
-	// otherwise, let resend_unsafe handle it
-	if (p->second->caller_cond) {
-	  p->second->kick = true;
-	  p->second->caller_cond->Signal();
-	}
-      } else {
-	send_request(p->second, session);
-      }
+      send_request(p->second, session);
     }
+  }
 }
 
 void Client::resend_unsafe_requests(MetaSession *session)
@@ -2027,6 +2018,25 @@ void Client::resend_unsafe_requests(MetaSession *session)
     send_request(*iter, session);
 }
 
+void Client::kick_requests_closed(MetaSession *session)
+{
+  ldout(cct, 10) << "kick_requests_closed for mds." << session->mds_num << dendl;
+  for (map<tid_t, MetaRequest*>::iterator p = mds_requests.begin();
+       p != mds_requests.end();
+       ++p) {
+    if (p->second->mds == session->mds_num) {
+      if (p->second->caller_cond) {
+	p->second->kick = true;
+	p->second->caller_cond->Signal();
+      }
+      p->second->item.remove_myself();
+      p->second->unsafe_item.remove_myself();
+    }
+  }
+  assert(session->requests.empty());
+  assert(session->unsafe_requests.empty());
+}
+
 
 
 
diff --git a/src/client/Client.h b/src/client/Client.h
index 61f29f39120..df59f235de4 100644
--- a/src/client/Client.h
+++ b/src/client/Client.h
@@ -277,7 +277,8 @@ public:
   void connect_mds_targets(int mds);
   void send_request(MetaRequest *request, MetaSession *session);
   MClientRequest *build_client_request(MetaRequest *request);
-  void kick_requests(MetaSession *session, bool signal);
+  void kick_requests(MetaSession *session);
+  void kick_requests_closed(MetaSession *session);
   void handle_client_request_forward(MClientRequestForward *reply);
   void handle_client_reply(MClientReply *reply);