osd: add a separate messenger for the Objecter

We will give the OSD's Objecter its own messenger so that it does not
interfere with the OSD when it marks things up or down.

Signed-off-by: Sage Weil <sage@inktank.com>

3 files changed, 14 insertions, 0 deletions

diff --git a/src/ceph_osd.cc b/src/ceph_osd.cc
index dc6f435bdcf..4c6702459e7 100644
--- a/src/ceph_osd.cc
+++ b/src/ceph_osd.cc
@@ -331,6 +331,9 @@ int main(int argc, const char **argv)
   Messenger *messenger_hb_front_server = Messenger::create(g_ceph_context,
 						    entity_name_t::OSD(whoami), "hb_front_server",
 						    getpid());
+  Messenger *messenger_objecter = Messenger::create(g_ceph_context,
+						    entity_name_t::OSD(whoami), "hbclient",
+						    getpid());
   cluster_messenger->set_cluster_protocol(CEPH_OSD_PROTOCOL);
   messenger_hbclient->set_cluster_protocol(CEPH_OSD_PROTOCOL);
   messenger_hb_back_server->set_cluster_protocol(CEPH_OSD_PROTOCOL);
@@ -386,6 +389,8 @@ int main(int argc, const char **argv)
   messenger_hb_front_server->set_policy(entity_name_t::TYPE_OSD,
 					Messenger::Policy::stateless_server(0, 0));
 
+  messenger_objecter->set_default_policy(Messenger::Policy::lossy_client(0, CEPH_FEATURE_OSDREPLYMUX));
+
   r = client_messenger->bind(g_conf->public_addr);
   if (r < 0)
     exit(1);
@@ -432,6 +437,7 @@ int main(int argc, const char **argv)
 
   osd = new OSD(whoami, cluster_messenger, client_messenger,
 		messenger_hbclient, messenger_hb_front_server, messenger_hb_back_server,
+		messenger_objecter,
 		&mc,
 		g_conf->osd_data, g_conf->osd_journal);
 
@@ -450,6 +456,7 @@ int main(int argc, const char **argv)
   messenger_hb_front_server->start();
   messenger_hb_back_server->start();
   cluster_messenger->start();
+  messenger_objecter->start();
 
   // start osd
   err = osd->init();
@@ -475,6 +482,7 @@ int main(int argc, const char **argv)
   messenger_hb_front_server->wait();
   messenger_hb_back_server->wait();
   cluster_messenger->wait();
+  messenger_objecter->wait();
 
   unregister_async_signal_handler(SIGHUP, sighup_handler);
   unregister_async_signal_handler(SIGINT, handle_osd_signal);
@@ -488,6 +496,7 @@ int main(int argc, const char **argv)
   delete messenger_hb_front_server;
   delete messenger_hb_back_server;
   delete cluster_messenger;
+  delete messenger_objecter;
   client_byte_throttler.reset();
   client_msg_throttler.reset();
   g_ceph_context->put();
diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 66022a3898a..5bde23335ad 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -882,6 +882,7 @@ OSD::OSD(int id, Messenger *internal_messenger, Messenger *external_messenger,
 	 Messenger *hb_clientm,
 	 Messenger *hb_front_serverm,
 	 Messenger *hb_back_serverm,
+	 Messenger *osdc_messenger,
 	 MonClient *mc,
 	 const std::string &dev, const std::string &jdev) :
   Dispatcher(external_messenger->cct),
@@ -897,6 +898,7 @@ OSD::OSD(int id, Messenger *internal_messenger, Messenger *external_messenger,
 								      cct->_conf->auth_service_required)),
   cluster_messenger(internal_messenger),
   client_messenger(external_messenger),
+  objecter_messenger(osdc_messenger),
   monc(mc),
   logger(NULL),
   recoverystate_perf(NULL),
@@ -1646,6 +1648,7 @@ int OSD::shutdown()
   client_messenger->shutdown();
   cluster_messenger->shutdown();
   hbclient_messenger->shutdown();
+  objecter_messenger->shutdown();
   hb_front_server_messenger->shutdown();
   hb_back_server_messenger->shutdown();
   peering_wq.clear();
diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index eae4bab21ca..bef71fdb28f 100644
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -627,6 +627,7 @@ protected:
 
   Messenger   *cluster_messenger;
   Messenger   *client_messenger;
+  Messenger   *objecter_messenger;
   MonClient   *monc;
   PerfCounters      *logger;
   PerfCounters      *recoverystate_perf;
@@ -1672,6 +1673,7 @@ protected:
    * be cleaned up properly*/
   OSD(int id, Messenger *internal, Messenger *external,
       Messenger *hb_client, Messenger *hb_front_server, Messenger *hb_back_server,
+      Messenger *osdc_messenger,
       MonClient *mc, const std::string &dev, const std::string &jdev);
   ~OSD();