osd: add a separate messenger for the Objecter

We will give the OSD's Objecter its own messenger so that it does not
interfere with the OSD when it marks things up or down.

Signed-off-by: Sage Weil <sage@inktank.com>

3 files changed, 14 insertions, 0 deletions

diff --git a/src/ceph_osd.cc b/src/ceph_osd.cc
index d8590bff817..23d6c360d86 100644
--- a/src/ceph_osd.cc
+++ b/src/ceph_osd.cc
@@ -331,6 +331,9 @@ int main(int argc, const char **argv)
   Messenger *messenger_hb_front_server = Messenger::create(g_ceph_context,
 						    entity_name_t::OSD(whoami), "hb_front_server",
 						    getpid());
+  Messenger *messenger_objecter = Messenger::create(g_ceph_context,
+						    entity_name_t::OSD(whoami), "hbclient",
+						    getpid());
   cluster_messenger->set_cluster_protocol(CEPH_OSD_PROTOCOL);
   messenger_hbclient->set_cluster_protocol(CEPH_OSD_PROTOCOL);
   messenger_hb_back_server->set_cluster_protocol(CEPH_OSD_PROTOCOL);
@@ -386,6 +389,8 @@ int main(int argc, const char **argv)
   messenger_hb_front_server->set_policy(entity_name_t::TYPE_OSD,
 					Messenger::Policy::stateless_server(0, 0));
 
+  messenger_objecter->set_default_policy(Messenger::Policy::lossy_client(0, CEPH_FEATURE_OSDREPLYMUX));
+
   r = client_messenger->bind(g_conf->public_addr);
   if (r < 0)
     exit(1);
@@ -432,6 +437,7 @@ int main(int argc, const char **argv)
 
   osd = new OSD(whoami, cluster_messenger, client_messenger,
 		messenger_hbclient, messenger_hb_front_server, messenger_hb_back_server,
+		messenger_objecter,
 		&mc,
 		g_conf->osd_data, g_conf->osd_journal);
 
@@ -450,6 +456,7 @@ int main(int argc, const char **argv)
   messenger_hb_front_server->start();
   messenger_hb_back_server->start();
   cluster_messenger->start();
+  messenger_objecter->start();
 
   // start osd
   err = osd->init();
@@ -473,6 +480,7 @@ int main(int argc, const char **argv)
   messenger_hb_front_server->wait();
   messenger_hb_back_server->wait();
   cluster_messenger->wait();
+  messenger_objecter->wait();
 
   unregister_async_signal_handler(SIGHUP, sighup_handler);
   unregister_async_signal_handler(SIGINT, handle_osd_signal);
@@ -486,6 +494,7 @@ int main(int argc, const char **argv)
   delete messenger_hb_front_server;
   delete messenger_hb_back_server;
   delete cluster_messenger;
+  delete messenger_objecter;
   client_byte_throttler.reset();
   client_msg_throttler.reset();
   g_ceph_context->put();
diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 3e19f5634c1..f2f6d1b4de2 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -882,6 +882,7 @@ OSD::OSD(int id, Messenger *internal_messenger, Messenger *external_messenger,
 	 Messenger *hb_clientm,
 	 Messenger *hb_front_serverm,
 	 Messenger *hb_back_serverm,
+	 Messenger *osdc_messenger,
 	 MonClient *mc,
 	 const std::string &dev, const std::string &jdev) :
   Dispatcher(external_messenger->cct),
@@ -897,6 +898,7 @@ OSD::OSD(int id, Messenger *internal_messenger, Messenger *external_messenger,
 								      cct->_conf->auth_service_required)),
   cluster_messenger(internal_messenger),
   client_messenger(external_messenger),
+  objecter_messenger(osdc_messenger),
   monc(mc),
   logger(NULL),
   recoverystate_perf(NULL),
@@ -1642,6 +1644,7 @@ int OSD::shutdown()
   client_messenger->shutdown();
   cluster_messenger->shutdown();
   hbclient_messenger->shutdown();
+  objecter_messenger->shutdown();
   hb_front_server_messenger->shutdown();
   hb_back_server_messenger->shutdown();
   peering_wq.clear();
diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index f0889df654e..1cec3bb24f7 100644
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -627,6 +627,7 @@ protected:
 
   Messenger   *cluster_messenger;
   Messenger   *client_messenger;
+  Messenger   *objecter_messenger;
   MonClient   *monc;
   PerfCounters      *logger;
   PerfCounters      *recoverystate_perf;
@@ -1672,6 +1673,7 @@ protected:
    * be cleaned up properly*/
   OSD(int id, Messenger *internal, Messenger *external,
       Messenger *hb_client, Messenger *hb_front_server, Messenger *hb_back_server,
+      Messenger *osdc_messenger,
       MonClient *mc, const std::string &dev, const std::string &jdev);
   ~OSD();