osd: add an Objecter instance

It gets its own lock, timer, and osdmap.

Signed-off-by: Sage Weil <sage@inktank.com>

5 files changed, 91 insertions, 4 deletions

diff --git a/src/Makefile.am b/src/Makefile.am
index 4b09c23e872..3bc5cbe0e11 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -96,7 +96,7 @@ bin_PROGRAMS += ceph_mon_store_converter
 
 # osd
 ceph_osd_SOURCES = ceph_osd.cc
-ceph_osd_LDADD = libosd.a $(LIBOS_LDA) $(LIBGLOBAL_LDA)
+ceph_osd_LDADD = libosd.a libosdc.la $(LIBOS_LDA) $(LIBGLOBAL_LDA)
 ceph_osd_CXXFLAGS = ${AM_CXXFLAGS}
 bin_PROGRAMS += ceph-osd
 
@@ -119,7 +119,7 @@ ceph_authtool_SOURCES = ceph_authtool.cc
 ceph_authtool_LDADD = $(LIBGLOBAL_LDA)
 ceph_filestore_dump_SOURCES = tools/ceph-filestore-dump.cc
 ceph_filestore_dump_SOURCES += perfglue/disabled_heap_profiler.cc
-ceph_filestore_dump_LDADD = libosd.a $(LIBOS_LDA) $(LIBGLOBAL_LDA) -lboost_program_options
+ceph_filestore_dump_LDADD = libosd.a libosdc.la $(LIBOS_LDA) $(LIBGLOBAL_LDA) -lboost_program_options
 if LINUX
 ceph_filestore_dump_LDADD += -ldl
 endif
@@ -835,7 +835,7 @@ check_PROGRAMS += unittest_osd_types
 
 unittest_pglog_SOURCES = test/osd/TestPGLog.cc perfglue/disabled_heap_profiler.cc
 unittest_pglog_CXXFLAGS = ${AM_CXXFLAGS} ${UNITTEST_CXXFLAGS}
-unittest_pglog_LDADD =  libosd.a $(LIBOS_LDA) $(LIBGLOBAL_LDA) ${UNITTEST_LDADD} 
+unittest_pglog_LDADD =  libosd.a libosdc.la $(LIBOS_LDA) $(LIBGLOBAL_LDA) ${UNITTEST_LDADD}
 check_PROGRAMS += unittest_pglog
 
 if LINUX
@@ -1733,7 +1733,7 @@ libosd_a_SOURCES = \
 	osd/OpRequest.cc \
 	osd/SnapMapper.cc \
 	objclass/class_api.cc
-libosd_a_CXXFLAGS= ${AM_CXXFLAGS}
+libosd_a_CXXFLAGS = ${AM_CXXFLAGS}
 noinst_LIBRARIES += libosd.a
 
 libosdc_la_SOURCES = \
diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index aac0bffe212..a29b4e3efd6 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -33,6 +33,7 @@
 #include "OSD.h"
 #include "OSDMap.h"
 #include "Watch.h"
+#include "osdc/Objecter.h"
 
 #include "common/ceph_argparse.h"
 #include "common/version.h"
@@ -175,6 +176,12 @@ OSDService::OSDService(OSD *osd) :
   pre_publish_lock("OSDService::pre_publish_lock"),
   sched_scrub_lock("OSDService::sched_scrub_lock"), scrubs_pending(0),
   scrubs_active(0),
+  objecter_lock("OSD::objecter_lock"),
+  objecter_timer(osd->client_messenger->cct, objecter_lock),
+  objecter(new Objecter(osd->client_messenger->cct, osd->objecter_messenger, osd->monc, &objecter_osdmap,
+			objecter_lock, objecter_timer)),
+  objecter_finisher(osd->client_messenger->cct),
+  objecter_dispatcher(this),
   watch_lock("OSD::watch_lock"),
   watch_timer(osd->client_messenger->cct, watch_lock),
   next_notif_id(0),
@@ -202,6 +209,11 @@ OSDService::OSDService(OSD *osd) :
 #endif
 {}
 
+OSDService::~OSDService()
+{
+  delete objecter;
+}
+
 void OSDService::_start_split(pg_t parent, const set<pg_t> &children)
 {
   for (set<pg_t>::const_iterator i = children.begin();
@@ -385,6 +397,15 @@ void OSDService::shutdown()
     Mutex::Locker l(watch_lock);
     watch_timer.shutdown();
   }
+
+  {
+    Mutex::Locker l(objecter_lock);
+    objecter_timer.shutdown();
+    objecter->shutdown_locked();
+  }
+  objecter->shutdown_unlocked();
+  objecter_finisher.stop();
+
   {
     Mutex::Locker l(backfill_request_lock);
     backfill_request_timer.shutdown();
@@ -396,6 +417,14 @@ void OSDService::shutdown()
 void OSDService::init()
 {
   reserver_finisher.start();
+  {
+    objecter_finisher.start();
+    objecter->init_unlocked();
+    Mutex::Locker l(objecter_lock);
+    objecter_timer.init();
+    objecter->set_client_incarnation(0);
+    objecter->init_locked();
+  }
   watch_timer.init();
 }
 
@@ -1214,6 +1243,8 @@ int OSD::init()
   hb_front_server_messenger->add_dispatcher_head(&heartbeat_dispatcher);
   hb_back_server_messenger->add_dispatcher_head(&heartbeat_dispatcher);
 
+  objecter_messenger->add_dispatcher_head(&service.objecter_dispatcher);
+
   monc->set_want_keys(CEPH_ENTITY_TYPE_MON | CEPH_ENTITY_TYPE_OSD);
   r = monc->init();
   if (r < 0)
@@ -4379,6 +4410,37 @@ bool OSD::heartbeat_dispatch(Message *m)
   return true;
 }
 
+bool OSDService::ObjecterDispatcher::ms_dispatch(Message *m)
+{
+  Mutex::Locker l(osd->objecter_lock);
+  osd->objecter->dispatch(m);
+  return true;
+}
+
+bool OSDService::ObjecterDispatcher::ms_handle_reset(Connection *con)
+{
+  Mutex::Locker l(osd->objecter_lock);
+  osd->objecter->ms_handle_reset(con);
+  return true;
+}
+
+void OSDService::ObjecterDispatcher::ms_handle_connect(Connection *con)
+{
+  Mutex::Locker l(osd->objecter_lock);
+  return osd->objecter->ms_handle_connect(con);
+}
+
+bool OSDService::ObjecterDispatcher::ms_get_authorizer(int dest_type,
+						       AuthAuthorizer **authorizer,
+						       bool force_new)
+{
+  if (dest_type == CEPH_ENTITY_TYPE_MON)
+    return true;
+  *authorizer = osd->monc->auth->build_authorizer(dest_type);
+  return *authorizer != NULL;
+}
+
+
 bool OSD::ms_dispatch(Message *m)
 {
   if (m->get_type() == MSG_OSD_MARK_ME_DOWN) {
diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index 9930e6f0812..bd5e3d0bbbd 100644
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -161,6 +161,7 @@ class OSDMap;
 class MLog;
 class MClass;
 class MOSDPGMissing;
+class Objecter;
 
 class Watch;
 class Notification;
@@ -417,6 +418,26 @@ public:
   void reply_op_error(OpRequestRef op, int err, eversion_t v, version_t uv);
   void handle_misdirected_op(PG *pg, OpRequestRef op);
 
+  // -- Objecter, for teiring reads/writes from/to other OSDs --
+  Mutex objecter_lock;
+  SafeTimer objecter_timer;
+  OSDMap objecter_osdmap;
+  Objecter *objecter;
+  Finisher objecter_finisher;
+  struct ObjecterDispatcher : public Dispatcher {
+    OSDService *osd;
+    bool ms_dispatch(Message *m);
+    bool ms_handle_reset(Connection *con);
+    void ms_handle_remote_reset(Connection *con) {}
+    void ms_handle_connect(Connection *con);
+    bool ms_get_authorizer(int dest_type,
+			   AuthAuthorizer **authorizer,
+			   bool force_new);
+    ObjecterDispatcher(OSDService *o) : Dispatcher(g_ceph_context), osd(o) {}
+  } objecter_dispatcher;
+  friend class ObjecterDispatcher;
+
+
   // -- Watch --
   Mutex watch_lock;
   SafeTimer watch_timer;
@@ -608,6 +629,7 @@ public:
 #endif
 
   OSDService(OSD *osd);
+  ~OSDService();
 };
 class OSD : public Dispatcher,
 	    public md_config_obs_t {
diff --git a/src/osd/ReplicatedPG.cc b/src/osd/ReplicatedPG.cc
index 36cad99bd10..234004341fd 100644
--- a/src/osd/ReplicatedPG.cc
+++ b/src/osd/ReplicatedPG.cc
@@ -50,6 +50,8 @@
 #include "include/compat.h"
 #include "common/cmdparse.h"
 
+#include "osdc/Objecter.h"
+
 #include "json_spirit/json_spirit_value.h"
 #include "json_spirit/json_spirit_reader.h"
 #include "include/assert.h"  // json_spirit clobbers it
diff --git a/src/vstart.sh b/src/vstart.sh
index 1a6f4f957b9..c112bfc9138 100755
--- a/src/vstart.sh
+++ b/src/vstart.sh
@@ -199,6 +199,7 @@ else
     COSDDEBUG='
         debug ms = 1
         debug osd = 25
+        debug objecter = 20
         debug monc = 20
         debug journal = 20
         debug filestore = 20