ErasureCodePlugin: plugin registry

A ErasureCodePluginRegistry singleton holds all erasure plugin objects
derived from ErasureCodePlugin and dlopen(2) handles for the lifetime
of the OSD and is cleaned up by the destructor.

The registry has a single entry point ( method factory ) and should
be used as follows:

  map<std::string,std::string> parameters;
  parameters["directory"] = "/usr/lib/ceph/erasure-code";
  ErasureCodeInterfaceRef erasure_code;
  ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance();
  instance.factory("jerasure", parameters, &erasure_code));

If the plugin requested ( "jerasure" in the example above ) is not
found in the *plugins* data member, the load method is called and will:

  * dlopen(parameters["erasure-code-directory"] + "jerasure")
  * f = dlsym("__erasure_code_init")
  * f("jerasure")
  * check that it registered "jerasure"

The plugin is expected to do something like

  instance.add(plugin_name, new ErasureCodePluginJerasure());

to register itself.

The factory method is protected with a Mutex to avoid race
conditions when using the same plugin from two threads.

The erasure_codelib_LTLIBRARIES variable is added to the Makefile
and the plugins are expected to add themselves and be installed
in the $(libdir)/erasure-code

http://tracker.ceph.com/issues/5878 refs #5878

Signed-off-by: Loic Dachary <loic@dachary.org>

3 files changed, 166 insertions, 0 deletions

diff --git a/src/osd/ErasureCodePlugin.cc b/src/osd/ErasureCodePlugin.cc
new file mode 100644
index 00000000000..10b65b2604b
--- /dev/null
+++ b/src/osd/ErasureCodePlugin.cc
@@ -0,0 +1,134 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2013 Cloudwatt <libre.licensing@cloudwatt.com>
+ *
+ * Author: Loic Dachary <loic@dachary.org>
+ *
+ *  This library is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU Lesser General Public
+ *  License as published by the Free Software Foundation; either
+ *  version 2.1 of the License, or (at your option) any later version.
+ * 
+ */
+
+#include "common/debug.h"
+
+#include <dlfcn.h>
+
+#include "ErasureCodePlugin.h"
+
+#define dout_subsys ceph_subsys_osd
+#undef dout_prefix
+#define dout_prefix _prefix(_dout)
+
+static ostream& _prefix(std::ostream* _dout)
+{
+  return *_dout << "ErasureCodePlugin: ";
+}
+
+#define PLUGIN_PREFIX "libec_"
+#define PLUGIN_SUFFIX ".so"
+#define PLUGIN_INIT_FUNCTION "__erasure_code_init"
+
+ErasureCodePluginRegistry ErasureCodePluginRegistry::singleton;
+
+ErasureCodePluginRegistry::ErasureCodePluginRegistry() :
+  lock("ErasureCodePluginRegistry::lock")
+{
+}
+
+ErasureCodePluginRegistry::~ErasureCodePluginRegistry()
+{
+  for (std::map<std::string,ErasureCodePlugin*>::iterator i = plugins.begin();
+       i != plugins.end();
+       i++) {
+    void *library = i->second->library;
+    delete i->second;
+    dlclose(library);
+  }
+}
+
+int ErasureCodePluginRegistry::add(const std::string &name,
+                                   ErasureCodePlugin* plugin)
+{
+  if (plugins.find(name) != plugins.end())
+    return -EEXIST;
+  plugins[name] = plugin;
+  return 0;
+}
+
+ErasureCodePlugin *ErasureCodePluginRegistry::get(const std::string &name)
+{
+  if (plugins.find(name) != plugins.end())
+    return plugins[name];
+  else
+    return 0;
+}
+
+int ErasureCodePluginRegistry::factory(const std::string &plugin_name,
+				       const map<std::string,std::string> &parameters,
+				       ErasureCodeInterfaceRef *erasure_code)
+{
+  Mutex::Locker l(lock);
+  int r = 0;
+  ErasureCodePlugin *plugin = get(plugin_name);
+  if (plugin == 0) {
+    r = load(plugin_name, parameters, &plugin);
+    if (r != 0)
+      return r;
+  }
+
+  return plugin->factory(parameters, erasure_code);
+}
+
+int ErasureCodePluginRegistry::load(const std::string &plugin_name,
+				    const map<std::string,std::string> &parameters,
+				    ErasureCodePlugin **plugin)
+{
+  assert(parameters.count("erasure-code-directory") != 0);
+  std::string fname = parameters.find("erasure-code-directory")->second
+    + "/" PLUGIN_PREFIX
+    + plugin_name + PLUGIN_SUFFIX;
+  dout(10) << "load " << plugin_name << " from " << fname << dendl;
+
+  void *library = dlopen(fname.c_str(), RTLD_NOW);
+  if (!library) {
+    derr << "load dlopen(" << fname
+	 << "): " << dlerror() << dendl;
+    return -EIO;
+  }
+
+  int (*erasure_code_init)(const char *) =
+    (int (*)(const char *))dlsym(library, PLUGIN_INIT_FUNCTION);
+  if (erasure_code_init) {
+    std::string name = plugin_name;
+    int r = erasure_code_init(name.c_str());
+    if (r != 0) {
+      derr << "erasure_code_init(" << plugin_name
+           << "): " << strerror(-r) << dendl;
+      return r;
+    }
+  } else {
+    derr << "load dlsym(" << fname
+	 << ", " << PLUGIN_INIT_FUNCTION
+	 << "): " << dlerror() << dendl;
+    dlclose(library);
+    return -ENOENT;
+  }
+
+  *plugin = get(plugin_name);
+  if (*plugin == 0) {
+    derr << "load " << PLUGIN_INIT_FUNCTION << "()"
+         << "did not register " << plugin_name << dendl;
+    dlclose(library);
+    return -EBADF;
+  }
+
+  (*plugin)->library = library;
+
+  return 0;
+}
+
diff --git a/src/osd/ErasureCodePlugin.h b/src/osd/ErasureCodePlugin.h
index 4c210d698d1..f1c1ccb31b3 100644
--- a/src/osd/ErasureCodePlugin.h
+++ b/src/osd/ErasureCodePlugin.h
@@ -17,6 +17,7 @@
 #ifndef CEPH_ERASURE_CODE_PLUGIN_H
 #define CEPH_ERASURE_CODE_PLUGIN_H
 
+#include "common/Mutex.h"
 #include "ErasureCodeInterface.h"
 
 extern "C" {
@@ -37,6 +38,32 @@ namespace ceph {
                         ErasureCodeInterfaceRef *erasure_code) = 0;
   };
 
+  class ErasureCodePluginRegistry {
+  public:
+    Mutex lock;
+    std::map<std::string,ErasureCodePlugin*> plugins;
+
+    static ErasureCodePluginRegistry singleton;
+
+    ErasureCodePluginRegistry();
+    ~ErasureCodePluginRegistry();
+
+    static ErasureCodePluginRegistry &instance() {
+      return singleton;
+    }
+
+    int factory(const std::string &plugin,
+		const map<std::string,std::string> &parameters,
+		ErasureCodeInterfaceRef *erasure_code);
+
+    int add(const std::string &name, ErasureCodePlugin *plugin);
+    ErasureCodePlugin *get(const std::string &name);
+
+    int load(const std::string &plugin_name,
+	     const map<std::string,std::string> &parameters,
+	     ErasureCodePlugin **plugin);
+
+  };
 }
 
 #endif
diff --git a/src/osd/Makefile.am b/src/osd/Makefile.am
index 382085f5f16..a6ab39275aa 100644
--- a/src/osd/Makefile.am
+++ b/src/osd/Makefile.am
@@ -1,4 +1,9 @@
+## erasure code plugins
+erasure_codelibdir = $(libdir)/erasure-code
+erasure_codelib_LTLIBRARIES =  
+
 libosd_la_SOURCES = \
+	osd/ErasureCodePlugin.cc \
 	osd/PG.cc \
 	osd/PGLog.cc \
 	osd/ReplicatedPG.cc \