Merge pull request #225 from ceph/wip-4543

Fixes #4543

Reviewed-by: Greg Farnum <greg@inktank.com>

6 files changed, 155 insertions, 28 deletions

diff --git a/src/ceph_mon.cc b/src/ceph_mon.cc
index 72354e18876..01072728db2 100644
--- a/src/ceph_mon.cc
+++ b/src/ceph_mon.cc
@@ -52,6 +52,53 @@ void handle_mon_signal(int signum)
     mon->handle_signal(signum);
 }
 
+
+int obtain_monmap(MonitorDBStore &store, bufferlist &bl)
+{
+  dout(10) << __func__ << dendl;
+  /*
+   * the monmap may be in one of three places:
+   *  'monmap:<latest_version_no>' - the monmap we'd really like to have
+   *  'mon_sync:latest_monmap'     - last monmap backed up for the last sync
+   *  'mkfs:monmap'                - a monmap resulting from mkfs
+   */
+
+  if (store.exists("monmap", "last_committed")) {
+    version_t latest_ver = store.get("monmap", "last_committed");
+    if (store.exists("monmap", latest_ver)) {
+      int err = store.get("monmap", latest_ver, bl);
+      assert(err == 0);
+      assert(bl.length() > 0);
+      dout(10) << __func__ << " read last committed monmap ver "
+               << latest_ver << dendl;
+      return 0;
+    }
+  }
+
+  if (store.exists("mon_sync", "in_sync")) {
+    dout(10) << __func__ << " detected aborted sync" << dendl;
+    if (store.exists("mon_sync", "latest_monmap")) {
+      int err = store.get("mon_sync", "latest_monmap", bl);
+      assert(err == 0);
+      assert(bl.length() > 0);
+      dout(10) << __func__ << " read backup monmap" << dendl;
+      return 0;
+    }
+  }
+
+  if (store.exists("mkfs", "monmap")) {
+    dout(10) << __func__ << " found mkfs monmap" << dendl;
+    int err = store.get("mkfs", "monmap", bl);
+    assert(err == 0);
+    assert(bl.length() > 0);
+    return 0;
+  }
+
+  derr << __func__ << " unable to find a monmap" << dendl;
+  return -ENOENT;
+}
+
+
 void usage()
 {
   cerr << "usage: ceph-mon -i monid [--mon-data=pathtodata] [flags]" << std::endl;
@@ -301,30 +348,22 @@ int main(int argc, const char **argv)
     exit(0);
   }
 
-
   // monmap?
   MonMap monmap;
   {
+    // note that even if we don't find a viable monmap, we should go ahead
+    // and try to build it up in the next if-else block.
     bufferlist mapbl;
-    bufferlist latest;
-    store.get("monmap", "latest", latest);
-    if (latest.length() > 0) {
-      bufferlist::iterator p = latest.begin();
-      version_t v;
-      ::decode(v, p);
-      ::decode(mapbl, p);
-    } else {
-      store.get("mkfs", "monmap", mapbl);
-      if (mapbl.length() == 0) {
-	cerr << "mon fs missing 'monmap/latest' and 'mkfs/monmap'" << std::endl;
-	exit(1);
+    int err = obtain_monmap(store, mapbl);
+    if (err >= 0) {
+      try {
+        monmap.decode(mapbl);
+      } catch (const buffer::error& e) {
+        cerr << "can't decode monmap: " << e.what() << std::endl;
       }
-    }
-    try {
-      monmap.decode(mapbl);
-    }
-    catch (const buffer::error& e) {
-      cerr << "can't decode monmap: " << e.what() << std::endl;
+    } else {
+      std::cerr << "unable to obtain a monmap: "
+                << cpp_strerror(err) << std::endl;
     }
   }
 
diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc
index cc8527ed7f7..1e39e9ca304 100644
--- a/src/mon/Monitor.cc
+++ b/src/mon/Monitor.cc
@@ -408,7 +408,7 @@ int Monitor::preinit()
     // We have a potentially inconsistent store state in hands. Get rid of it
     // and start fresh.
     bool clear_store = false;
-    if (store->get("mon_sync", "in_sync") > 0) {
+    if (is_sync_on_going()) {
       dout(1) << __func__ << " clean up potentially inconsistent store state"
 	      << dendl;
       clear_store = true;
@@ -421,8 +421,12 @@ int Monitor::preinit()
 
     if (clear_store) {
       set<string> sync_prefixes = get_sync_targets_names();
-      sync_prefixes.insert("mon_sync");
       store->clear(sync_prefixes);
+
+      MonitorDBStore::Transaction t;
+      t.erase("mon_sync", "in_sync");
+      t.erase("mon_sync", "force_sync");
+      store->apply_transaction(t);
     }
   }
 
@@ -1276,6 +1280,47 @@ void Monitor::sync_requester_abort()
 }
 
 /**
+ *
+ */
+void Monitor::sync_store_init()
+{
+  MonitorDBStore::Transaction t;
+  t.put("mon_sync", "in_sync", 1);
+
+  bufferlist latest_monmap;
+  int err = monmon()->get_monmap(latest_monmap);
+  if (err < 0) {
+    if (err != -ENOENT) {
+      derr << __func__
+           << " something wrong happened while reading the store: "
+           << cpp_strerror(err) << dendl;
+      assert(0 == "error reading the store");
+      return; // this is moot
+    } else {
+      dout(10) << __func__ << " backup current monmap" << dendl;
+      monmap->encode(latest_monmap, get_quorum_features());
+    }
+  }
+
+  t.put("mon_sync", "latest_monmap", latest_monmap);
+
+  store->apply_transaction(t);
+}
+
+void Monitor::sync_store_cleanup()
+{
+  MonitorDBStore::Transaction t;
+  t.erase("mon_sync", "in_sync");
+  t.erase("mon_sync", "latest_monmap");
+  store->apply_transaction(t);
+}
+
+bool Monitor::is_sync_on_going()
+{
+  return store->exists("mon_sync", "in_sync");
+}
+
+/**
  * Start Sync process
  *
  * Create SyncEntity instances for the leader and the provider;
@@ -1321,9 +1366,8 @@ void Monitor::sync_start(entity_inst_t &other)
   targets.insert("mon_sync");
   store->clear(targets);
 
-  MonitorDBStore::Transaction t;
-  t.put("mon_sync", "in_sync", 1);
-  store->apply_transaction(t);
+
+  sync_store_init();
 
   // assume 'other' as the leader. We will update the leader once we receive
   // a reply to the sync start.
@@ -1618,9 +1662,7 @@ void Monitor::handle_sync_finish_reply(MMonSync *m)
 
   paxos->reapply_all_versions();
 
-  MonitorDBStore::Transaction t;
-  t.erase("mon_sync", "in_sync");
-  store->apply_transaction(t);
+  sync_store_cleanup();
 
   init_paxos();
 
diff --git a/src/mon/Monitor.h b/src/mon/Monitor.h
index f5512efd775..c08942d4a50 100644
--- a/src/mon/Monitor.h
+++ b/src/mon/Monitor.h
@@ -364,6 +364,10 @@ private:
     }
   };
 
+  void sync_store_init();
+  void sync_store_cleanup();
+  bool is_sync_on_going();
+
   /**
    * Send a heartbeat message to another entity.
    *
diff --git a/src/mon/MonmapMonitor.cc b/src/mon/MonmapMonitor.cc
index f1c6dfa3325..badac7e0922 100644
--- a/src/mon/MonmapMonitor.cc
+++ b/src/mon/MonmapMonitor.cc
@@ -502,3 +502,33 @@ void MonmapMonitor::get_health(list<pair<health_status_t, string> >& summary,
     }
   }
 }
+
+int MonmapMonitor::get_monmap(bufferlist &bl)
+{
+  version_t latest_ver = get_last_committed();
+  dout(10) << __func__ << " ver " << latest_ver << dendl;
+
+  if (!exists_version(latest_ver))
+    return -ENOENT;
+
+  int err = get_version(latest_ver, bl);
+  if (err < 0) {
+    dout(1) << __func__ << " error obtaining monmap: "
+            << cpp_strerror(err) << dendl;
+    return err;
+  }
+  return 0;
+}
+
+int MonmapMonitor::get_monmap(MonMap &m)
+{
+  dout(10) << __func__ << dendl;
+  bufferlist monmap_bl;
+
+  int err = get_monmap(monmap_bl);
+  if (err < 0) {
+    return err;
+  }
+  m.decode(monmap_bl);
+  return 0;
+}
diff --git a/src/mon/MonmapMonitor.h b/src/mon/MonmapMonitor.h
index 2861fbbd34f..0690fb7409f 100644
--- a/src/mon/MonmapMonitor.h
+++ b/src/mon/MonmapMonitor.h
@@ -70,6 +70,9 @@ class MonmapMonitor : public PaxosService {
   void get_health(list<pair<health_status_t,string> >& summary,
 		  list<pair<health_status_t,string> > *detail) const;
 
+  int get_monmap(bufferlist &bl);
+  int get_monmap(MonMap &m);
+
   /*
    * Since monitors are pretty
    * important, this implementation will just write 0.0.
diff --git a/src/mon/PaxosService.h b/src/mon/PaxosService.h
index f16bc77cdeb..268253599bf 100644
--- a/src/mon/PaxosService.h
+++ b/src/mon/PaxosService.h
@@ -908,6 +908,15 @@ public:
   int get_mkfs(bufferlist& bl) {
     return mon->store->get(mkfs_name, get_service_name(), bl);
   }
+
+  bool exists_key(const string &key) {
+    return mon->store->exists(get_service_name(), key);
+  }
+
+  bool exists_version(const version_t v) {
+    return exists_key(stringify(v));
+  }
+
   /**
    * Checks if a given key composed by @p prefix and @p name exists.
    *
@@ -917,7 +926,7 @@ public:
    */
   bool exists_key(const string& prefix, const string& name) {
     string key = mon->store->combine_strings(prefix, name);
-    return mon->store->exists(get_service_name(), key);
+    return exists_key(key);
   }
 
   /**