rgw: change watch init ordering, don't distribute if can't

Backport: dumpling

Moving back the watch initialization after the zone init,
as the zone info holds the control pool name. Since zone
init might need to create a new system object (that needs
to distribute cache), don't try to distribute cache if
watch is not yet initialized.

Reviewed-by: Sage Weil <sage@inktank.com>
Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>

2 files changed, 20 insertions, 8 deletions

diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index 03cc1ebfdb3..05db9bff782 100644
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -875,14 +875,6 @@ int RGWRados::init_complete()
 {
   int ret;
 
-  if (need_watch_notify()) {
-    ret = init_watch();
-    if (ret < 0) {
-      lderr(cct) << "ERROR: failed to initialize watch" << dendl;
-      return ret;
-    }
-  }
-
   ret = region.init(cct, this);
   if (ret < 0)
     return ret;
@@ -920,6 +912,14 @@ int RGWRados::init_complete()
     }
   }
 
+  if (need_watch_notify()) {
+    ret = init_watch();
+    if (ret < 0) {
+      lderr(cct) << "ERROR: failed to initialize watch" << dendl;
+      return ret;
+    }
+  }
+
   map<string, RGWZone>::iterator ziter;
   for (ziter = region.zones.begin(); ziter != region.zones.end(); ++ziter) {
     const string& name = ziter->first;
@@ -1109,6 +1109,8 @@ int RGWRados::init_watch()
       return r;
   }
 
+  watch_initialized = true;
+
   return 0;
 }
 
@@ -4903,6 +4905,14 @@ int RGWRados::append_async(rgw_obj& obj, size_t size, bufferlist& bl)
 
 int RGWRados::distribute(const string& key, bufferlist& bl)
 {
+  /*
+   * we were called before watch was initialized. This can only happen if we're updating some system
+   * config object (e.g., zone info) during init. Don't try to distribute the cache info for these
+   * objects, they're currently only read on startup anyway.
+   */
+  if (!watch_initialized)
+    return 0;
+
   string notify_oid;
   pick_control_oid(key, notify_oid);
 
diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h
index e6ab244afa9..0da6f4669bf 100644
--- a/src/rgw/rgw_rados.h
+++ b/src/rgw/rgw_rados.h
@@ -801,6 +801,7 @@ class RGWRados
   uint64_t *watch_handles;
   librados::IoCtx root_pool_ctx;      // .rgw
   librados::IoCtx control_pool_ctx;   // .rgw.control
+  bool watch_initialized;
 
   Mutex bucket_id_lock;
   uint64_t max_bucket_id;
@@ -864,6 +865,7 @@ public:
   RGWRados() : lock("rados_timer_lock"), timer(NULL),
                gc(NULL), use_gc_thread(false),
                num_watchers(0), watchers(NULL), watch_handles(NULL),
+               watch_initialized(false),
                bucket_id_lock("rados_bucket_id"), max_bucket_id(0),
                cct(NULL), rados(NULL),
                pools_initialized(false),