rgw: log in the same shard for bucket entry point and instancewip-rgw-geo-bucketinstance

We'd like to have bucket entry point and instance info at the same
log shard, so that we can process them in order.

Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>

3 files changed, 33 insertions, 10 deletions

diff --git a/src/rgw/rgw_bucket.cc b/src/rgw/rgw_bucket.cc
index 0426bede76c..bd17b25562d 100644
--- a/src/rgw/rgw_bucket.cc
+++ b/src/rgw/rgw_bucket.cc
@@ -1597,6 +1597,21 @@ public:
     list_keys_info *info = (list_keys_info *)handle;
     delete info;
   }
+
+  /*
+   * hash entry for mdlog placement. Use the same hash key we'd have for the bucket entry
+   * point, so that the log entries end up at the same log shard, so that we process them
+   * in order
+   */
+  virtual void get_hash_key(const string& section, const string& key, string& hash_key) {
+    string k;
+    int pos = key.find(':');
+    if (pos < 0)
+      k = key;
+    else
+      k = key.substr(0, pos);
+    hash_key = "bucket:" + k;
+  }
 };
 
 void rgw_bucket_init(RGWMetadataManager *mm)
diff --git a/src/rgw/rgw_metadata.cc b/src/rgw/rgw_metadata.cc
index d27ba14d009..7ace5b317a6 100644
--- a/src/rgw/rgw_metadata.cc
+++ b/src/rgw/rgw_metadata.cc
@@ -74,10 +74,13 @@ struct RGWMetadataLogData {
 WRITE_CLASS_ENCODER(RGWMetadataLogData);
 
 
-int RGWMetadataLog::add_entry(RGWRados *store, const string& section, const string& key, bufferlist& bl) {
+int RGWMetadataLog::add_entry(RGWRados *store, RGWMetadataHandler *handler, const string& section, const string& key, bufferlist& bl) {
   string oid;
 
-  store->shard_name(prefix, cct->_conf->rgw_md_log_max_shards, section, key, oid);
+  string hash_key;
+  handler->get_hash_key(section, key, hash_key);
+
+  store->shard_name(prefix, cct->_conf->rgw_md_log_max_shards, hash_key, oid);
   utime_t now = ceph_clock_now(cct);
   return store->time_log_add(oid, now, section, key, bl);
 }
@@ -494,14 +497,14 @@ int RGWMetadataManager::pre_modify(RGWMetadataHandler *handler, string& section,
   bufferlist logbl;
   ::encode(log_data, logbl);
 
-  int ret = md_log->add_entry(store, section, key, logbl);
+  int ret = md_log->add_entry(store, handler, section, key, logbl);
   if (ret < 0)
     return ret;
 
   return 0;
 }
 
-int RGWMetadataManager::post_modify(const string& section, const string& key, RGWMetadataLogData& log_data,
+int RGWMetadataManager::post_modify(RGWMetadataHandler *handler, const string& section, const string& key, RGWMetadataLogData& log_data,
                                     RGWObjVersionTracker *objv_tracker, int ret)
 {
   if (ret >= 0)
@@ -512,7 +515,7 @@ int RGWMetadataManager::post_modify(const string& section, const string& key, RG
   bufferlist logbl;
   ::encode(log_data, logbl);
 
-  int r = md_log->add_entry(store, section, key, logbl);
+  int r = md_log->add_entry(store, handler, section, key, logbl);
   if (ret < 0)
     return ret;
 
@@ -541,7 +544,7 @@ int RGWMetadataManager::put_entry(RGWMetadataHandler *handler, const string& key
                            objv_tracker, mtime, pattrs);
   /* cascading ret into post_modify() */
 
-  ret = post_modify(section, key, log_data, objv_tracker, ret);
+  ret = post_modify(handler, section, key, log_data, objv_tracker, ret);
   if (ret < 0)
     return ret;
 
@@ -566,7 +569,7 @@ int RGWMetadataManager::remove_entry(RGWMetadataHandler *handler, string& key, R
   ret = store->delete_obj(NULL, obj);
   /* cascading ret into post_modify() */
 
-  ret = post_modify(section, key, log_data, objv_tracker, ret);
+  ret = post_modify(handler, section, key, log_data, objv_tracker, ret);
   if (ret < 0)
     return ret;
 
@@ -587,7 +590,7 @@ int RGWMetadataManager::set_attrs(RGWMetadataHandler *handler, string& key,
   ret = store->set_attrs(NULL, obj, attrs, rmattrs, objv_tracker);
   /* cascading ret into post_modify() */
 
-  ret = post_modify(section, key, log_data, objv_tracker, ret);
+  ret = post_modify(handler, section, key, log_data, objv_tracker, ret);
   if (ret < 0)
     return ret;
 
diff --git a/src/rgw/rgw_metadata.h b/src/rgw/rgw_metadata.h
index 7c991e852ba..43670007cd5 100644
--- a/src/rgw/rgw_metadata.h
+++ b/src/rgw/rgw_metadata.h
@@ -57,6 +57,11 @@ public:
   virtual int list_keys_init(RGWRados *store, void **phandle) = 0;
   virtual int list_keys_next(void *handle, int max, list<string>& keys, bool *truncated) = 0;
   virtual void list_keys_complete(void *handle) = 0;
+
+  /* key to use for hashing entries for log shard placement */
+  virtual void get_hash_key(const string& section, const string& key, string& hash_key) {
+    hash_key = section + ":" + key;
+  }
 };
 
 #define META_LOG_OBJ_PREFIX "meta.log."
@@ -77,7 +82,7 @@ public:
     prefix = META_LOG_OBJ_PREFIX;
   }
 
-  int add_entry(RGWRados *store, const string& section, const string& key, bufferlist& bl);
+  int add_entry(RGWRados *store, RGWMetadataHandler *handler, const string& section, const string& key, bufferlist& bl);
 
   struct LogListCtx {
     int cur_shard;
@@ -117,7 +122,7 @@ class RGWMetadataManager {
   int pre_modify(RGWMetadataHandler *handler, string& section, const string& key,
                  RGWMetadataLogData& log_data, RGWObjVersionTracker *objv_tracker,
                  RGWMDLogStatus op_type);
-  int post_modify(const string& section, const string& key, RGWMetadataLogData& log_data,
+  int post_modify(RGWMetadataHandler *handler, const string& section, const string& key, RGWMetadataLogData& log_data,
                  RGWObjVersionTracker *objv_tracker, int ret);
 
 public: