rgw: decouple bucket data pool from bucket index pool

Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>

7 files changed, 154 insertions, 111 deletions

diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index 872c5dd2eeb..6418e4c8a29 100644
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -433,7 +433,8 @@ int bucket_stats(rgw_bucket& bucket, Formatter *formatter)
   }
   formatter->open_object_section("stats");
   formatter->dump_string("bucket", bucket.name);
-  formatter->dump_string("pool", bucket.pool);
+  formatter->dump_string("pool", bucket.data_pool);
+  formatter->dump_string("index_pool", bucket.index_pool);
   
   formatter->dump_string("id", bucket.bucket_id);
   formatter->dump_string("marker", bucket.marker);
diff --git a/src/rgw/rgw_bucket.cc b/src/rgw/rgw_bucket.cc
index 03cefaff132..d13d359a11c 100644
--- a/src/rgw/rgw_bucket.cc
+++ b/src/rgw/rgw_bucket.cc
@@ -277,7 +277,8 @@ void check_bad_user_bucket_mapping(RGWRados *store, const string& user_id, bool
     rgw_bucket& actual_bucket = bucket_info.bucket;
 
     if (actual_bucket.name.compare(bucket.name) != 0 ||
-        actual_bucket.pool.compare(bucket.pool) != 0 ||
+        actual_bucket.data_pool.compare(bucket.data_pool) != 0 ||
+        actual_bucket.index_pool.compare(bucket.index_pool) != 0 ||
         actual_bucket.marker.compare(bucket.marker) != 0 ||
         actual_bucket.bucket_id.compare(bucket.bucket_id) != 0) {
       cout << "bucket info mismatch: expected " << actual_bucket << " got " << bucket << std::endl;
@@ -907,8 +908,8 @@ static int bucket_stats(RGWRados *store, std::string&  bucket_name, Formatter *f
 
   formatter->open_object_section("stats");
   formatter->dump_string("bucket", bucket.name);
-  formatter->dump_string("pool", bucket.pool);
-
+  formatter->dump_string("pool", bucket.data_pool);
+  formatter->dump_string("index_pool", bucket.index_pool);
   formatter->dump_string("id", bucket.bucket_id);
   formatter->dump_string("marker", bucket.marker);
   formatter->dump_string("owner", bucket_info.owner);
diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h
index aead4598061..d74e2e66bc8 100644
--- a/src/rgw/rgw_common.h
+++ b/src/rgw/rgw_common.h
@@ -479,38 +479,41 @@ WRITE_CLASS_ENCODER(RGWUserInfo)
 
 struct rgw_bucket {
   std::string name;
-  std::string pool;
+  std::string data_pool;
+  std::string index_pool;
   std::string marker;
   std::string bucket_id;
 
   rgw_bucket() { }
   rgw_bucket(const char *n) : name(n) {
     assert(*n == '.'); // only rgw private buckets should be initialized without pool
-    pool = n;
+    data_pool = index_pool = n;
     marker = "";
   }
-  rgw_bucket(const char *n, const char *p, const char *m, const char *id) :
-    name(n), pool(p), marker(m), bucket_id(id) {}
+  rgw_bucket(const char *n, const char *dp, const char *ip, const char *m, const char *id) :
+    name(n), data_pool(dp), index_pool(ip), marker(m), bucket_id(id) {}
 
   void clear() {
     name = "";
-    pool = "";
+    data_pool = "";
+    index_pool = "";
     marker = "";
     bucket_id = "";
   }
 
   void encode(bufferlist& bl) const {
-     ENCODE_START(4, 3, bl);
+     ENCODE_START(5, 3, bl);
     ::encode(name, bl);
-    ::encode(pool, bl);
+    ::encode(data_pool, bl);
     ::encode(marker, bl);
     ::encode(bucket_id, bl);
+    ::encode(index_pool, bl);
     ENCODE_FINISH(bl);
   }
   void decode(bufferlist::iterator& bl) {
-    DECODE_START_LEGACY_COMPAT_LEN(4, 3, 3, bl);
+    DECODE_START_LEGACY_COMPAT_LEN(5, 3, 3, bl);
     ::decode(name, bl);
-    ::decode(pool, bl);
+    ::decode(data_pool, bl);
     if (struct_v >= 2) {
       ::decode(marker, bl);
       if (struct_v <= 3) {
@@ -523,6 +526,11 @@ struct rgw_bucket {
         ::decode(bucket_id, bl);
       }
     }
+    if (struct_v >= 5) {
+      ::decode(index_pool, bl);
+    } else {
+      index_pool = data_pool;
+    }
     DECODE_FINISH(bl);
   }
   void dump(Formatter *f) const;
@@ -532,8 +540,12 @@ WRITE_CLASS_ENCODER(rgw_bucket)
 
 inline ostream& operator<<(ostream& out, const rgw_bucket &b) {
   out << b.name;
-  if (b.name.compare(b.pool))
-    out << "(@" << b.pool << "[" << b.marker << "])";
+  if (b.name.compare(b.data_pool)) {
+    out << "(@";
+    if (!b.index_pool.empty() && b.data_pool.compare(b.index_pool))
+      out << "{i=" << b.index_pool << "}";
+    out << b.data_pool << "[" << b.marker << "])";
+  }
   return out;
 }
 
diff --git a/src/rgw/rgw_dencoder.cc b/src/rgw/rgw_dencoder.cc
index 1f4e3522788..59675c315e0 100644
--- a/src/rgw/rgw_dencoder.cc
+++ b/src/rgw/rgw_dencoder.cc
@@ -13,7 +13,7 @@ void RGWObjManifestPart::generate_test_instances(std::list<RGWObjManifestPart*>&
   o.push_back(new RGWObjManifestPart);
 
   RGWObjManifestPart *p = new RGWObjManifestPart;
-  rgw_bucket b("bucket", ".pool", "marker_", "12");
+  rgw_bucket b("bucket", ".pool", ".index_pool", "marker_", "12");
   p->loc = rgw_obj(b, "object");
   p->loc_ofs = 512 * 1024;
   p->size = 128 * 1024;
@@ -25,7 +25,7 @@ void RGWObjManifest::generate_test_instances(std::list<RGWObjManifest*>& o)
   RGWObjManifest *m = new RGWObjManifest;
   for (int i = 0; i<10; i++) {
     RGWObjManifestPart p;
-    rgw_bucket b("bucket", ".pool", "marker_", "12");
+    rgw_bucket b("bucket", ".pool", ".index_pool", "marker_", "12");
     p.loc = rgw_obj(b, "object");
     p.loc_ofs = 0;
     p.size = 512 * 1024;
@@ -63,7 +63,7 @@ void rgw_log_entry::generate_test_instances(list<rgw_log_entry*>& o)
 void rgw_intent_log_entry::generate_test_instances(list<rgw_intent_log_entry*>& o)
 {
   rgw_intent_log_entry *e = new rgw_intent_log_entry;
-  rgw_bucket b("bucket", "pool", "marker", "10");
+  rgw_bucket b("bucket", "pool", ".index_pool", "marker", "10");
   e->obj = rgw_obj(b, "object");
   e->intent = DEL_OBJ;
   o.push_back(e);
@@ -254,7 +254,7 @@ void RGWUserInfo::generate_test_instances(list<RGWUserInfo*>& o)
 
 void rgw_bucket::generate_test_instances(list<rgw_bucket*>& o)
 {
-  rgw_bucket *b = new rgw_bucket("name", "pool", "marker", "123");
+  rgw_bucket *b = new rgw_bucket("name", "pool", ".index_pool", "marker", "123");
   o.push_back(b);
   o.push_back(new rgw_bucket);
 }
@@ -262,7 +262,7 @@ void rgw_bucket::generate_test_instances(list<rgw_bucket*>& o)
 void RGWBucketInfo::generate_test_instances(list<RGWBucketInfo*>& o)
 {
   RGWBucketInfo *i = new RGWBucketInfo;
-  i->bucket = rgw_bucket("bucket", "pool", "marker", "10");
+  i->bucket = rgw_bucket("bucket", "pool", ".index_pool", "marker", "10");
   i->owner = "owner";
   i->flags = BUCKET_SUSPENDED;
   o.push_back(i);
@@ -272,7 +272,7 @@ void RGWBucketInfo::generate_test_instances(list<RGWBucketInfo*>& o)
 void RGWBucketEnt::generate_test_instances(list<RGWBucketEnt*>& o)
 {
   RGWBucketEnt *e = new RGWBucketEnt;
-  e->bucket = rgw_bucket("bucket", "pool", "marker", "10");
+  e->bucket = rgw_bucket("bucket", "pool", ".index_pool", "marker", "10");
   e->size = 1024;
   e->size_rounded = 4096;
   e->count = 1;
@@ -292,7 +292,7 @@ void RGWUploadPartInfo::generate_test_instances(list<RGWUploadPartInfo*>& o)
 
 void rgw_obj::generate_test_instances(list<rgw_obj*>& o)
 {
-  rgw_bucket b = rgw_bucket("bucket", "pool", "marker", "10");
+  rgw_bucket b = rgw_bucket("bucket", "pool", ".index_pool", "marker", "10");
   rgw_obj *obj = new rgw_obj(b, "object");
   o.push_back(obj);
   o.push_back(new rgw_obj);
diff --git a/src/rgw/rgw_json_enc.cc b/src/rgw/rgw_json_enc.cc
index 895ef0a7eea..60948619076 100644
--- a/src/rgw/rgw_json_enc.cc
+++ b/src/rgw/rgw_json_enc.cc
@@ -385,7 +385,8 @@ void RGWUserInfo::decode_json(JSONObj *obj)
 void rgw_bucket::dump(Formatter *f) const
 {
   encode_json("name", name, f);
-  encode_json("pool", pool, f);
+  encode_json("pool", data_pool, f);
+  encode_json("index_pool", index_pool, f);
   encode_json("marker", marker, f);
   encode_json("bucket_id", bucket_id, f);
 }
@@ -437,16 +438,16 @@ void rgw_obj::dump(Formatter *f) const
 
 void RGWZoneParams::dump(Formatter *f) const
 {
-  encode_json("domain_root", domain_root.pool, f);
-  encode_json("control_pool", control_pool.pool, f);
-  encode_json("gc_pool", gc_pool.pool, f);
-  encode_json("log_pool", log_pool.pool, f);
-  encode_json("intent_log_pool", intent_log_pool.pool, f);
-  encode_json("usage_log_pool", usage_log_pool.pool, f);
-  encode_json("user_keys_pool", user_keys_pool.pool, f);
-  encode_json("user_email_pool", user_email_pool.pool, f);
-  encode_json("user_swift_pool", user_swift_pool.pool, f);
-  encode_json("user_uid_pool ", user_uid_pool.pool, f);
+  encode_json("domain_root", domain_root.data_pool, f);
+  encode_json("control_pool", control_pool.data_pool, f);
+  encode_json("gc_pool", gc_pool.data_pool, f);
+  encode_json("log_pool", log_pool.data_pool, f);
+  encode_json("intent_log_pool", intent_log_pool.data_pool, f);
+  encode_json("usage_log_pool", usage_log_pool.data_pool, f);
+  encode_json("user_keys_pool", user_keys_pool.data_pool, f);
+  encode_json("user_email_pool", user_email_pool.data_pool, f);
+  encode_json("user_swift_pool", user_swift_pool.data_pool, f);
+  encode_json("user_uid_pool ", user_uid_pool.data_pool, f);
 }
 
 static void decode_json(const char *field, rgw_bucket& bucket, JSONObj *obj)
diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index e710aef2e62..0f5fc53dae5 100644
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -737,9 +737,9 @@ void RGWRados::pick_control_oid(const string& key, string& notify_oid)
   notify_oid.append(buf);
 }
 
-int RGWRados::open_bucket_ctx(rgw_bucket& bucket, librados::IoCtx&  io_ctx)
+int RGWRados::open_bucket_pool_ctx(const string& bucket_name, const string& pool, librados::IoCtx&  io_ctx)
 {
-  int r = rados->ioctx_create(bucket.pool.c_str(), io_ctx);
+  int r = rados->ioctx_create(pool.c_str(), io_ctx);
   if (r != -ENOENT)
     return r;
 
@@ -751,15 +751,33 @@ int RGWRados::open_bucket_ctx(rgw_bucket& bucket, librados::IoCtx&  io_ctx)
      the bucket object .. which will trigger update of osdmap
      if that is the case */
   time_t mtime;
-  r = root_pool_ctx.stat(bucket.name, NULL, &mtime);
+  r = root_pool_ctx.stat(bucket_name, NULL, &mtime);
   if (r < 0)
     return -ENOENT;
 
-  r = rados->ioctx_create(bucket.pool.c_str(), io_ctx);
+  r = rados->ioctx_create(pool.c_str(), io_ctx);
 
   return r;
 }
 
+int RGWRados::open_bucket_data_ctx(rgw_bucket& bucket, librados::IoCtx& data_ctx)
+{
+  int r = open_bucket_pool_ctx(bucket.name, bucket.data_pool, data_ctx);
+  if (r < 0)
+    return r;
+
+  return 0;
+}
+
+int RGWRados::open_bucket_index_ctx(rgw_bucket& bucket, librados::IoCtx& index_ctx)
+{
+  int r = open_bucket_pool_ctx(bucket.name, bucket.index_pool, index_ctx);
+  if (r < 0)
+    return r;
+
+  return 0;
+}
+
 /**
  * set up a bucket listing.
  * handle is filled in.
@@ -1230,13 +1248,21 @@ int RGWRados::create_pool(rgw_bucket& bucket)
 {
   int ret = 0;
 
-  ret = rados->pool_create(bucket.pool.c_str(), 0);
+  string pool = bucket.index_pool;
+
+  ret = rados->pool_create(pool.c_str(), 0);
   if (ret == -EEXIST)
     ret = 0;
   if (ret < 0)
     return ret;
 
-  bucket.pool = bucket.name;
+  if (bucket.data_pool != pool) {
+    ret = rados->pool_create(bucket.data_pool.c_str(), 0);
+    if (ret == -EEXIST)
+      ret = 0;
+    if (ret < 0)
+      return ret;
+  }
 
   return 0;
 }
@@ -1252,9 +1278,9 @@ int RGWRados::create_bucket(string& owner, rgw_bucket& bucket,
   ret = select_bucket_placement(bucket.name, bucket);
   if (ret < 0)
     return ret;
-  librados::IoCtx io_ctx; // context for new bucket
+  librados::IoCtx index_ctx; // context for new bucket
 
-  int r = open_bucket_ctx(bucket, io_ctx);
+  int r = open_bucket_index_ctx(bucket, index_ctx);
   if (r < 0)
     return r;
 
@@ -1281,7 +1307,7 @@ int RGWRados::create_bucket(string& owner, rgw_bucket& bucket,
 
   librados::ObjectWriteOperation op;
   op.create(true);
-  r = cls_rgw_init_index(io_ctx, op, dir_oid);
+  r = cls_rgw_init_index(index_ctx, op, dir_oid);
   if (r < 0 && r != -EEXIST)
     return r;
 
@@ -1290,7 +1316,7 @@ int RGWRados::create_bucket(string& owner, rgw_bucket& bucket,
   info.owner = owner;
   ret = store_bucket_info(info, &attrs, exclusive);
   if (ret == -EEXIST) {
-    io_ctx.remove(dir_oid);
+    index_ctx.remove(dir_oid);
   }
 
   return ret;
@@ -1380,7 +1406,9 @@ read_omap:
     miter = m.begin();
     pool_name = miter->first;
   }
-  bucket.pool = pool_name;
+  bucket.data_pool = pool_name;
+#warning FIXME
+  bucket.index_pool = pool_name;
   bucket.name = bucket_name;
 
   return 0;
@@ -1514,7 +1542,7 @@ int RGWRados::put_obj_meta_impl(void *ctx, rgw_obj& obj,  uint64_t size,
   librados::IoCtx io_ctx;
   RGWRadosCtx *rctx = static_cast<RGWRadosCtx *>(ctx);
 
-  int r = open_bucket_ctx(bucket, io_ctx);
+  int r = open_bucket_data_ctx(bucket, io_ctx);
   if (r < 0)
     return r;
 
@@ -1673,7 +1701,7 @@ int RGWRados::aio_put_obj_data(void *ctx, rgw_obj& obj, bufferlist& bl,
   get_obj_bucket_and_oid_key(obj, bucket, oid, key);
   librados::IoCtx io_ctx;
 
-  int r = open_bucket_ctx(bucket, io_ctx);
+  int r = open_bucket_data_ctx(bucket, io_ctx);
   if (r < 0)
     return r;
 
@@ -1809,7 +1837,7 @@ int RGWRados::copy_obj(void *ctx,
   librados::IoCtx io_ctx;
   PutObjMetaExtraParams ep;
 
-  ret = open_bucket_ctx(bucket, io_ctx);
+  ret = open_bucket_data_ctx(bucket, io_ctx);
   if (ret < 0)
     return ret;
 
@@ -1986,8 +2014,9 @@ done_err:
  */
 int RGWRados::delete_bucket(rgw_bucket& bucket)
 {
-  librados::IoCtx list_ctx;
-  int r = open_bucket_ctx(bucket, list_ctx);
+  librados::IoCtx index_ctx;
+  string oid;
+  int r = open_bucket_index(bucket, index_ctx, oid);
   if (r < 0)
     return r;
 
@@ -2020,10 +2049,8 @@ int RGWRados::delete_bucket(rgw_bucket& bucket)
 
   ObjectWriteOperation op;
   op.remove();
-  string oid = dir_oid_prefix;
-  oid.append(bucket.marker);
   librados::AioCompletion *completion = rados->aio_create_completion(NULL, NULL, NULL);
-  r = list_ctx.aio_operate(oid, completion, &op);
+  r = index_ctx.aio_operate(oid, completion, &op);
   completion->release();
   if (r < 0)
     return r;
@@ -2117,7 +2144,7 @@ int RGWRados::complete_atomic_overwrite(RGWRadosCtx *rctx, RGWObjState *state, r
     string oid, key;
     rgw_bucket bucket;
     get_obj_bucket_and_oid_key(mobj, bucket, oid, key);
-    chain.push_obj(bucket.pool, oid, key);
+    chain.push_obj(bucket.data_pool, oid, key);
   }
 
   string tag = state->obj_tag.c_str();
@@ -2126,12 +2153,12 @@ int RGWRados::complete_atomic_overwrite(RGWRadosCtx *rctx, RGWObjState *state, r
   return ret;
 }
 
-int RGWRados::open_bucket(rgw_bucket& bucket, librados::IoCtx& io_ctx, string& bucket_oid)
+int RGWRados::open_bucket_index(rgw_bucket& bucket, librados::IoCtx& index_ctx, string& bucket_oid)
 {
   if (bucket_is_system(bucket))
     return -EINVAL;
 
-  int r = open_bucket_ctx(bucket, io_ctx);
+  int r = open_bucket_index_ctx(bucket, index_ctx);
   if (r < 0)
     return r;
 
@@ -2164,17 +2191,17 @@ int RGWRados::bucket_check_index(rgw_bucket& bucket,
 				 map<RGWObjCategory, RGWBucketStats> *existing_stats,
 				 map<RGWObjCategory, RGWBucketStats> *calculated_stats)
 {
-  librados::IoCtx io_ctx;
+  librados::IoCtx index_ctx;
   string oid;
 
-  int ret = open_bucket(bucket, io_ctx, oid);
+  int ret = open_bucket_index(bucket, index_ctx, oid);
   if (ret < 0)
     return ret;
 
   rgw_bucket_dir_header existing_header;
   rgw_bucket_dir_header calculated_header;
 
-  ret = cls_rgw_bucket_check_index_op(io_ctx, oid, &existing_header, &calculated_header);
+  ret = cls_rgw_bucket_check_index_op(index_ctx, oid, &existing_header, &calculated_header);
   if (ret < 0)
     return ret;
 
@@ -2186,14 +2213,14 @@ int RGWRados::bucket_check_index(rgw_bucket& bucket,
 
 int RGWRados::bucket_rebuild_index(rgw_bucket& bucket)
 {
-  librados::IoCtx io_ctx;
+  librados::IoCtx index_ctx;
   string oid;
 
-  int ret = open_bucket(bucket, io_ctx, oid);
+  int ret = open_bucket_index(bucket, index_ctx, oid);
   if (ret < 0)
     return ret;
 
-  return cls_rgw_bucket_rebuild_index_op(io_ctx, oid);
+  return cls_rgw_bucket_rebuild_index_op(index_ctx, oid);
 }
 
 
@@ -2243,7 +2270,7 @@ int RGWRados::delete_obj_impl(void *ctx, rgw_obj& obj)
   get_obj_bucket_and_oid_key(obj, bucket, oid, key);
   librados::IoCtx io_ctx;
   RGWRadosCtx *rctx = static_cast<RGWRadosCtx *>(ctx);
-  int r = open_bucket_ctx(bucket, io_ctx);
+  int r = open_bucket_data_ctx(bucket, io_ctx);
   if (r < 0)
     return r;
 
@@ -2434,7 +2461,7 @@ int RGWRados::get_attr(void *ctx, rgw_obj& obj, const char *name, bufferlist& de
     actual_bucket = zone.domain_root;
   }
 
-  int r = open_bucket_ctx(actual_bucket, io_ctx);
+  int r = open_bucket_data_ctx(actual_bucket, io_ctx);
   if (r < 0)
     return r;
 
@@ -2571,7 +2598,7 @@ int RGWRados::set_attr(void *ctx, rgw_obj& obj, const char *name, bufferlist& bl
     actual_bucket = zone.domain_root;
   }
 
-  int r = open_bucket_ctx(actual_bucket, io_ctx);
+  int r = open_bucket_data_ctx(actual_bucket, io_ctx);
   if (r < 0)
     return r;
 
@@ -2621,7 +2648,7 @@ int RGWRados::set_attrs(void *ctx, rgw_obj& obj,
     actual_bucket = zone.domain_root;
   }
 
-  int r = open_bucket_ctx(actual_bucket, io_ctx);
+  int r = open_bucket_data_ctx(actual_bucket, io_ctx);
   if (r < 0)
     return r;
 
@@ -2730,7 +2757,7 @@ int RGWRados::prepare_get_obj(void *ctx, rgw_obj& obj,
 
   *handle = state;
 
-  r = open_bucket_ctx(bucket, state->io_ctx);
+  r = open_bucket_data_ctx(bucket, state->io_ctx);
   if (r < 0)
     goto done_err;
 
@@ -2924,7 +2951,7 @@ int RGWRados::clone_objs_impl(void *ctx, rgw_obj& dst_obj,
   bool update_index = (category == RGW_OBJ_CATEGORY_MAIN ||
                        category == RGW_OBJ_CATEGORY_MULTIMETA);
 
-  int r = open_bucket_ctx(bucket, io_ctx);
+  int r = open_bucket_data_ctx(bucket, io_ctx);
   if (r < 0)
     return r;
   io_ctx.locator_set_key(dst_key);
@@ -3601,7 +3628,7 @@ int RGWRados::read(void *ctx, rgw_obj& obj, off_t ofs, size_t size, bufferlist&
   librados::IoCtx io_ctx;
   RGWRadosCtx *rctx = static_cast<RGWRadosCtx *>(ctx);
   RGWObjState *astate = NULL;
-  int r = open_bucket_ctx(bucket, io_ctx);
+  int r = open_bucket_data_ctx(bucket, io_ctx);
   if (r < 0)
     return r;
 
@@ -3633,7 +3660,7 @@ int RGWRados::obj_stat(void *ctx, rgw_obj& obj, uint64_t *psize, time_t *pmtime,
   std::string oid, key;
   get_obj_bucket_and_oid_key(obj, bucket, oid, key);
   librados::IoCtx io_ctx;
-  int r = open_bucket_ctx(bucket, io_ctx);
+  int r = open_bucket_data_ctx(bucket, io_ctx);
   if (r < 0)
     return r;
 
@@ -3698,7 +3725,8 @@ int RGWRados::get_bucket_info(void *ctx, string& bucket_name, RGWBucketInfo& inf
       return ret;
 
     info.bucket.name = bucket_name;
-    info.bucket.pool = bucket_name; // for now
+    info.bucket.data_pool = bucket_name; // for now
+    info.bucket.index_pool = bucket_name; // for now
     return 0;
   }
 
@@ -3733,7 +3761,7 @@ int RGWRados::omap_get_all(rgw_obj& obj, bufferlist& header, std::map<string, bu
   rgw_bucket bucket;
   std::string oid, key;
   get_obj_bucket_and_oid_key(obj, bucket, oid, key);
-  int r = open_bucket_ctx(bucket, io_ctx);
+  int r = open_bucket_data_ctx(bucket, io_ctx);
   if (r < 0)
     return r;
 
@@ -3757,7 +3785,7 @@ int RGWRados::omap_set(rgw_obj& obj, std::string& key, bufferlist& bl)
   ldout(cct, 15) << "omap_set bucket=" << bucket << " oid=" << oid << " key=" << key << dendl;
 
   librados::IoCtx io_ctx;
-  int r = open_bucket_ctx(bucket, io_ctx);
+  int r = open_bucket_data_ctx(bucket, io_ctx);
   if (r < 0)
     return r;
 
@@ -3778,7 +3806,7 @@ int RGWRados::omap_set(rgw_obj& obj, std::map<std::string, bufferlist>& m)
   get_obj_bucket_and_oid_key(obj, bucket, oid, key);
 
   librados::IoCtx io_ctx;
-  int r = open_bucket_ctx(bucket, io_ctx);
+  int r = open_bucket_data_ctx(bucket, io_ctx);
   if (r < 0)
     return r;
 
@@ -3796,7 +3824,7 @@ int RGWRados::omap_del(rgw_obj& obj, std::string& key)
   get_obj_bucket_and_oid_key(obj, bucket, oid, okey);
 
   librados::IoCtx io_ctx;
-  int r = open_bucket_ctx(bucket, io_ctx);
+  int r = open_bucket_data_ctx(bucket, io_ctx);
   if (r < 0)
     return r;
 
@@ -3843,7 +3871,7 @@ int RGWRados::append_async(rgw_obj& obj, size_t size, bufferlist& bl)
   std::string oid, key;
   get_obj_bucket_and_oid_key(obj, bucket, oid, key);
   librados::IoCtx io_ctx;
-  int r = open_bucket_ctx(bucket, io_ctx);
+  int r = open_bucket_data_ctx(bucket, io_ctx);
   if (r < 0)
     return r;
   librados::AioCompletion *completion = rados->aio_create_completion(NULL, NULL, NULL);
@@ -3870,7 +3898,7 @@ int RGWRados::pool_iterate_begin(rgw_bucket& bucket, RGWPoolIterCtx& ctx)
   librados::IoCtx& io_ctx = ctx.io_ctx;
   librados::ObjectIterator& iter = ctx.iter;
 
-  int r = open_bucket_ctx(bucket, io_ctx);
+  int r = open_bucket_data_ctx(bucket, io_ctx);
   if (r < 0)
     return r;
 
@@ -3953,14 +3981,14 @@ int RGWRados::list_bi_log_entries(rgw_bucket& bucket, string& marker, uint32_t m
 {
   result.clear();
 
-  librados::IoCtx io_ctx;
+  librados::IoCtx index_ctx;
   string oid;
-  int r = open_bucket(bucket, io_ctx, oid);
+  int r = open_bucket_index(bucket, index_ctx, oid);
   if (r < 0)
     return r;
 
   std::list<rgw_bi_log_entry> entries;
-  int ret = cls_rgw_bi_log_list(io_ctx, oid, marker, max - result.size(), entries, truncated);
+  int ret = cls_rgw_bi_log_list(index_ctx, oid, marker, max - result.size(), entries, truncated);
   if (ret < 0)
     return ret;
 
@@ -4000,27 +4028,27 @@ int RGWRados::process_gc()
   return gc->process();
 }
 
-int RGWRados::cls_rgw_init_index(librados::IoCtx& io_ctx, librados::ObjectWriteOperation& op, string& oid)
+int RGWRados::cls_rgw_init_index(librados::IoCtx& index_ctx, librados::ObjectWriteOperation& op, string& oid)
 {
   bufferlist in;
   cls_rgw_bucket_init(op);
-  int r = io_ctx.operate(oid, &op);
+  int r = index_ctx.operate(oid, &op);
   return r;
 }
 
 int RGWRados::cls_obj_prepare_op(rgw_bucket& bucket, RGWModifyOp op, string& tag,
                                  string& name, string& locator)
 {
-  librados::IoCtx io_ctx;
+  librados::IoCtx index_ctx;
   string oid;
 
-  int r = open_bucket(bucket, io_ctx, oid);
+  int r = open_bucket_index(bucket, index_ctx, oid);
   if (r < 0)
     return r;
 
   ObjectWriteOperation o;
   cls_rgw_bucket_prepare_op(o, op, tag, name, locator);
-  r = io_ctx.operate(oid, &o);
+  r = index_ctx.operate(oid, &o);
   return r;
 }
 
@@ -4029,10 +4057,10 @@ int RGWRados::cls_obj_complete_op(rgw_bucket& bucket, RGWModifyOp op, string& ta
                                   RGWObjEnt& ent, RGWObjCategory category,
 				  list<string> *remove_objs)
 {
-  librados::IoCtx io_ctx;
+  librados::IoCtx index_ctx;
   string oid;
 
-  int r = open_bucket(bucket, io_ctx, oid);
+  int r = open_bucket_index(bucket, index_ctx, oid);
   if (r < 0)
     return r;
 
@@ -4052,7 +4080,7 @@ int RGWRados::cls_obj_complete_op(rgw_bucket& bucket, RGWModifyOp op, string& ta
   cls_rgw_bucket_complete_op(o, op, tag, ver, ent.name, dir_meta, remove_objs);
 
   AioCompletion *c = librados::Rados::aio_create_completion(NULL, NULL, NULL);
-  r = io_ctx.aio_operate(oid, c, &o);
+  r = index_ctx.aio_operate(oid, c, &o);
   c->release();
   return r;
 }
@@ -4083,17 +4111,17 @@ int RGWRados::cls_obj_complete_cancel(rgw_bucket& bucket, string& tag, string& n
 
 int RGWRados::cls_obj_set_bucket_tag_timeout(rgw_bucket& bucket, uint64_t timeout)
 {
-  librados::IoCtx io_ctx;
+  librados::IoCtx index_ctx;
   string oid;
 
-  int r = open_bucket(bucket, io_ctx, oid);
+  int r = open_bucket_index(bucket, index_ctx, oid);
   if (r < 0)
     return r;
 
   ObjectWriteOperation o;
   cls_rgw_bucket_set_tag_timeout(o, timeout);
 
-  r = io_ctx.operate(oid, &o);
+  r = index_ctx.operate(oid, &o);
 
   return r;
 }
@@ -4105,14 +4133,14 @@ int RGWRados::cls_bucket_list(rgw_bucket& bucket, string start, string prefix,
 {
   ldout(cct, 10) << "cls_bucket_list " << bucket << " start " << start << " num " << num << dendl;
 
-  librados::IoCtx io_ctx;
+  librados::IoCtx index_ctx;
   string oid;
-  int r = open_bucket(bucket, io_ctx, oid);
+  int r = open_bucket_index(bucket, index_ctx, oid);
   if (r < 0)
     return r;
 
   struct rgw_bucket_dir dir;
-  r = cls_rgw_list_op(io_ctx, oid, start, prefix, num, &dir, is_truncated);
+  r = cls_rgw_list_op(index_ctx, oid, start, prefix, num, &dir, is_truncated);
   if (r < 0)
     return r;
 
@@ -4144,7 +4172,7 @@ int RGWRados::cls_bucket_list(rgw_bucket& bucket, string start, string prefix,
       /* there are uncommitted ops. We need to check the current state,
        * and if the tags are old we need to do cleanup as well. */
       librados::IoCtx sub_ctx;
-      sub_ctx.dup(io_ctx);
+      sub_ctx.dup(index_ctx);
       r = check_disk_state(sub_ctx, bucket, dirent, e, updates);
       if (r < 0) {
         if (r == -ENOENT)
@@ -4166,7 +4194,7 @@ int RGWRados::cls_bucket_list(rgw_bucket& bucket, string start, string prefix,
     cls_rgw_suggest_changes(o, updates);
     // we don't care if we lose suggested updates, send them off blindly
     AioCompletion *c = librados::Rados::aio_create_completion(NULL, NULL, NULL);
-    r = io_ctx.aio_operate(oid, c, &o);
+    r = index_ctx.aio_operate(oid, c, &o);
     c->release();
   }
   return m.size();
@@ -4233,15 +4261,13 @@ int RGWRados::cls_obj_usage_log_trim(string& oid, string& user, uint64_t start_e
 
 int RGWRados::remove_objs_from_index(rgw_bucket& bucket, list<string>& oid_list)
 {
-  librados::IoCtx io_ctx;
+  librados::IoCtx index_ctx;
+  string dir_oid;
 
-  int r = open_bucket_ctx(bucket, io_ctx);
+  int r = open_bucket_index(bucket, index_ctx, dir_oid);
   if (r < 0)
     return r;
 
-  string dir_oid = dir_oid_prefix;
-  dir_oid.append(bucket.marker);
-
   bufferlist updates;
 
   list<string>::iterator iter;
@@ -4258,7 +4284,7 @@ int RGWRados::remove_objs_from_index(rgw_bucket& bucket, list<string>& oid_list)
 
   bufferlist out;
 
-  r = io_ctx.exec(dir_oid, "rgw", "dir_suggest_changes", updates, out);
+  r = index_ctx.exec(dir_oid, "rgw", "dir_suggest_changes", updates, out);
 
   return r;
 }
@@ -4368,13 +4394,13 @@ int RGWRados::check_disk_state(librados::IoCtx io_ctx,
 
 int RGWRados::cls_bucket_head(rgw_bucket& bucket, struct rgw_bucket_dir_header& header)
 {
-  librados::IoCtx io_ctx;
+  librados::IoCtx index_ctx;
   string oid;
-  int r = open_bucket(bucket, io_ctx, oid);
+  int r = open_bucket_index(bucket, index_ctx, oid);
   if (r < 0)
     return r;
 
-  r = cls_rgw_get_dir_header(io_ctx, oid, &header);
+  r = cls_rgw_get_dir_header(index_ctx, oid, &header);
   if (r < 0)
     return r;
 
@@ -4530,19 +4556,19 @@ int RGWRados::process_intent_log(rgw_bucket& bucket, string& oid,
         complete = false;
         break;
       } else {
-        librados::IoCtx io_ctx;
-        int r = open_bucket_ctx(entry.obj.bucket, io_ctx);
+        librados::IoCtx index_ctx;
+        string oid;
+        int r = open_bucket_index(entry.obj.bucket, index_ctx, oid);
         if (r < 0)
           return r;
         ObjectWriteOperation op;
         op.remove();
-        string oid = dir_oid_prefix;
         oid.append(entry.obj.bucket.marker);
         librados::AioCompletion *completion = rados->aio_create_completion(NULL, NULL, NULL);
-        r = io_ctx.aio_operate(oid, completion, &op);
+        r = index_ctx.aio_operate(oid, completion, &op);
         completion->release();
         if (r < 0 && r != -ENOENT) {
-          cerr << "failed to remove pool: " << entry.obj.bucket.pool << std::endl;
+          cerr << "failed to remove bucket: " << entry.obj.bucket << std::endl;
           complete = false;
         }
       }
diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h
index e81fbe28d3a..bf816d36c75 100644
--- a/src/rgw/rgw_rados.h
+++ b/src/rgw/rgw_rados.h
@@ -465,8 +465,10 @@ class RGWRados
   int open_root_pool_ctx();
   int open_gc_pool_ctx();
 
-  int open_bucket_ctx(rgw_bucket& bucket, librados::IoCtx&  io_ctx);
-  int open_bucket(rgw_bucket& bucket, librados::IoCtx&  io_ctx, string& bucket_oid);
+  int open_bucket_pool_ctx(const string& bucket_name, const string& pool, librados::IoCtx&  io_ctx);
+  int open_bucket_index_ctx(rgw_bucket& bucket, librados::IoCtx&  index_ctx);
+  int open_bucket_data_ctx(rgw_bucket& bucket, librados::IoCtx&  io_ctx);
+  int open_bucket_index(rgw_bucket& bucket, librados::IoCtx&  index_ctx, string& bucket_oid);
 
   struct GetObjState {
     librados::IoCtx io_ctx;