rgw: a tool to fix buckets with leaked multipart references

Checks specified bucket for the #4011 symptoms, optionally fix
the issue.

sytax:
  radosgw-admin bucket check --bucket=<bucket> [--fix]

Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>

3 files changed, 109 insertions, 9 deletions

diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index f0781504efc..86194e6ceb5 100644
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -125,6 +125,7 @@ enum {
   OPT_BUCKET_LINK,
   OPT_BUCKET_UNLINK,
   OPT_BUCKET_STATS,
+  OPT_BUCKET_CHECK,
   OPT_OBJECT_UNLINK,
   OPT_POLICY,
   OPT_POOL_ADD,
@@ -256,6 +257,8 @@ static int get_cmd(const char *cmd, const char *prev_cmd, bool *need_more)
       return OPT_BUCKET_UNLINK;
     if (strcmp(cmd, "stats") == 0)
       return OPT_BUCKET_STATS;
+    if (strcmp(cmd, "check") == 0)
+      return OPT_BUCKET_CHECK;
   } else if (strcmp(prev_cmd, "object") == 0) {
     if (strcmp(cmd, "unlink") == 0)
       return OPT_OBJECT_UNLINK;
@@ -522,6 +525,88 @@ enum ObjectKeyType {
   KEY_TYPE_S3,
 };
 
+void check_bad_index_multipart(rgw_bucket& bucket, bool fix)
+{
+  int max = 1000;
+  string prefix;
+  string marker;
+  string delim;
+
+  map<string, bool> common_prefixes;
+  string ns = "multipart";
+
+  bool is_truncated;
+  list<string> objs_to_unlink;
+  map<string, bool> meta_objs;
+  map<string, string> all_objs;
+
+  do {
+    vector<RGWObjEnt> result;
+    int r = rgwstore->list_objects(bucket, max, prefix, delim, marker,
+				 result, common_prefixes, false, ns,
+				 &is_truncated, NULL);
+
+    if (r < 0) {
+      cerr << "failed to list objects in bucket=" << bucket << " err=" << cpp_strerror(-r) << std::endl;
+      return;
+    }
+
+    vector<RGWObjEnt>::iterator iter;
+    for (iter = result.begin(); iter != result.end(); ++iter) {
+      RGWObjEnt& ent = *iter;
+
+      rgw_obj obj(bucket, ent.name);
+      obj.set_ns(ns);
+
+      string& oid = obj.object;
+      marker = oid;
+
+      int pos = oid.find_last_of('.');
+      if (pos < 0)
+	continue;
+
+      string name = oid.substr(0, pos);
+      string suffix = oid.substr(pos + 1);
+
+      if (suffix.compare("meta") == 0) {
+	meta_objs[name] = true;
+      } else {
+	all_objs[oid] = name;
+      }
+    }
+
+  } while (is_truncated);
+
+  map<string, string>::iterator aiter;
+  for (aiter = all_objs.begin(); aiter != all_objs.end(); ++aiter) {
+    string& name = aiter->second;
+
+    if (meta_objs.find(name) == meta_objs.end()) {
+      objs_to_unlink.push_back(aiter->first);
+    }
+  }
+
+  if (objs_to_unlink.empty())
+    return;
+
+  if (!fix) {
+    cout << "Need to unlink the following objects from bucket=" << bucket << std::endl;
+  } else {
+    cout << "Unlinking the following objects from bucket=" << bucket << std::endl;
+  }
+  for (list<string>::iterator oiter = objs_to_unlink.begin(); oiter != objs_to_unlink.end(); ++oiter) {
+    cout << *oiter << std::endl;
+  }
+
+  if (fix) {
+    int r = rgwstore->remove_objs_from_index(bucket, objs_to_unlink);
+    if (r < 0) {
+      cerr << "ERROR: remove_obj_from_index() returned error: " << cpp_strerror(-r) << std::endl;
+    }
+  }
+
+}
+
 static void parse_date(string& date, uint64_t *epoch, string *out_date = NULL, string *out_time = NULL)
 {
   struct tm tm;
@@ -594,6 +679,7 @@ int main(int argc, char **argv)
   int purge_keys = false;
   int yes_i_really_mean_it = false;
   int max_buckets = -1;
+  int fix = false;
 
   std::string val;
   std::ostringstream errs;
@@ -678,6 +764,8 @@ int main(int argc, char **argv)
       // do nothing
     } else if (ceph_argparse_binary_flag(args, i, &yes_i_really_mean_it, NULL, "--yes-i-really-mean-it", (char*)NULL)) {
       // do nothing
+    } else if (ceph_argparse_binary_flag(args, i, &fix, NULL, "--fix", (char*)NULL)) {
+      // do nothing
     } else {
       ++i;
     }
@@ -1503,12 +1591,18 @@ next:
   }
 
   if (opt_cmd == OPT_OBJECT_UNLINK) {
-    int ret = rgwstore->remove_obj_from_index(bucket, object);
+    list<string> oid_list;
+    oid_list.push_back(object);
+    int ret = rgwstore->remove_objs_from_index(bucket, oid_list);
     if (ret < 0) {
       cerr << "ERROR: remove_obj_from_index() returned error: " << cpp_strerror(-ret) << std::endl;
       return 1;
     }
   }
 
+  if (opt_cmd == OPT_BUCKET_CHECK) {
+    check_bad_index_multipart(bucket, fix);
+  }
+
   return 0;
 }
diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index a5655a9038e..cb396bfadbc 100644
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -2751,7 +2751,7 @@ int RGWRados::cls_obj_usage_log_trim(string& oid, string& user, uint64_t start_e
   return r;
 }
 
-int RGWRados::remove_obj_from_index(rgw_bucket& bucket, const string& oid)
+int RGWRados::remove_objs_from_index(rgw_bucket& bucket, list<string>& oid_list)
 {
   librados::IoCtx io_ctx;
 
@@ -2762,13 +2762,19 @@ int RGWRados::remove_obj_from_index(rgw_bucket& bucket, const string& oid)
   string dir_oid = dir_oid_prefix;
   dir_oid.append(bucket.marker);
 
-  rgw_bucket_dir_entry entry;
-  entry.epoch = (uint64_t)-1; // ULLONG_MAX, needed to that objclass doesn't skip out request
-  entry.name = oid;
-
   bufferlist updates;
-  updates.append(CEPH_RGW_REMOVE);
-  ::encode(entry, updates);
+
+  list<string>::iterator iter;
+
+  for (iter = oid_list.begin(); iter != oid_list.end(); ++iter) {
+    string& oid = *iter;
+    dout(2) << "RGWRados::remove_objs_from_index bucket=" << bucket << " oid=" << oid << dendl;
+    rgw_bucket_dir_entry entry;
+    entry.epoch = (uint64_t)-1; // ULLONG_MAX, needed to that objclass doesn't skip out request
+    entry.name = oid;
+    updates.append(CEPH_RGW_REMOVE);
+    ::encode(entry, updates);
+  }
 
   bufferlist out;
 
diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h
index 7d681c9e74b..7e4e840da99 100644
--- a/src/rgw/rgw_rados.h
+++ b/src/rgw/rgw_rados.h
@@ -583,7 +583,7 @@ public:
   /// clean up/process any temporary objects older than given date[/time]
   int remove_temp_objects(string date, string time);
 
-  int remove_obj_from_index(rgw_bucket& bucket, const string& oid);
+  int remove_objs_from_index(rgw_bucket& bucket, list<string>& oid_list);
 
  private:
   int process_intent_log(rgw_bucket& bucket, string& oid,