diff options
| -rw-r--r-- | src/rgw/rgw_admin.cc | 50 | ||||
| -rw-r--r-- | src/rgw/rgw_common.h | 2 | ||||
| -rw-r--r-- | src/rgw/rgw_metadata.cc | 180 | ||||
| -rw-r--r-- | src/rgw/rgw_metadata.h | 14 | ||||
| -rw-r--r-- | src/rgw/rgw_user.cc | 19 |
5 files changed, 239 insertions, 26 deletions
diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc index 781206cbb69..2791eac7fea 100644 --- a/src/rgw/rgw_admin.cc +++ b/src/rgw/rgw_admin.cc @@ -85,6 +85,7 @@ void _usage() cerr << " gc process manually process garbage\n"; cerr << " metadata get get metadata info\n"; cerr << " metadata put put metadata info\n"; + cerr << " metadata rm remove metadata info\n"; cerr << " metadata list list metadata info\n"; cerr << " mdlog show show metadata log\n"; cerr << "options:\n"; @@ -190,6 +191,7 @@ enum { OPT_CAPS_RM, OPT_METADATA_GET, OPT_METADATA_PUT, + OPT_METADATA_RM, OPT_METADATA_LIST, OPT_MDLOG_SHOW, }; @@ -361,6 +363,8 @@ static int get_cmd(const char *cmd, const char *prev_cmd, bool *need_more) return OPT_METADATA_GET; if (strcmp(cmd, "put") == 0) return OPT_METADATA_PUT; + if (strcmp(cmd, "rm") == 0) + return OPT_METADATA_RM; if (strcmp(cmd, "list") == 0) return OPT_METADATA_LIST; } else if (strcmp(prev_cmd, "mdlog") == 0) { @@ -830,6 +834,23 @@ public: } }; +static int parse_date_str(const string& date_str, utime_t& ut) +{ + uint64_t epoch = 0; + + if (!date_str.empty()) { + int ret = parse_date(date_str, &epoch); + if (ret < 0) { + cerr << "ERROR: failed to parse date: " << date_str << std::endl; + return -EINVAL; + } + } + + ut = utime_t(epoch, 0); + + return 0; +} + int main(int argc, char **argv) { vector<const char*> args; @@ -1012,6 +1033,7 @@ int main(int argc, char **argv) switch (opt_cmd) { case OPT_METADATA_GET: case OPT_METADATA_PUT: + case OPT_METADATA_RM: case OPT_METADATA_LIST: metadata_key = *i; break; @@ -1482,7 +1504,7 @@ int main(int argc, char **argv) info.subusers[subuser] = u; } - if ((err = rgw_store_user_info(store, info, &old_info, NULL, false)) < 0) { + if ((err = rgw_store_user_info(store, info, &old_info, &objv_tracker, false)) < 0) { cerr << "error storing user info: " << cpp_strerror(-err) << std::endl; break; } @@ -2191,6 +2213,14 @@ next: } } + if (opt_cmd == OPT_METADATA_RM) { + int ret = store->meta_mgr->remove(metadata_key); + if (ret < 0) { + cerr << "ERROR: can't remove key: " << cpp_strerror(-ret) << std::endl; + return -ret; + } + } + if (opt_cmd == OPT_METADATA_LIST) { void *handle; int max = 1000; @@ -2229,11 +2259,19 @@ next: void *handle; list<cls_log_entry> entries; + utime_t start_time, end_time; + + int ret = parse_date_str(start_date, start_time); + if (ret < 0) + return -ret; + + ret = parse_date_str(end_date, end_time); + if (ret < 0) + return -ret; + RGWMetadataLog *meta_log = store->meta_mgr->get_log(); - utime_t from_time; - utime_t end_time; - meta_log->init_list_entries(store, from_time, end_time, &handle); + meta_log->init_list_entries(store, start_time, end_time, &handle); bool truncated; @@ -2247,9 +2285,7 @@ next: for (list<cls_log_entry>::iterator iter = entries.begin(); iter != entries.end(); ++iter) { cls_log_entry& entry = *iter; - formatter->open_object_section("entry"); - formatter->dump_string("name", entry.name); - formatter->close_section(); + store->meta_mgr->dump_log_entry(entry, formatter); } formatter->flush(cout); } while (truncated); diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h index bb203ef72bb..50845aa0737 100644 --- a/src/rgw/rgw_common.h +++ b/src/rgw/rgw_common.h @@ -57,7 +57,7 @@ using ceph::crypto::MD5; #define RGW_ATTR_MANIFEST RGW_ATTR_PREFIX "manifest" #define RGW_ATTR_USER_MANIFEST RGW_ATTR_PREFIX "user_manifest" -#define RGW_BUCKETS_OBJ_PREFIX ".buckets" +#define RGW_BUCKETS_OBJ_SUFFIX ".buckets" #define RGW_MAX_CHUNK_SIZE (512*1024) #define RGW_MAX_PENDING_CHUNKS 16 diff --git a/src/rgw/rgw_metadata.cc b/src/rgw/rgw_metadata.cc index d22f5d63597..695372c6399 100644 --- a/src/rgw/rgw_metadata.cc +++ b/src/rgw/rgw_metadata.cc @@ -6,6 +6,74 @@ #include "rgw_rados.h" +#define dout_subsys ceph_subsys_rgw + +enum RGWMDLogStatus { + MDLOG_STATUS_UNKNOWN, + MDLOG_STATUS_WRITING, + MDLOG_STATUS_REMOVING, + MDLOG_STATUS_COMPLETE, +}; + +struct LogStatusDump { + RGWMDLogStatus status; + + LogStatusDump(RGWMDLogStatus _status) : status(_status) {} + void dump(Formatter *f) const { + string s; + switch (status) { + case MDLOG_STATUS_WRITING: + s = "writing"; + break; + case MDLOG_STATUS_REMOVING: + s = "removing"; + break; + case MDLOG_STATUS_COMPLETE: + s = "complete"; + break; + default: + s = "unknown"; + break; + } + encode_json("status", s, f); + } +}; + +struct RGWMetadataLogData { + obj_version read_version; + obj_version write_version; + RGWMDLogStatus status; + + RGWMetadataLogData() : status(MDLOG_STATUS_UNKNOWN) {} + + void encode(bufferlist& bl) const { + ENCODE_START(1, 1, bl); + ::encode(read_version, bl); + ::encode(write_version, bl); + uint32_t s = (uint32_t)status; + ::encode(s, bl); + ENCODE_FINISH(bl); + } + + void decode(bufferlist::iterator& bl) { + DECODE_START(1, bl); + ::decode(read_version, bl); + ::decode(write_version, bl); + uint32_t s; + ::decode(s, bl); + status = (RGWMDLogStatus)s; + DECODE_FINISH(bl); + } + + void dump(Formatter *f) const { + encode_json("read_version", read_version, f); + encode_json("write_version", write_version, f); + encode_json("status", LogStatusDump(status), f); + }; +}; +WRITE_CLASS_ENCODER(RGWMetadataLogData); + + int RGWMetadataLog::add_entry(RGWRados *store, string& section, string& key, bufferlist& bl) { string oid; @@ -49,7 +117,7 @@ int RGWMetadataLog::list_entries(void *handle, bool is_truncated; int ret = store->time_log_list(ctx->cur_oid, ctx->from_time, ctx->end_time, max_entries - entries.size(), ents, ctx->marker, &is_truncated); - if (ret = -ENOENT) { + if (ret == -ENOENT) { is_truncated = false; ret = 0; } @@ -95,8 +163,9 @@ public: virtual int get(RGWRados *store, string& entry, RGWMetadataObject **obj) { return -ENOTSUP; } virtual int put(RGWRados *store, string& entry, RGWObjVersionTracker& objv_tracker, JSONObj *obj) { return -ENOTSUP; } - virtual int put_obj(RGWRados *store, string& key, bufferlist& bl, bool exclusive, - RGWObjVersionTracker *objv_tracker, map<string, bufferlist> *pattrs) { return -ENOTSUP; } + virtual int put_entry(RGWRados *store, string& key, bufferlist& bl, bool exclusive, + RGWObjVersionTracker *objv_tracker, map<string, bufferlist> *pattrs) { return -ENOTSUP; } + virtual int remove(RGWRados *store, string& entry, RGWObjVersionTracker& objv_tracker) { return -ENOTSUP; } virtual int list_keys_init(RGWRados *store, void **phandle) { iter_data *data = new iter_data; @@ -126,7 +195,7 @@ public: static RGWMetadataTopHandler md_top_handler; -RGWMetadataManager::RGWMetadataManager(CephContext *_cct, RGWRados *_store) : store(_store) +RGWMetadataManager::RGWMetadataManager(CephContext *_cct, RGWRados *_store) : cct(_cct), store(_store) { md_log = new RGWMetadataLog(_cct, _store); } @@ -236,7 +305,6 @@ int RGWMetadataManager::put(string& metadata_key, bufferlist& bl) return -EINVAL; } - string meadata_key; RGWObjVersionTracker objv_tracker; obj_version *objv = &objv_tracker.write_version; @@ -253,6 +321,59 @@ int RGWMetadataManager::put(string& metadata_key, bufferlist& bl) return handler->put(store, entry, objv_tracker, jo); } +int RGWMetadataManager::remove(string& metadata_key) +{ + RGWMetadataHandler *handler; + string entry; + + int ret = find_handler(metadata_key, &handler, entry); + if (ret < 0) + return ret; + + RGWMetadataObject *obj; + + ret = handler->get(store, entry, &obj); + if (ret < 0) { + return ret; + } + + RGWObjVersionTracker objv_tracker; + + objv_tracker.read_version = obj->get_version(); + + + RGWMetadataLogData log_data; + bufferlist logbl; + + log_data.read_version = objv_tracker.read_version; + log_data.write_version = objv_tracker.write_version; + + log_data.status = MDLOG_STATUS_REMOVING; + + ::encode(log_data, logbl); + + string section, key; + parse_metadata_key(entry, section, key); + + ret = md_log->add_entry(store, section, key, logbl); + if (ret < 0) + return ret; + + ret = handler->remove(store, entry, objv_tracker); + if (ret < 0) { + return ret; + } + + logbl.clear(); + log_data.status = MDLOG_STATUS_COMPLETE; + + ret = md_log->add_entry(store, section, key, logbl); + if (ret < 0) + return ret; + + return 0; +} + struct list_keys_handle { void *handle; @@ -305,6 +426,25 @@ void RGWMetadataManager::list_keys_complete(void *handle) delete h; } +void RGWMetadataManager::dump_log_entry(cls_log_entry& entry, Formatter *f) +{ + f->open_object_section("entry"); + f->dump_string("section", entry.section); + f->dump_string("name", entry.name); + entry.timestamp.gmtime(f->dump_stream("timestamp")); + + try { + RGWMetadataLogData log_data; + bufferlist::iterator iter = entry.data.begin(); + ::decode(log_data, iter); + + encode_json("data", log_data, f); + } catch (buffer::error& err) { + lderr(cct) << "failed to decode log entry: " << entry.section << ":" << entry.name<< " ts=" << entry.timestamp << dendl; + } + f->close_section(); +} + void RGWMetadataManager::get_sections(list<string>& sections) { for (map<string, RGWMetadataHandler *>::iterator iter = handlers.begin(); iter != handlers.end(); ++iter) { @@ -312,20 +452,42 @@ void RGWMetadataManager::get_sections(list<string>& sections) } } - -int RGWMetadataManager::put_obj(RGWMetadataHandler *handler, string& key, bufferlist& bl, bool exclusive, - RGWObjVersionTracker *objv_tracker, map<string, bufferlist> *pattrs) +int RGWMetadataManager::put_entry(RGWMetadataHandler *handler, string& key, bufferlist& bl, bool exclusive, + RGWObjVersionTracker *objv_tracker, map<string, bufferlist> *pattrs) { bufferlist logbl; string section = handler->get_type(); + + /* if write version has not been set, and there's a read version, set it so that we can + * log it + */ + if (objv_tracker && objv_tracker->read_version.ver && + !objv_tracker->write_version.ver) { + objv_tracker->write_version = objv_tracker->read_version; + objv_tracker->write_version.ver++; + } + + RGWMetadataLogData log_data; + log_data.read_version = objv_tracker->read_version; + log_data.write_version = objv_tracker->write_version; + + log_data.status = MDLOG_STATUS_WRITING; + + ::encode(log_data, logbl); + int ret = md_log->add_entry(store, section, key, logbl); if (ret < 0) return ret; - ret = handler->put_obj(store, key, bl, exclusive, objv_tracker, pattrs); + ret = handler->put_entry(store, key, bl, exclusive, objv_tracker, pattrs); if (ret < 0) return ret; + log_data.status = MDLOG_STATUS_COMPLETE; + + logbl.clear(); + ::encode(log_data, logbl); + ret = md_log->add_entry(store, section, key, logbl); if (ret < 0) return ret; diff --git a/src/rgw/rgw_metadata.h b/src/rgw/rgw_metadata.h index bee940779f5..686e41ca116 100644 --- a/src/rgw/rgw_metadata.h +++ b/src/rgw/rgw_metadata.h @@ -33,15 +33,15 @@ class RGWMetadataHandler { friend class RGWMetadataManager; protected: - virtual int put_obj(RGWRados *store, string& key, bufferlist& bl, bool exclusive, - RGWObjVersionTracker *objv_tracker, map<string, bufferlist> *pattrs = NULL) = 0; + virtual int put_entry(RGWRados *store, string& key, bufferlist& bl, bool exclusive, + RGWObjVersionTracker *objv_tracker, map<string, bufferlist> *pattrs = NULL) = 0; public: virtual ~RGWMetadataHandler() {} virtual string get_type() = 0; virtual int get(RGWRados *store, string& entry, RGWMetadataObject **obj) = 0; virtual int put(RGWRados *store, string& entry, RGWObjVersionTracker& objv_tracker, JSONObj *obj) = 0; - + virtual int remove(RGWRados *store, string& entry, RGWObjVersionTracker& objv_tracker) = 0; virtual int list_keys_init(RGWRados *store, void **phandle) = 0; virtual int list_keys_next(void *handle, int max, list<string>& keys, bool *truncated) = 0; @@ -92,6 +92,7 @@ public: class RGWMetadataManager { map<string, RGWMetadataHandler *> handlers; + CephContext *cct; RGWRados *store; RGWMetadataLog *md_log; @@ -107,15 +108,18 @@ public: RGWMetadataHandler *get_handler(const char *type); - int put_obj(RGWMetadataHandler *handler, string& key, bufferlist& bl, bool exclusive, - RGWObjVersionTracker *objv_tracker, map<string, bufferlist> *pattrs = NULL); + int put_entry(RGWMetadataHandler *handler, string& key, bufferlist& bl, bool exclusive, + RGWObjVersionTracker *objv_tracker, map<string, bufferlist> *pattrs = NULL); int get(string& metadata_key, Formatter *f); int put(string& metadata_key, bufferlist& bl); + int remove(string& metadata_key); int list_keys_init(string& section, void **phandle); int list_keys_next(void *handle, int max, list<string>& keys, bool *truncated); void list_keys_complete(void *handle); + void dump_log_entry(cls_log_entry& entry, Formatter *f); + void get_sections(list<string>& sections); RGWMetadataLog *get_log() { return md_log; } diff --git a/src/rgw/rgw_user.cc b/src/rgw/rgw_user.cc index 91329bf4538..867de4c2c1a 100644 --- a/src/rgw/rgw_user.cc +++ b/src/rgw/rgw_user.cc @@ -85,7 +85,7 @@ int rgw_store_user_info(RGWRados *store, RGWUserInfo& info, RGWUserInfo *old_inf ::encode(ui, data_bl); ::encode(info, data_bl); - ret = store->meta_mgr->put_obj(user_meta_handler, info.user_id, data_bl, exclusive, objv_tracker); + ret = store->meta_mgr->put_entry(user_meta_handler, info.user_id, data_bl, exclusive, objv_tracker); if (ret < 0) return ret; @@ -212,7 +212,7 @@ extern int rgw_get_user_info_by_access_key(RGWRados *store, string& access_key, static void get_buckets_obj(string& user_id, string& buckets_obj_id) { buckets_obj_id = user_id; - buckets_obj_id += RGW_BUCKETS_OBJ_PREFIX; + buckets_obj_id += RGW_BUCKETS_OBJ_SUFFIX; } static int rgw_read_buckets_from_attr(RGWRados *store, string& user_id, RGWUserBuckets& buckets) @@ -529,9 +529,20 @@ public: return 0; } + int remove(RGWRados *store, string& entry, RGWObjVersionTracker& objv_tracker) { +#warning FIXME: use objv_tracker - int put_obj(RGWRados *store, string& key, bufferlist& bl, bool exclusive, - RGWObjVersionTracker *objv_tracker, map<string, bufferlist> *pattrs) { + RGWUserInfo info; + RGWObjVersionTracker ot; + int ret = rgw_get_user_info_by_uid(store, entry, info, &ot); + if (ret < 0) + return ret; + + return rgw_delete_user(store, info); + } + + int put_entry(RGWRados *store, string& key, bufferlist& bl, bool exclusive, + RGWObjVersionTracker *objv_tracker, map<string, bufferlist> *pattrs) { return rgw_put_system_obj(store, store->zone.user_uid_pool, key, bl.c_str(), bl.length(), exclusive, objv_tracker, pattrs); |