diff options
author | Babu Shanmugam <anbu@enovance.com> | 2013-06-13 07:57:45 +0530 |
---|---|---|
committer | Yehuda Sadeh <yehuda@inktank.com> | 2013-06-20 13:32:29 -0700 |
commit | 3b4c11b2d9edb00acde419da3f5707b1a85c8283 (patch) | |
tree | 9f618fc1188af270ff08cb82eb82c8a29f9f7d8d | |
parent | 96c9493d7be1f584e47221552f380831bb751296 (diff) | |
download | ceph-3b4c11b2d9edb00acde419da3f5707b1a85c8283.tar.gz |
rgw: add max-entries, marker for log operations
Signed-off-by: Babu Shanmugam <anbu@enovance.com>
Conflicts:
src/rgw/rgw_admin.cc
src/rgw/rgw_rest_log.cc
-rw-r--r-- | src/rgw/rgw_admin.cc | 3 | ||||
-rw-r--r-- | src/rgw/rgw_bucket.cc | 3 | ||||
-rw-r--r-- | src/rgw/rgw_metadata.cc | 10 | ||||
-rw-r--r-- | src/rgw/rgw_metadata.h | 5 | ||||
-rw-r--r-- | src/rgw/rgw_rados.cc | 9 | ||||
-rw-r--r-- | src/rgw/rgw_rados.h | 1 | ||||
-rw-r--r-- | src/rgw/rgw_rest_log.cc | 98 | ||||
-rw-r--r-- | src/test/test_rgw_admin_log.cc | 29 |
8 files changed, 108 insertions, 50 deletions
diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc index 0d06e1352bb..ce18e2da43f 100644 --- a/src/rgw/rgw_admin.cc +++ b/src/rgw/rgw_admin.cc @@ -1798,7 +1798,6 @@ next: if (ret < 0) return -ret; - int i = (specified_shard_id ? shard_id : 0); formatter->open_array_section("entries"); @@ -1808,7 +1807,7 @@ next: list<cls_log_entry> entries; - meta_log->init_list_entries(i, start_time, end_time, &handle); + meta_log->init_list_entries(i, start_time, end_time, marker, &handle); bool truncated; do { diff --git a/src/rgw/rgw_bucket.cc b/src/rgw/rgw_bucket.cc index 7a6423f660b..7b758b08fd1 100644 --- a/src/rgw/rgw_bucket.cc +++ b/src/rgw/rgw_bucket.cc @@ -1159,7 +1159,7 @@ int RGWDataChangesLog::list_entries(int shard, utime_t& start_time, utime_t& end list<cls_log_entry> log_entries; int ret = store->time_log_list(oids[shard], start_time, end_time, - max_entries, log_entries, marker, truncated); + max_entries, log_entries, marker, truncated); if (ret < 0) return ret; @@ -1182,7 +1182,6 @@ int RGWDataChangesLog::list_entries(int shard, utime_t& start_time, utime_t& end int RGWDataChangesLog::list_entries(utime_t& start_time, utime_t& end_time, int max_entries, list<rgw_data_change>& entries, LogMarker& marker, bool *ptruncated) { bool truncated; - entries.clear(); for (; marker.shard < num_shards && (int)entries.size() < max_entries; diff --git a/src/rgw/rgw_metadata.cc b/src/rgw/rgw_metadata.cc index 2a90f0d0e0c..6e0d4b3b632 100644 --- a/src/rgw/rgw_metadata.cc +++ b/src/rgw/rgw_metadata.cc @@ -79,13 +79,15 @@ int RGWMetadataLog::add_entry(RGWRados *store, string& section, string& key, buf return store->time_log_add(oid, now, section, key, bl); } -void RGWMetadataLog::init_list_entries(int shard_id, utime_t& from_time, utime_t& end_time, void **handle) +void RGWMetadataLog::init_list_entries(int shard_id, utime_t& from_time, utime_t& end_time, + string& marker, void **handle) { LogListCtx *ctx = new LogListCtx(); ctx->cur_shard = shard_id; ctx->from_time = from_time; - ctx->end_time = end_time; + ctx->end_time = end_time; + ctx->marker = marker; get_shard_oid(ctx->cur_shard, ctx->cur_oid); @@ -99,7 +101,7 @@ void RGWMetadataLog::complete_list_entries(void *handle) { int RGWMetadataLog::list_entries(void *handle, int max_entries, - list<cls_log_entry>& entries, + list<cls_log_entry>& entries, bool *truncated) { LogListCtx *ctx = (LogListCtx *)handle; @@ -108,8 +110,6 @@ int RGWMetadataLog::list_entries(void *handle, return 0; } - entries.clear(); - int ret = store->time_log_list(ctx->cur_oid, ctx->from_time, ctx->end_time, max_entries, entries, ctx->marker, truncated); if ((ret < 0) && (ret != -ENOENT)) diff --git a/src/rgw/rgw_metadata.h b/src/rgw/rgw_metadata.h index eaac1fac987..5f0fafefc5f 100644 --- a/src/rgw/rgw_metadata.h +++ b/src/rgw/rgw_metadata.h @@ -91,12 +91,11 @@ public: LogListCtx() : done(false) {} }; - void init_list_entries(int shard_id, utime_t& from_time, utime_t& end_time, void **handle); + void init_list_entries(int shard_id, utime_t& from_time, utime_t& end_time, string& marker, void **handle); void complete_list_entries(void *handle); int list_entries(void *handle, int max_entries, - list<cls_log_entry>& entries, - bool *truncated); + list<cls_log_entry>& entries, bool *truncated); int trim(int shard_id, utime_t& from_time, utime_t& end_time); int lock_exclusive(int shard_id, utime_t& duration, string& owner_id); diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index 7ad84c44617..9a485600b6d 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -60,6 +60,7 @@ static string region_info_oid_prefix = "region_info."; static string default_region_info_oid = "default.region"; static string region_map_oid = "region_map"; +static string log_lock_name = "rgw_log_lock"; static RGWObjCategory main_category = RGW_OBJ_CATEGORY_MAIN; @@ -1454,8 +1455,8 @@ int RGWRados::time_log_list(const string& oid, utime_t& start_time, utime_t& end int r = rados->ioctx_create(log_pool, io_ctx); if (r < 0) return r; - librados::ObjectReadOperation op; + cls_log_list(op, start_time, end_time, marker, max_entries, entries, &marker, truncated); bufferlist obl; @@ -1489,8 +1490,7 @@ int RGWRados::lock_exclusive(rgw_bucket& pool, const string& oid, utime_t& durat if (r < 0) return r; - string lock_name = RGW_INDEX_LOCK_NAME; - rados::cls::lock::Lock l(lock_name); + rados::cls::lock::Lock l(log_lock_name); l.set_duration(duration); l.set_cookie(owner_id); @@ -1506,8 +1506,7 @@ int RGWRados::unlock(rgw_bucket& pool, const string& oid, string& owner_id) { if (r < 0) return r; - string lock_name = RGW_INDEX_LOCK_NAME; - rados::cls::lock::Lock l(lock_name); + rados::cls::lock::Lock l(log_lock_name); l.set_cookie(owner_id); return l.unlock(&io_ctx, oid); diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h index f292f47cb0e..7e08d8016c0 100644 --- a/src/rgw/rgw_rados.h +++ b/src/rgw/rgw_rados.h @@ -25,7 +25,6 @@ class RGWGC; #define RGW_OBJ_NS_MULTIPART "multipart" #define RGW_OBJ_NS_SHADOW "shadow" -#define RGW_INDEX_LOCK_NAME "rgw_process" static inline void prepend_bucket_marker(rgw_bucket& bucket, string& orig_oid, string& oid) { diff --git a/src/rgw/rgw_rest_log.cc b/src/rgw/rgw_rest_log.cc index ba821c9b9a2..f066b6b4b04 100644 --- a/src/rgw/rgw_rest_log.cc +++ b/src/rgw/rgw_rest_log.cc @@ -20,6 +20,7 @@ #include "rgw_client_io.h" #include "common/errno.h" +#define LOG_CLASS_LIST_MAX_ENTRIES (1000) #define dout_subsys ceph_subsys_rgw static int parse_date_str(string& in, utime_t& out) { @@ -37,16 +38,17 @@ static int parse_date_str(string& in, utime_t& out) { void RGWOp_MDLog_List::execute() { string shard = s->info.args.get("id"); - + string max_entries_str = s->info.args.get("max-entries"); string st = s->info.args.get("start-time"), et = s->info.args.get("end-time"), + marker = s->info.args.get("marker"), err; utime_t ut_st, ut_et; void *handle; - int shard_id; + unsigned shard_id, max_entries = 0; - shard_id = strict_strtol(shard.c_str(), 10, &err); + shard_id = (unsigned)strict_strtol(shard.c_str(), 10, &err); if (!err.empty()) { dout(5) << "Error parsing shard_id " << shard << dendl; http_ret = -EINVAL; @@ -63,13 +65,29 @@ void RGWOp_MDLog_List::execute() { return; } + if (!max_entries_str.empty()) { + max_entries = (unsigned)strict_strtol(max_entries_str.c_str(), 10, &err); + if (!err.empty()) { + dout(5) << "Error parsing max-entries " << max_entries_str << dendl; + http_ret = -EINVAL; + return; + } + } else + max_entries = LOG_CLASS_LIST_MAX_ENTRIES; + RGWMetadataLog *meta_log = store->meta_mgr->get_log(); - meta_log->init_list_entries(shard_id, ut_st, ut_et, &handle); + meta_log->init_list_entries(shard_id, ut_st, ut_et, marker, &handle); bool truncated; + do { + http_ret = meta_log->list_entries(handle, max_entries, entries, &truncated); + if (http_ret < 0) + break; - http_ret = meta_log->list_entries(handle, 1000, entries, &truncated); + if (!max_entries_str.empty()) + max_entries -= entries.size(); + } while (truncated && (max_entries > 0)); } void RGWOp_MDLog_List::send_response() { @@ -114,11 +132,11 @@ void RGWOp_MDLog_Delete::execute() { err; utime_t ut_st, ut_et; - int shard_id; + unsigned shard_id; http_ret = 0; - shard_id = strict_strtol(shard.c_str(), 10, &err); + shard_id = (unsigned)strict_strtol(shard.c_str(), 10, &err); if (!err.empty()) { dout(5) << "Error parsing shard_id " << shard << dendl; http_ret = -EINVAL; @@ -145,7 +163,7 @@ void RGWOp_MDLog_Delete::execute() { void RGWOp_MDLog_Lock::execute() { string shard_id_str, duration_str, lock_id; - int shard_id; + unsigned shard_id; http_ret = 0; @@ -162,7 +180,7 @@ void RGWOp_MDLog_Lock::execute() { } string err; - shard_id = strict_strtol(shard_id_str.c_str(), 10, &err); + shard_id = (unsigned)strict_strtol(shard_id_str.c_str(), 10, &err); if (!err.empty()) { dout(5) << "Error parsing shard_id param " << shard_id_str << dendl; http_ret = -EINVAL; @@ -170,8 +188,8 @@ void RGWOp_MDLog_Lock::execute() { } RGWMetadataLog *meta_log = store->meta_mgr->get_log(); - int dur; - dur = strict_strtol(duration_str.c_str(), 10, &err); + unsigned dur; + dur = (unsigned)strict_strtol(duration_str.c_str(), 10, &err); if (!err.empty() || dur <= 0) { dout(5) << "invalid length param " << duration_str << dendl; http_ret = -EINVAL; @@ -183,7 +201,7 @@ void RGWOp_MDLog_Lock::execute() { void RGWOp_MDLog_Unlock::execute() { string shard_id_str, lock_id; - int shard_id; + unsigned shard_id; http_ret = 0; @@ -198,7 +216,7 @@ void RGWOp_MDLog_Unlock::execute() { } string err; - shard_id = strict_strtol(shard_id_str.c_str(), 10, &err); + shard_id = (unsigned)strict_strtol(shard_id_str.c_str(), 10, &err); if (!err.empty()) { dout(5) << "Error parsing shard_id param " << shard_id_str << dendl; http_ret = -EINVAL; @@ -214,7 +232,7 @@ void RGWOp_BILog_List::execute() { marker = s->info.args.get("marker"), max_entries_str = s->info.args.get("max-entries"); RGWBucketInfo bucket_info; - int max_entries; + unsigned max_entries; if (bucket_name.empty()) { dout(5) << "ERROR: bucket not specified" << dendl; @@ -229,12 +247,12 @@ void RGWOp_BILog_List::execute() { } bool truncated; - int count = 0; + unsigned count = 0; string err; - max_entries = strict_strtol(max_entries_str.c_str(), 10, &err); + max_entries = (unsigned)strict_strtol(max_entries_str.c_str(), 10, &err); if (!err.empty()) - max_entries = 1000; + max_entries = LOG_CLASS_LIST_MAX_ENTRIES; send_response(); do { @@ -318,12 +336,14 @@ void RGWOp_DATALog_List::execute() { string st = s->info.args.get("start-time"), et = s->info.args.get("end-time"), + max_entries_str = s->info.args.get("max-entries"), + marker = s->info.args.get("marker"), err; utime_t ut_st, ut_et; - int shard_id; + unsigned shard_id, max_entries = 0; - shard_id = strict_strtol(shard.c_str(), 10, &err); + shard_id = (unsigned)strict_strtol(shard.c_str(), 10, &err); if (!err.empty()) { dout(5) << "Error parsing shard_id " << shard << dendl; http_ret = -EINVAL; @@ -340,12 +360,26 @@ void RGWOp_DATALog_List::execute() { return; } - string marker; - bool truncated; -#define DATALOG_LIST_MAX_ENTRIES 1000 + if (!max_entries_str.empty()) { + max_entries = (unsigned)strict_strtol(max_entries_str.c_str(), 10, &err); + if (!err.empty()) { + dout(5) << "Error parsing max-entries " << max_entries_str << dendl; + http_ret = -EINVAL; + return; + } + } else + max_entries = LOG_CLASS_LIST_MAX_ENTRIES; - http_ret = store->data_log->list_entries(shard_id, ut_st, ut_et, - DATALOG_LIST_MAX_ENTRIES, entries, marker, &truncated); + bool truncated; + do { + http_ret = store->data_log->list_entries(shard_id, ut_st, ut_et, + max_entries, entries, marker, &truncated); + if (http_ret < 0) + break; + + if (!max_entries_str.empty()) + max_entries -= entries.size(); + } while (truncated && (max_entries > 0)); } void RGWOp_DATALog_List::send_response() { @@ -386,7 +420,7 @@ void RGWOp_DATALog_GetShardsInfo::send_response() { void RGWOp_DATALog_Lock::execute() { string shard_id_str, duration_str, lock_id; - int shard_id; + unsigned shard_id; http_ret = 0; @@ -403,15 +437,15 @@ void RGWOp_DATALog_Lock::execute() { } string err; - shard_id = strict_strtol(shard_id_str.c_str(), 10, &err); + shard_id = (unsigned)strict_strtol(shard_id_str.c_str(), 10, &err); if (!err.empty()) { dout(5) << "Error parsing shard_id param " << shard_id_str << dendl; http_ret = -EINVAL; return; } - int dur; - dur = strict_strtol(duration_str.c_str(), 10, &err); + unsigned dur; + dur = (unsigned)strict_strtol(duration_str.c_str(), 10, &err); if (!err.empty() || dur <= 0) { dout(5) << "invalid length param " << duration_str << dendl; http_ret = -EINVAL; @@ -423,7 +457,7 @@ void RGWOp_DATALog_Lock::execute() { void RGWOp_DATALog_Unlock::execute() { string shard_id_str, lock_id; - int shard_id; + unsigned shard_id; http_ret = 0; @@ -438,7 +472,7 @@ void RGWOp_DATALog_Unlock::execute() { } string err; - shard_id = strict_strtol(shard_id_str.c_str(), 10, &err); + shard_id = (unsigned)strict_strtol(shard_id_str.c_str(), 10, &err); if (!err.empty()) { dout(5) << "Error parsing shard_id param " << shard_id_str << dendl; http_ret = -EINVAL; @@ -455,11 +489,11 @@ void RGWOp_DATALog_Delete::execute() { err; utime_t ut_st, ut_et; - int shard_id; + unsigned shard_id; http_ret = 0; - shard_id = strict_strtol(shard.c_str(), 10, &err); + shard_id = (unsigned)strict_strtol(shard.c_str(), 10, &err); if (!err.empty()) { dout(5) << "Error parsing shard_id " << shard << dendl; http_ret = -EINVAL; diff --git a/src/test/test_rgw_admin_log.cc b/src/test/test_rgw_admin_log.cc index 40d0f70c3bc..3ce853ab67e 100644 --- a/src/test/test_rgw_admin_log.cc +++ b/src/test/test_rgw_admin_log.cc @@ -799,6 +799,16 @@ TEST(TestRGWAdmin, datalog_list) { } ss.str(""); + ss << "/admin/log?type=data&id=" << shard_id << "&start-time=" << start_time + << "&max-entries=1"; + rest_req = ss.str(); + g_test->send_request(string("GET"), rest_req); + EXPECT_EQ(200U, g_test->get_resp_code()); + entries.clear(); + get_datalog_list(entries); + EXPECT_EQ(1U, entries.size()); + + ss.str(""); ss << "/admin/log?type=data&id=" << shard_id << "&start-time=" << start_time << "&end-time=" << end_time; rest_req = ss.str(); @@ -1157,6 +1167,25 @@ TEST(TestRGWAdmin, mdlog_list) { EXPECT_EQ(get_log_list(entries), 0); EXPECT_EQ(entries.size(), 14U); + ss.str(""); + ss << "/admin/log?type=metadata&id=" << shard_id << "&start-time=" << start_time + << "&max-entries=" << 1; + rest_req = ss.str(); + g_test->send_request(string("GET"), rest_req); + EXPECT_EQ(200U, g_test->get_resp_code()); + entries.clear(); + EXPECT_EQ(get_log_list(entries), 0); + EXPECT_EQ(entries.size(), 1U); + + ss.str(""); + ss << "/admin/log?type=metadata&id=" << shard_id << "&start-time=" << start_time + << "&max-entries=" << 6; + rest_req = ss.str(); + g_test->send_request(string("GET"), rest_req); + EXPECT_EQ(200U, g_test->get_resp_code()); + entries.clear(); + EXPECT_EQ(get_log_list(entries), 0); + EXPECT_EQ(entries.size(), 6U); ASSERT_EQ(0, caps_rm(cname, perm)); ss.str(""); |