From 488a20de84402a4570bcc3081a99fa8c3d155468 Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Thu, 14 Feb 2013 13:20:08 -0800 Subject: rgw: region creation Signed-off-by: Yehuda Sadeh --- src/common/config_opts.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'src/common/config_opts.h') diff --git a/src/common/config_opts.h b/src/common/config_opts.h index 02ceda5703e..1607c33066f 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -567,7 +567,11 @@ OPTION(rgw_op_thread_suicide_timeout, OPT_INT, 0) OPTION(rgw_thread_pool_size, OPT_INT, 100) OPTION(rgw_num_control_oids, OPT_INT, 8) -OPTION(rgw_zone_root_pool, OPT_STR, ".rgw.root") +OPTION(rgw_zone, OPT_STR, "") // zone name +OPTION(rgw_zone_root_pool, OPT_STR, ".rgw.root") // pool where zone specific info is stored +OPTION(rgw_region, OPT_STR, "") // region name +OPTION(rgw_region_root_pool, OPT_STR, ".rgw.root") // pool where all region info is stored +OPTION(rgw_default_region_info_oid, OPT_STR, "default.region") // oid where default region info is stored OPTION(rgw_log_nonexistent_bucket, OPT_BOOL, false) OPTION(rgw_log_object_name, OPT_STR, "%Y-%m-%d-%H-%i-%n") // man date to see codes (a subset are supported) OPTION(rgw_log_object_name_utc, OPT_BOOL, false) -- cgit v1.2.1 From cb6d4de4940cdcfa0dc270ffd0c3c21b9f9aea0a Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Tue, 19 Mar 2013 14:52:47 -0700 Subject: rgw: log user operations Signed-off-by: Yehuda Sadeh --- src/common/config_opts.h | 1 + 1 file changed, 1 insertion(+) (limited to 'src/common/config_opts.h') diff --git a/src/common/config_opts.h b/src/common/config_opts.h index 1607c33066f..5160a611402 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -601,6 +601,7 @@ OPTION(rgw_get_obj_window_size, OPT_INT, 16 << 20) // window size in bytes for s OPTION(rgw_get_obj_max_req_size, OPT_INT, 4 << 20) // max length of a single get obj rados op OPTION(rgw_relaxed_s3_bucket_names, OPT_BOOL, false) // enable relaxed bucket name rules for US region buckets OPTION(rgw_list_buckets_max_chunk, OPT_INT, 1000) // max buckets to retrieve in a single op when listing user buckets +OPTION(rgw_md_log_max_shards, OPT_INT, 64) // max shards for metadata log OPTION(mutex_perf_counter, OPT_BOOL, false) // enable/disable mutex perf counter -- cgit v1.2.1 From 3385c2c5be176e65e3d4a74fca31075890659909 Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Mon, 6 May 2013 13:41:29 -0700 Subject: rgw: more data changes log implementation Remove a bunch of hard coded stuff. Make renew thread updates expiration. Only renew if there was more than one request through the current window. Signed-off-by: Yehuda Sadeh --- src/common/config_opts.h | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'src/common/config_opts.h') diff --git a/src/common/config_opts.h b/src/common/config_opts.h index 5160a611402..ed52297bd78 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -603,6 +603,11 @@ OPTION(rgw_relaxed_s3_bucket_names, OPT_BOOL, false) // enable relaxed bucket na OPTION(rgw_list_buckets_max_chunk, OPT_INT, 1000) // max buckets to retrieve in a single op when listing user buckets OPTION(rgw_md_log_max_shards, OPT_INT, 64) // max shards for metadata log +OPTION(rgw_data_log_window, OPT_INT, 30) // data log entries window (in seconds) +OPTION(rgw_data_log_changes_size, OPT_INT, 1000) // number of in-memory entries to hold for data changes log +OPTION(rgw_data_log_num_shards, OPT_INT, 128) // number of objects to keep data changes log on +OPTION(rgw_data_log_obj_prefix, OPT_STR, "data_log") // + OPTION(mutex_perf_counter, OPT_BOOL, false) // enable/disable mutex perf counter // This will be set to true when it is safe to start threads. -- cgit v1.2.1 From d628109fded061eccbffa5d8d15cb5c2d8bc741e Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Tue, 18 Jun 2013 20:45:42 -0700 Subject: rgw: specialized obj zone copy state for statelog Signed-off-by: Yehuda Sadeh --- src/common/config_opts.h | 1 + 1 file changed, 1 insertion(+) (limited to 'src/common/config_opts.h') diff --git a/src/common/config_opts.h b/src/common/config_opts.h index ed52297bd78..d95724fce8f 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -602,6 +602,7 @@ OPTION(rgw_get_obj_max_req_size, OPT_INT, 4 << 20) // max length of a single get OPTION(rgw_relaxed_s3_bucket_names, OPT_BOOL, false) // enable relaxed bucket name rules for US region buckets OPTION(rgw_list_buckets_max_chunk, OPT_INT, 1000) // max buckets to retrieve in a single op when listing user buckets OPTION(rgw_md_log_max_shards, OPT_INT, 64) // max shards for metadata log +OPTION(rgw_num_zone_copy_state_shards, OPT_INT, 128) // max shards for keeping inter-region copy progress info OPTION(rgw_data_log_window, OPT_INT, 30) // data log entries window (in seconds) OPTION(rgw_data_log_changes_size, OPT_INT, 1000) // number of in-memory entries to hold for data changes log -- cgit v1.2.1 From 8b1524be1f403f092af1c0b57e174179336ef6ef Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Tue, 18 Jun 2013 23:02:05 -0700 Subject: rgw: OpState internal api Add new higher level functions to set, renew, list, and remove logged operation state. Signed-off-by: Yehuda Sadeh --- src/common/config_opts.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/common/config_opts.h') diff --git a/src/common/config_opts.h b/src/common/config_opts.h index d95724fce8f..b48cfd5db1e 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -602,7 +602,7 @@ OPTION(rgw_get_obj_max_req_size, OPT_INT, 4 << 20) // max length of a single get OPTION(rgw_relaxed_s3_bucket_names, OPT_BOOL, false) // enable relaxed bucket name rules for US region buckets OPTION(rgw_list_buckets_max_chunk, OPT_INT, 1000) // max buckets to retrieve in a single op when listing user buckets OPTION(rgw_md_log_max_shards, OPT_INT, 64) // max shards for metadata log -OPTION(rgw_num_zone_copy_state_shards, OPT_INT, 128) // max shards for keeping inter-region copy progress info +OPTION(rgw_num_zone_opstate_shards, OPT_INT, 128) // max shards for keeping inter-region copy progress info OPTION(rgw_data_log_window, OPT_INT, 30) // data log entries window (in seconds) OPTION(rgw_data_log_changes_size, OPT_INT, 1000) // number of in-memory entries to hold for data changes log -- cgit v1.2.1 From 02de43ad731137179e750a318edf857eb67941e8 Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Thu, 20 Jun 2013 17:01:25 -0700 Subject: rgw: tie opstate into intra-region copy operations Signed-off-by: Yehuda Sadeh --- src/common/config_opts.h | 1 + 1 file changed, 1 insertion(+) (limited to 'src/common/config_opts.h') diff --git a/src/common/config_opts.h b/src/common/config_opts.h index b48cfd5db1e..66582c0407e 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -603,6 +603,7 @@ OPTION(rgw_relaxed_s3_bucket_names, OPT_BOOL, false) // enable relaxed bucket na OPTION(rgw_list_buckets_max_chunk, OPT_INT, 1000) // max buckets to retrieve in a single op when listing user buckets OPTION(rgw_md_log_max_shards, OPT_INT, 64) // max shards for metadata log OPTION(rgw_num_zone_opstate_shards, OPT_INT, 128) // max shards for keeping inter-region copy progress info +OPTION(rgw_opstate_ratelimit_sec, OPT_INT, 30) // min time between opstate updates on a single upload (0 for disabling ratelimit) OPTION(rgw_data_log_window, OPT_INT, 30) // data log entries window (in seconds) OPTION(rgw_data_log_changes_size, OPT_INT, 1000) // number of in-memory entries to hold for data changes log -- cgit v1.2.1 From 241ad07394f40eb39646e270a9f534a4bb031666 Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Fri, 21 Jun 2013 11:53:49 -0700 Subject: rgw: make replica log object name configurable Signed-off-by: Yehuda Sadeh --- src/common/config_opts.h | 1 + 1 file changed, 1 insertion(+) (limited to 'src/common/config_opts.h') diff --git a/src/common/config_opts.h b/src/common/config_opts.h index 66582c0407e..91f8167a407 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -609,6 +609,7 @@ OPTION(rgw_data_log_window, OPT_INT, 30) // data log entries window (in seconds) OPTION(rgw_data_log_changes_size, OPT_INT, 1000) // number of in-memory entries to hold for data changes log OPTION(rgw_data_log_num_shards, OPT_INT, 128) // number of objects to keep data changes log on OPTION(rgw_data_log_obj_prefix, OPT_STR, "data_log") // +OPTION(rgw_replica_log_obj_prefix, OPT_STR, "replica_log") // OPTION(mutex_perf_counter, OPT_BOOL, false) // enable/disable mutex perf counter -- cgit v1.2.1 From 395262cd82f6fe9a7db7461be8a644fd8fc84e62 Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Mon, 8 Jul 2013 18:55:19 -0700 Subject: rgw: call appropriate curl calls for waiting on sockets If libcurl supports curl_multi_wait() then use it, otherwise use select() and force a timeout, even if it has been disabled. Otherwise we may wait forever for events that we can't wait for as select() only uses fds < 1024. Signed-off-by: Yehuda Sadeh --- src/common/config_opts.h | 1 + 1 file changed, 1 insertion(+) (limited to 'src/common/config_opts.h') diff --git a/src/common/config_opts.h b/src/common/config_opts.h index f8e888b56b0..45f0bb4f56a 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -670,6 +670,7 @@ OPTION(rgw_list_buckets_max_chunk, OPT_INT, 1000) // max buckets to retrieve in OPTION(rgw_md_log_max_shards, OPT_INT, 64) // max shards for metadata log OPTION(rgw_num_zone_opstate_shards, OPT_INT, 128) // max shards for keeping inter-region copy progress info OPTION(rgw_opstate_ratelimit_sec, OPT_INT, 30) // min time between opstate updates on a single upload (0 for disabling ratelimit) +OPTION(rgw_curl_wait_timeout_ms, OPT_INT, 1000) // timeout for certain curl calls OPTION(rgw_data_log_window, OPT_INT, 30) // data log entries window (in seconds) OPTION(rgw_data_log_changes_size, OPT_INT, 1000) // number of in-memory entries to hold for data changes log -- cgit v1.2.1