OSD: add config option for peering_wq batch size

Large peering_wq batch sizes may excessively delay
peering messages resulting in unreasonably long
peering.  This may speed up peering.

Backport: cuttlefish
Related: #5084
Signed-off-by: Samuel Just <sam.just@inktank.com>
Reviewed-by: Sage Weil <sage@inktank.com>
(cherry picked from commit 39e5a2a406b77fa82e9a78c267b679d49927e3c3)

3 files changed, 6 insertions, 5 deletions

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index c31282c1997..4fe4277a50d 100644
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -388,6 +388,7 @@ OPTION(osd_map_cache_size, OPT_INT, 500)
 OPTION(osd_map_message_max, OPT_INT, 100)  // max maps per MOSDMap message
 OPTION(osd_map_share_max_epochs, OPT_INT, 100)  // cap on # of inc maps we send to peers, clients
 OPTION(osd_op_threads, OPT_INT, 2)    // 0 == no threading
+OPTION(osd_peering_wq_batch_size, OPT_U64, 20)
 OPTION(osd_op_pq_max_tokens_per_priority, OPT_U64, 4194304)
 OPTION(osd_op_pq_min_cost, OPT_U64, 65536)
 OPTION(osd_disk_threads, OPT_INT, 1)
diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 825a2fea99f..6202bac3461 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -908,7 +908,7 @@ OSD::OSD(int id, Messenger *internal_messenger, Messenger *external_messenger,
   finished_lock("OSD::finished_lock"),
   test_ops_hook(NULL),
   op_wq(this, g_conf->osd_op_thread_timeout, &op_tp),
-  peering_wq(this, g_conf->osd_op_thread_timeout, &op_tp, 200),
+  peering_wq(this, g_conf->osd_op_thread_timeout, &op_tp),
   map_lock("OSD::map_lock"),
   peer_map_epoch_lock("OSD::peer_map_epoch_lock"),
   debug_drop_pg_create_probability(g_conf->osd_debug_drop_pg_create_probability),
diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index ac2c634c1f2..2db9d3b8c44 100644
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -798,10 +798,9 @@ private:
     list<PG*> peering_queue;
     OSD *osd;
     set<PG*> in_use;
-    const size_t batch_size;
-    PeeringWQ(OSD *o, time_t ti, ThreadPool *tp, size_t batch_size)
+    PeeringWQ(OSD *o, time_t ti, ThreadPool *tp)
       : ThreadPool::BatchWorkQueue<PG>(
-	"OSD::PeeringWQ", ti, ti*10, tp), osd(o), batch_size(batch_size) {}
+	"OSD::PeeringWQ", ti, ti*10, tp), osd(o) {}
 
     void _dequeue(PG *pg) {
       for (list<PG*>::iterator i = peering_queue.begin();
@@ -826,7 +825,8 @@ private:
     void _dequeue(list<PG*> *out) {
       set<PG*> got;
       for (list<PG*>::iterator i = peering_queue.begin();
-	   i != peering_queue.end() && out->size() < batch_size;
+	   i != peering_queue.end() &&
+	     out->size() < g_conf->osd_peering_wq_batch_size;
 	   ) {
 	if (in_use.count(*i)) {
 	  ++i;