vswitchd: Make packet-in controller queue size configurable

The ofconn packet-in queue for packets that can't be immediately sent
on the rconn connection was limited to 100 packets (hardcoded value).
While increasing this limit is usually not recommended as it might
create buffer bloat and increase latency, in scaled scenarios it is
useful if the administrator (or CMS) can adjust the queue size.

One such situation was noticed while performing scale testing of the
OVN IGMP functionality: triggering ~200 simultaneous IGMP reports
was causing tail drops on the packet-in queue towards ovn-controller.

This commit adds the possibility to configure the queue size for:
- management controller (br-int.mgmt): through the
  other_config:controller-queue-size column of the Bridge table. This
  value is limited to 512 as large queues definitely affect latency. If
  not present the default value of 100 is used. This is done in order to
  maintain the same default behavior as before the commit.
- other controllers: through the controller_queue_size column of the
  Controller table. This value is also limited to 512. If not present
  the code uses the Bridge:other_config:controller-queue-size
  configuration.

Acked-by: Mark Michelson <mmichels@redhat.com>
Signed-off-by: Dumitru Ceara <dceara@redhat.com>
Signed-off-by: Ben Pfaff <blp@ovn.org>

1 files changed, 26 insertions, 0 deletions

diff --git a/vswitchd/bridge.c b/vswitchd/bridge.c
index 8c390382f..8ed51c961 100644
--- a/vswitchd/bridge.c
+++ b/vswitchd/bridge.c
@@ -227,6 +227,11 @@ static struct if_notifier *ifnotifier;
 static struct seq *ifaces_changed;
 static uint64_t last_ifaces_changed;
 
+/* Default/min/max packet-in queue sizes towards the controllers. */
+#define BRIDGE_CONTROLLER_PACKET_QUEUE_DEFAULT_SIZE 100
+#define BRIDGE_CONTROLLER_PACKET_QUEUE_MIN_SIZE 1
+#define BRIDGE_CONTROLLER_PACKET_QUEUE_MAX_SIZE 512
+
 static void add_del_bridges(const struct ovsrec_open_vswitch *);
 static void bridge_run__(void);
 static void bridge_create(const struct ovsrec_bridge *);
@@ -1123,6 +1128,25 @@ bridge_get_allowed_versions(struct bridge *br)
                                          br->cfg->n_protocols);
 }
 
+static int
+bridge_get_controller_queue_size(struct bridge *br,
+                                 struct ovsrec_controller *c)
+{
+    if (c && c->controller_queue_size) {
+        return *c->controller_queue_size;
+    }
+
+    int queue_size = smap_get_int(&br->cfg->other_config,
+                                  "controller-queue-size",
+                                  BRIDGE_CONTROLLER_PACKET_QUEUE_DEFAULT_SIZE);
+    if (queue_size < BRIDGE_CONTROLLER_PACKET_QUEUE_MIN_SIZE ||
+            queue_size > BRIDGE_CONTROLLER_PACKET_QUEUE_MAX_SIZE) {
+        return BRIDGE_CONTROLLER_PACKET_QUEUE_DEFAULT_SIZE;
+    }
+
+    return queue_size;
+}
+
 /* Set NetFlow configuration on 'br'. */
 static void
 bridge_configure_netflow(struct bridge *br)
@@ -3616,6 +3640,7 @@ bridge_configure_remotes(struct bridge *br,
         .band = OFPROTO_OUT_OF_BAND,
         .enable_async_msgs = true,
         .allowed_versions = bridge_get_allowed_versions(br),
+        .max_pktq_size = bridge_get_controller_queue_size(br, NULL),
     };
     shash_add_nocopy(
         &ocs, xasprintf("punix:%s/%s.mgmt", ovs_rundir(), br->name), oc);
@@ -3691,6 +3716,7 @@ bridge_configure_remotes(struct bridge *br,
             .enable_async_msgs = (!c->enable_async_messages
                                   || *c->enable_async_messages),
             .allowed_versions = bridge_get_allowed_versions(br),
+            .max_pktq_size = bridge_get_controller_queue_size(br, c),
             .rate_limit = (c->controller_rate_limit
                            ? *c->controller_rate_limit : 0),
             .burst_limit = (c->controller_burst_limit