netdev: Add function for configuring tx and rx queues.

This commit adds a new API to the 'struct netdev_class' which
allows user to configure the number of tx queues and rx queues
of 'netdev'.  Upcoming patches will use this function to set
multiple tx/rx queues when adding the netdev to dpif-netdev.

Currently, only netdev-dpdk module implements this function.

Signed-off-by: Alex Wang <alexw@nicira.com>
Acked-by: Pravin B Shelar <pshelar@nicira.com>

8 files changed, 83 insertions, 9 deletions

diff --git a/lib/netdev-bsd.c b/lib/netdev-bsd.c
index 5b012996a..1bd91082a 100644
--- a/lib/netdev-bsd.c
+++ b/lib/netdev-bsd.c
@@ -1563,6 +1563,7 @@ netdev_bsd_update_flags(struct netdev *netdev_, enum netdev_flags off,
     NULL, /* set_config */                           \
     NULL, /* get_tunnel_config */                    \
     NULL, /* get_numa_id */                          \
+    NULL, /* set_multiq */                           \
                                                      \
     netdev_bsd_send,                                 \
     netdev_bsd_send_wait,                            \
diff --git a/lib/netdev-dpdk.c b/lib/netdev-dpdk.c
index 8f1fdb5b6..1c1c6e6f7 100644
--- a/lib/netdev-dpdk.c
+++ b/lib/netdev-dpdk.c
@@ -398,13 +398,14 @@ dpdk_eth_dev_init(struct netdev_dpdk *dev) OVS_REQUIRES(dpdk_mutex)
         return ENODEV;
     }
 
-    diag = rte_eth_dev_configure(dev->port_id, NR_QUEUE, NR_QUEUE,  &port_conf);
+    diag = rte_eth_dev_configure(dev->port_id, dev->up.n_rxq, dev->up.n_txq,
+                                 &port_conf);
     if (diag) {
         VLOG_ERR("eth dev config error %d",diag);
         return -diag;
     }
 
-    for (i = 0; i < NR_QUEUE; i++) {
+    for (i = 0; i < dev->up.n_txq; i++) {
         diag = rte_eth_tx_queue_setup(dev->port_id, i, NIC_PORT_TX_Q_SIZE,
                                       dev->socket_id, &tx_conf);
         if (diag) {
@@ -413,7 +414,7 @@ dpdk_eth_dev_init(struct netdev_dpdk *dev) OVS_REQUIRES(dpdk_mutex)
         }
     }
 
-    for (i = 0; i < NR_QUEUE; i++) {
+    for (i = 0; i < dev->up.n_rxq; i++) {
         diag = rte_eth_rx_queue_setup(dev->port_id, i, NIC_PORT_RX_Q_SIZE,
                                       dev->socket_id,
                                       &rx_conf, dev->dpdk_mp->mp);
@@ -490,12 +491,12 @@ netdev_dpdk_init(struct netdev *netdev_, unsigned int port_no) OVS_REQUIRES(dpdk
         goto unlock;
     }
 
+    netdev_->n_txq = NR_QUEUE;
+    netdev_->n_rxq = NR_QUEUE;
     err = dpdk_eth_dev_init(netdev);
     if (err) {
         goto unlock;
     }
-    netdev_->n_txq = NR_QUEUE;
-    netdev_->n_rxq = NR_QUEUE;
 
     list_push_back(&dpdk_list, &netdev->list_node);
 
@@ -589,6 +590,30 @@ netdev_dpdk_get_numa_id(const struct netdev *netdev_)
     return netdev->socket_id;
 }
 
+/* Sets the number of tx queues and rx queues for the dpdk interface.
+ * If the configuration fails, do not try restoring its old configuration
+ * and just returns the error. */
+static int
+netdev_dpdk_set_multiq(struct netdev *netdev_, unsigned int n_txq,
+                       unsigned int n_rxq)
+{
+    struct netdev_dpdk *netdev = netdev_dpdk_cast(netdev_);
+    int err = 0;
+
+    if (netdev->up.n_txq == n_txq && netdev->up.n_rxq == n_rxq) {
+        return err;
+    }
+
+    ovs_mutex_lock(&netdev->mutex);
+    rte_eth_dev_stop(netdev->port_id);
+    netdev->up.n_txq = n_txq;
+    netdev->up.n_rxq = n_rxq;
+    err = dpdk_eth_dev_init(netdev);
+    ovs_mutex_unlock(&netdev->mutex);
+
+    return err;
+}
+
 static struct netdev_rxq *
 netdev_dpdk_rxq_alloc(void)
 {
@@ -686,7 +711,11 @@ netdev_dpdk_rxq_recv(struct netdev_rxq *rxq_, struct dpif_packet **packets,
     struct netdev_dpdk *dev = netdev_dpdk_cast(netdev);
     int nb_rx;
 
-    dpdk_queue_flush(dev, rxq_->queue_id);
+    /* There is only one tx queue for this core.  Do not flush other
+     * queueus. */
+    if (rxq_->queue_id == rte_lcore_id()) {
+        dpdk_queue_flush(dev, rxq_->queue_id);
+    }
 
     nb_rx = rte_eth_rx_burst(rx->port_id, rxq_->queue_id,
                              (struct rte_mbuf **) packets,
@@ -1326,7 +1355,7 @@ unlock_dpdk:
     return err;
 }
 
-#define NETDEV_DPDK_CLASS(NAME, INIT, CONSTRUCT)              \
+#define NETDEV_DPDK_CLASS(NAME, INIT, CONSTRUCT, MULTIQ)      \
 {                                                             \
     NAME,                                                     \
     INIT,                       /* init */                    \
@@ -1341,6 +1370,7 @@ unlock_dpdk:
     NULL,                       /* netdev_dpdk_set_config */  \
     NULL,                       /* get_tunnel_config */       \
     netdev_dpdk_get_numa_id,    /* get_numa_id */             \
+    MULTIQ,                     /* set_multiq */              \
                                                               \
     netdev_dpdk_send,           /* send */                    \
     NULL,                       /* send_wait */               \
@@ -1427,13 +1457,15 @@ const struct netdev_class dpdk_class =
     NETDEV_DPDK_CLASS(
         "dpdk",
         dpdk_class_init,
-        netdev_dpdk_construct);
+        netdev_dpdk_construct,
+        netdev_dpdk_set_multiq);
 
 const struct netdev_class dpdk_ring_class =
     NETDEV_DPDK_CLASS(
         "dpdkr",
         NULL,
-        netdev_dpdk_ring_construct);
+        netdev_dpdk_ring_construct,
+        NULL);
 
 void
 netdev_dpdk_register(void)
diff --git a/lib/netdev-dummy.c b/lib/netdev-dummy.c
index ca048121d..a2b1f2c0f 100644
--- a/lib/netdev-dummy.c
+++ b/lib/netdev-dummy.c
@@ -1033,6 +1033,7 @@ static const struct netdev_class dummy_class = {
     netdev_dummy_set_config,
     NULL,                       /* get_tunnel_config */
     NULL,                       /* get_numa_id */
+    NULL,                       /* set_multiq */
 
     netdev_dummy_send,          /* send */
     NULL,                       /* send_wait */
diff --git a/lib/netdev-linux.c b/lib/netdev-linux.c
index 084af4e09..6b6f9b0b3 100644
--- a/lib/netdev-linux.c
+++ b/lib/netdev-linux.c
@@ -2716,6 +2716,7 @@ netdev_linux_update_flags(struct netdev *netdev_, enum netdev_flags off,
     NULL,                       /* set_config */                \
     NULL,                       /* get_tunnel_config */         \
     NULL,                       /* get_numa_id */               \
+    NULL,                       /* set_multiq */                \
                                                                 \
     netdev_linux_send,                                          \
     netdev_linux_send_wait,                                     \
diff --git a/lib/netdev-provider.h b/lib/netdev-provider.h
index 7f266fdc5..0e4cda5b5 100644
--- a/lib/netdev-provider.h
+++ b/lib/netdev-provider.h
@@ -258,6 +258,16 @@ struct netdev_class {
      * such info, returns NETDEV_NUMA_UNSPEC. */
     int (*get_numa_id)(const struct netdev *netdev);
 
+    /* Configures the number of tx queues and rx queues of 'netdev'.
+     * Return 0 if successful, otherwise a positive errno value.
+     *
+     * On error, the tx queue and rx queue configuration is indeterminant.
+     * Caller should make decision on whether to restore the previous or
+     * the default configuration.  Also, caller must make sure there is no
+     * other thread accessing the queues at the same time. */
+    int (*set_multiq)(struct netdev *netdev, unsigned int n_txq,
+                      unsigned int n_rxq);
+
     /* Sends buffers on 'netdev'.
      * Returns 0 if successful (for every buffer), otherwise a positive errno
      * value.  Returns EAGAIN without blocking if one or more packets cannot be
diff --git a/lib/netdev-vport.c b/lib/netdev-vport.c
index c258f6bac..83f129624 100644
--- a/lib/netdev-vport.c
+++ b/lib/netdev-vport.c
@@ -776,6 +776,7 @@ get_stats(const struct netdev *netdev, struct netdev_stats *stats)
     SET_CONFIG,                                             \
     GET_TUNNEL_CONFIG,                                      \
     NULL,                       /* get_numa_id */           \
+    NULL,                       /* set_multiq */            \
                                                             \
     NULL,                       /* send */                  \
     NULL,                       /* send_wait */             \
diff --git a/lib/netdev.c b/lib/netdev.c
index fb176260a..1fd5121fc 100644
--- a/lib/netdev.c
+++ b/lib/netdev.c
@@ -663,6 +663,33 @@ netdev_rxq_drain(struct netdev_rxq *rx)
             : 0);
 }
 
+/* Configures the number of tx queues and rx queues of 'netdev'.
+ * Return 0 if successful, otherwise a positive errno value.
+ *
+ * On error, the tx queue and rx queue configuration is indeterminant.
+ * Caller should make decision on whether to restore the previous or
+ * the default configuration.  Also, caller must make sure there is no
+ * other thread accessing the queues at the same time. */
+int
+netdev_set_multiq(struct netdev *netdev, unsigned int n_txq,
+                  unsigned int n_rxq)
+{
+    int error;
+
+    error = (netdev->netdev_class->set_multiq
+             ? netdev->netdev_class->set_multiq(netdev,
+                                                MAX(n_txq, 1),
+                                                MAX(n_rxq, 1))
+             : EOPNOTSUPP);
+
+    if (error != EOPNOTSUPP) {
+        VLOG_DBG_RL(&rl, "failed to set tx/rx queue for network device %s:"
+                    "%s", netdev_get_name(netdev), ovs_strerror(error));
+    }
+
+    return error;
+}
+
 /* Sends 'buffers' on 'netdev'.  Returns 0 if successful (for every packet),
  * otherwise a positive errno value.  Returns EAGAIN without blocking if
  * at least one the packets cannot be queued immediately.  Returns EMSGSIZE
diff --git a/lib/netdev.h b/lib/netdev.h
index 1f91d9ae5..fc4180a74 100644
--- a/lib/netdev.h
+++ b/lib/netdev.h
@@ -162,6 +162,7 @@ const char *netdev_get_type_from_name(const char *);
 int netdev_get_mtu(const struct netdev *, int *mtup);
 int netdev_set_mtu(const struct netdev *, int mtu);
 int netdev_get_ifindex(const struct netdev *);
+int netdev_set_multiq(struct netdev *, unsigned int n_txq, unsigned int n_rxq);
 
 /* Packet reception. */
 int netdev_rxq_open(struct netdev *, struct netdev_rxq **, int id);