dpif-netdev: Fix non-pmd thread queue id.

Non pmd threads have a core_id == UINT32_MAX, while queue ids used by
netdevs range from 0 to the number of CPUs.  Therefore core ids cannot
be used directly to select a queue.

This commit introduces a simple mapping to fix the problem: pmd threads
continue using queues 0 to N (where N is the number of CPUs in the
system), while non pmd threads use queue N+1.

Fixes: d5c199ea7ff7 ("netdev-dpdk: Properly support non pmd threads.")

Reported-by: 차은호 <eunho.cha@atto-research.com
Signed-off-by: Daniele Di Proietto <diproiettod@vmware.com>
Signed-off-by: Mark D. Gray <mark.d.gray@intel.com>
Signed-off-by: Ethan Jackson <ethan@nicira.com>
Acked-by: Flavio Leitner <fbl@redhat.com>
Acked-by: Ethan Jackson <ethan@nicira.com>

1 files changed, 20 insertions, 5 deletions

diff --git a/lib/dpif-netdev.c b/lib/dpif-netdev.c
index 76d100335..7df95239d 100644
--- a/lib/dpif-netdev.c
+++ b/lib/dpif-netdev.c
@@ -415,6 +415,8 @@ struct dp_netdev_pmd_thread {
                                     /* threads on same numa node. */
     unsigned core_id;               /* CPU core id of this pmd thread. */
     int numa_id;                    /* numa node id of this pmd thread. */
+    int tx_qid;                     /* Queue id used by this pmd thread to
+                                     * send packets on all netdevs */
 
     /* Only a pmd thread can write on its own 'cycles' and 'stats'.
      * The main thread keeps 'stats_zero' and 'cycles_zero' as base
@@ -1067,8 +1069,9 @@ do_add_port(struct dp_netdev *dp, const char *devname, const char *type,
             return ENOENT;
         }
         /* There can only be ovs_numa_get_n_cores() pmd threads,
-         * so creates a txq for each. */
-        error = netdev_set_multiq(netdev, n_cores, dp->n_dpdk_rxqs);
+         * so creates a txq for each, and one extra for the non
+         * pmd threads. */
+        error = netdev_set_multiq(netdev, n_cores + 1, dp->n_dpdk_rxqs);
         if (error && (error != EOPNOTSUPP)) {
             VLOG_ERR("%s, cannot set multiq", devname);
             return errno;
@@ -2402,7 +2405,8 @@ dpif_netdev_pmd_set(struct dpif *dpif, unsigned int n_rxqs, const char *cmask)
                 }
 
                 /* Sets the new rx queue config.  */
-                err = netdev_set_multiq(port->netdev, ovs_numa_get_n_cores(),
+                err = netdev_set_multiq(port->netdev,
+                                        ovs_numa_get_n_cores() + 1,
                                         n_rxqs);
                 if (err && (err != EOPNOTSUPP)) {
                     VLOG_ERR("Failed to set dpdk interface %s rx_queue to:"
@@ -2806,6 +2810,16 @@ dp_netdev_pmd_get_next(struct dp_netdev *dp, struct cmap_position *pos)
     return next;
 }
 
+static int
+core_id_to_qid(unsigned core_id)
+{
+    if (core_id != NON_PMD_CORE_ID) {
+        return core_id;
+    } else {
+        return ovs_numa_get_n_cores();
+    }
+}
+
 /* Configures the 'pmd' based on the input argument. */
 static void
 dp_netdev_configure_pmd(struct dp_netdev_pmd_thread *pmd, struct dp_netdev *dp,
@@ -2814,6 +2828,7 @@ dp_netdev_configure_pmd(struct dp_netdev_pmd_thread *pmd, struct dp_netdev *dp,
     pmd->dp = dp;
     pmd->index = index;
     pmd->core_id = core_id;
+    pmd->tx_qid = core_id_to_qid(core_id);
     pmd->numa_id = numa_id;
 
     ovs_refcount_init(&pmd->ref_cnt);
@@ -3329,7 +3344,7 @@ dpif_netdev_register_upcall_cb(struct dpif *dpif, upcall_callback *cb,
 }
 
 static void
-dp_netdev_drop_packets(struct dp_packet ** packets, int cnt, bool may_steal)
+dp_netdev_drop_packets(struct dp_packet **packets, int cnt, bool may_steal)
 {
     if (may_steal) {
         int i;
@@ -3387,7 +3402,7 @@ dp_execute_cb(void *aux_, struct dp_packet **packets, int cnt,
     case OVS_ACTION_ATTR_OUTPUT:
         p = dp_netdev_lookup_port(dp, u32_to_odp(nl_attr_get_u32(a)));
         if (OVS_LIKELY(p)) {
-            netdev_send(p->netdev, pmd->core_id, packets, cnt, may_steal);
+            netdev_send(p->netdev, pmd->tx_qid, packets, cnt, may_steal);
             return;
         }
         break;