dpdk: Fix device cleanup.

Commit 5dcde09c80a8 was introduced to make detaching more
automatic without using an additional command beyond
ovs-vsctl del-port <br> <port>.

Sometimes, since commit 5dcde09c80a8, dpdk devices are
not detached when del-port is issued; command example:

sudo ovs-vsctl del-port br0 dpdk1

This can happen when vswitchd is (re)started with an existing
database and devices are already bound to dpdk.

A minimal recipe to reproduce the issue is:

1/ Starting with

darrell@prmh-nsx-perf-server125:~$ sudo ovs-vsctl show
1c50d8ee-b17f-4fac-a595-03b0da8c8275
    Bridge "br0"
        Port "br0"
            Interface "br0"
                type: internal
        Port "dpdk1"
            Interface "dpdk1"
                type: dpdk
                options: {dpdk-devargs="0000:04:00.1"}
        Port "dpdk0"
            Interface "dpdk0"
                type: dpdk
                options: {dpdk-devargs="0000:04:00.0"}

darrell@prmh-nsx-perf-server125:~$ /usr/src/dpdk-16.11/tools/dpdk-devbind.py --status

Network devices using DPDK-compatible driver

============================================
0000:04:00.0 'Ethernet Controller 10-Gigabit X540-AT2' drv=uio_pci_generic unused=ixgbe,vfio-pci
0000:04:00.1 'Ethernet Controller 10-Gigabit X540-AT2' drv=uio_pci_generic unused=ixgbe,vfio-pci

2/ restart vswitchd

3/ run
 sudo ovs-vsctl del-port br0 dpdk1

and find the interface is NOT detached; there is
no info log ‘Device '0000:04:00.1' detached’.

A more verbose discussion is here:
https://mail.openvswitch.org/pipermail/ovs-dev/2017-June/333462.html
along with another possible solution.

Since we are nearing the end of a release, a safe approach is needed,
at this time.
One approach is to revert 5dcde09c80a8.  This patch does not do that
but reinstates the command ovs-appctl netdev-dpdk/detach to handle
cases when del-port will not work.

To detach the device, run the reinstated command
ovs-appctl netdev-dpdk/detach 0000:04:00.1
Observe console output
‘Device '0000:04:00.1' has been detached’

Fixes: 5dcde09c80a8 ("netdev-dpdk: Fix device leak on port deletion.")
CC: Ilya Maximets <i.maximets@samsung.com>
Acked-by: Aaron Conole <aconole@redhat.com>
Acked-by: Fischetti, Antonio <antonio.fischetti@intel.com>
Signed-off-by: Darrell Ball <dlu998@gmail.com>
Signed-off-by: Ben Pfaff <blp@ovn.org>

1 files changed, 52 insertions, 1 deletions

diff --git a/lib/netdev-dpdk.c b/lib/netdev-dpdk.c
index 633ab1676..1d82bca50 100644
--- a/lib/netdev-dpdk.c
+++ b/lib/netdev-dpdk.c
@@ -1075,7 +1075,7 @@ netdev_dpdk_destruct(struct netdev *netdev)
         if (rte_eth_dev_detach(dev->port_id, devname) < 0) {
             VLOG_ERR("Device '%s' can not be detached", dev->devargs);
         } else {
-            VLOG_INFO("Device '%s' detached", devname);
+            VLOG_INFO("Device '%s' has been detached", devname);
         }
     }
 
@@ -2511,6 +2511,53 @@ netdev_dpdk_set_admin_state(struct unixctl_conn *conn, int argc,
     unixctl_command_reply(conn, "OK");
 }
 
+static void
+netdev_dpdk_detach(struct unixctl_conn *conn, int argc OVS_UNUSED,
+                   const char *argv[], void *aux OVS_UNUSED)
+{
+    int ret;
+    char *response;
+    uint8_t port_id;
+    char devname[RTE_ETH_NAME_MAX_LEN];
+    struct netdev_dpdk *dev;
+
+    ovs_mutex_lock(&dpdk_mutex);
+
+    if (!rte_eth_dev_count() || rte_eth_dev_get_port_by_name(argv[1],
+                                                             &port_id)) {
+        response = xasprintf("Device '%s' not found in DPDK", argv[1]);
+        goto error;
+    }
+
+    dev = netdev_dpdk_lookup_by_port_id(port_id);
+    if (dev) {
+        response = xasprintf("Device '%s' is being used by interface '%s'. "
+                             "Remove it before detaching",
+                             argv[1], netdev_get_name(&dev->up));
+        goto error;
+    }
+
+    rte_eth_dev_close(port_id);
+
+    ret = rte_eth_dev_detach(port_id, devname);
+    if (ret < 0) {
+        response = xasprintf("Device '%s' can not be detached", argv[1]);
+        goto error;
+    }
+
+    response = xasprintf("Device '%s' has been detached", argv[1]);
+
+    ovs_mutex_unlock(&dpdk_mutex);
+    unixctl_command_reply(conn, response);
+    free(response);
+    return;
+
+error:
+    ovs_mutex_unlock(&dpdk_mutex);
+    unixctl_command_reply_error(conn, response);
+    free(response);
+}
+
 /*
  * Set virtqueue flags so that we do not receive interrupts.
  */
@@ -2763,6 +2810,10 @@ netdev_dpdk_class_init(void)
                                  "[netdev] up|down", 1, 2,
                                  netdev_dpdk_set_admin_state, NULL);
 
+        unixctl_command_register("netdev-dpdk/detach",
+                                 "pci address of device", 1, 1,
+                                 netdev_dpdk_detach, NULL);
+
         ovsthread_once_done(&once);
     }