datapath: Prevent panic

On RHEL 7.x kernels we observe a panic induced by a paging error
when the timer kicks off a job that subsequently accesses memory
that belonged to the openvswitch kernel module but was since
unloaded - thus the paging error.

The panic can be induced on any RHEL 7.x kernel with the following test:

while `true`
do
    make check-kmod TESTSUITEFLAGS="-k \!gre"
done

On the systems I've been testing on it generally takes anywhere from a
minute to 15 minutes or so to repro but never longer than that.  Similar
results have been seen by other testers.

This patch does not fix the underlying bug, which does need to be
investigated and fixed, but it does prevent it from occurring. We
would like to prevent customer systems from panicking while we do
futher investigation to find the root cause.

Here is the trace:
[252257.801809] BUG: unable to handle kernel paging request at
ffffffffc07c6298
[252257.802451] IP: [<ffffffff810996e0>] run_timer_softirq+0xe0/0x310
[252257.803055] PGD 19f5067 PUD 19f7067 PMD 2fb5fc2067 PTE 0
[252257.803559] Oops: 0002 [#1] SMP
[252257.804138] Modules linked in: geneve ip6_udp_tunnel xt_statistic
xt_physdev xt_nat xt_recent xt_comment xt_mark ipt_MASQUERADE
nf_nat_masquerade_ipv4 iptable_nat xt_addrtype ipt_REJECT nf_reject_ipv4
xt_conntrack iptable_filter ip_tables nf_conntrack_netlink br_netfilter
overlay(T) sch_htb veth udp_tunnel 8021q garp mrp tun ip_set nfnetlink
bridge stp llc nf_conntrack_ipv6 nf_defrag_ipv6 nf_nat_ipv6
nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 nf_nat nf_conntrack
iTCO_wdt iTCO_vendor_support dcdbas mxm_wmi sb_edac edac_core
intel_powerclamp coretemp intel_rapl iosf_mbi kvm_intel kvm irqbypass
crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper
ablk_helper cryptd joydev mei_me sg mei ipmi_ssif pcspkr shpchp lpc_ich
ipmi_si ipmi_devintf ipmi_msghandler acpi_power_meter wmi nfsd
auth_rpcgss
[252257.808079] nfs_acl lockd grace sunrpc xfs libcrc32c sr_mod sd_mod
cdrom crc_t10dif crct10dif_generic uas usb_storage mgag200
drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops ttm drm
crct10dif_pclmul crct10dif_common crc32c_intel ahci libahci ixgbe libata
igb megaraid_sas mdio ptp i2c_algo_bit pps_core i2c_core dca dm_mirror
dm_region_hash dm_log dm_mod [last unloaded: openvswitch]
[252257.811056] CPU: 33 PID: 0 Comm: swapper/33 Tainted: G OE
------------ T 3.10.0-693.el7.x86_64 #1
[252257.811826] Hardware name: Dell Inc. PowerEdge R630/02C2CP, BIOS
2.1.5 04/11/2016
[252257.812605] task: ffff8830b7708fd0 ti: ffff8830b7718000 task.ti:
ffff8830b7718000
[252257.813447] RIP: 0010:[<ffffffff810996e0>] [<ffffffff810996e0>]
run_timer_softirq+0xe0/0x310
[252257.814298] RSP: 0018:ffff885fbe203e68 EFLAGS: 00010082
[252257.827524] Call Trace:
[252257.828410] <IRQ>
[252257.828417]
[252257.829292] [<ffffffff8132bfe0>] ? timerqueue_add+0x60/0xb0
[252257.830164] [<ffffffff81090b3f>] __do_softirq+0xef/0x280
[252257.831010] [<ffffffff816b6a5c>] call_softirq+0x1c/0x30
[252257.831849] [<ffffffff8102d3c5>] do_softirq+0x65/0xa0
[252257.832669] [<ffffffff81090ec5>] irq_exit+0x105/0x110
[252257.833501] [<ffffffff816b76c2>] smp_apic_timer_interrupt+0x42/0x50
[252257.834330] [<ffffffff816b5c1d>] apic_timer_interrupt+0x6d/0x80
[252257.835152] <EOI>
[252257.835159]
[252257.835944] [<ffffffff81527a02>] ? cpuidle_enter_state+0x52/0xc0
[252257.837231] [<ffffffff81527b48>] cpuidle_idle_call+0xd8/0x210
[252257.838224] [<ffffffff81034fee>] arch_cpu_idle+0xe/0x30
[252257.839133] [<ffffffff810e7bca>] cpu_startup_entry+0x14a/0x1c0
[252257.839933] [<ffffffff81051af6>] start_secondary+0x1b6/0x230
[252257.840684] Code: 00 00 00 44 0f b6 e0 45 85 e4 0f 84 a7 01 00 00 49
63 d4 48 83 43 10 01 48 8d 75 c0 48 c1 e2 04 48 01 da 48 8b 4a 28 48 8d
42 28 <48> 89 71 08 48 89 4d c0 48 8b 4a 30 48 89 4d c8 48 89 31 48 89
[252257.842366] RIP [<ffffffff810996e0>] run_timer_softirq+0xe0/0x310

Signed-off-by: Greg Rose <gvrose8192@gmail.com>
Acked-by: Pravin B Shelar <pshelar@ovn.org>

3 files changed, 13 insertions, 0 deletions

diff --git a/datapath/datapath.c b/datapath/datapath.c
index db8a18a13..c04826f8a 100644
--- a/datapath/datapath.c
+++ b/datapath/datapath.c
@@ -2440,6 +2440,16 @@ error:
 
 static void dp_cleanup(void)
 {
+#if RHEL_RELEASE_CODE < RHEL_RELEASE_VERSION(8,0)
+	/* On RHEL 7.x kernels we hit a kernel paging error without
+	 * this barrier and subsequent hefty delay.  A process will
+	 * attempt to access openvwitch memory after it has been
+	 * unloaded.  Further debugging is needed on that but for
+	 * now let's not let customer machines panic.
+	 */
+	rcu_barrier();
+	msleep(3000);
+#endif
 	dp_unregister_genl(ARRAY_SIZE(dp_genl_families));
 	ovs_netdev_exit();
 	unregister_netdevice_notifier(&ovs_dp_device_notifier);
diff --git a/tests/system-kmod-macros.at b/tests/system-kmod-macros.at
index e1b570792..2d652deea 100644
--- a/tests/system-kmod-macros.at
+++ b/tests/system-kmod-macros.at
@@ -23,6 +23,7 @@ m4_define([OVS_TRAFFIC_VSWITCHD_START],
                on_exit 'modprobe -q -r mod'
               ])
    on_exit 'ovs-dpctl del-dp ovs-system'
+   on_exit 'ovs-appctl dpctl/flush-conntrack'
    _OVS_VSWITCHD_START([])
    dnl Add bridges, ports, etc.
    AT_CHECK([ovs-vsctl -- _ADD_BR([br0]) -- $1 m4_if([$2], [], [], [| ${PERL} $srcdir/uuidfilt.pl])], [0], [$2])
diff --git a/utilities/ovs-ctl.in b/utilities/ovs-ctl.in
index 7c61ff0d7..5778f4012 100755
--- a/utilities/ovs-ctl.in
+++ b/utilities/ovs-ctl.in
@@ -387,6 +387,8 @@ force_reload_kmod () {
         action "Removing datapath: $dp" ovs-dpctl del-dp "$dp"
     done
 
+    action "ovs-appctl dpctl/flush-conntrack"
+
     for vport in `awk '/^vport_/ { print $1 }' /proc/modules`; do
         action "Removing $vport module" rmmod $vport
         if ! grep -q $vport /proc/modules; then