From a7da5b79f137abc3941b40cf80789359357a4348 Mon Sep 17 00:00:00 2001
From: Simon MacMullen <simon@rabbitmq.com>
Date: Tue, 25 Nov 2014 12:33:22 +0000
Subject: Do not attempt to "fix up" the partitions field; if Mnesia does not
 detect the partition it will behave wrong anyway.

---
 src/rabbit_node_monitor.erl | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/src/rabbit_node_monitor.erl b/src/rabbit_node_monitor.erl
index a4ae2a5e..fc82af0c 100644
--- a/src/rabbit_node_monitor.erl
+++ b/src/rabbit_node_monitor.erl
@@ -342,7 +342,7 @@ handle_cast({check_partial_partition, _Node, _Reporter,
     {noreply, State};
 
 handle_cast({partial_partition, NotReallyDown, Proxy, MyGUID},
-            State = #state{guid = MyGUID, partitions = Partitions}) ->
+            State = #state{guid = MyGUID}) ->
     FmtBase = "Partial partition detected:~n"
         " * We saw DOWN from ~s~n"
         " * We can still see ~s which can see ~s~n",
@@ -360,11 +360,7 @@ handle_cast({partial_partition, NotReallyDown, Proxy, MyGUID},
               FmtBase ++ "We will therefore intentionally disconnect from ~s~n",
               ArgsBase ++ [Proxy]),
             erlang:disconnect_node(Proxy),
-            %% In the event of explicitly disconnecting from a node,
-            %% sometimes Mnesia does not log that we were partitioned
-            %% - so note it here.
-            Partitions1 = lists:usort([Proxy | Partitions]),
-            {noreply, State#state{partitions = Partitions1}}
+            {noreply, State}
     end;
 
 handle_cast({partial_partition, _GUID, _Reporter, _Proxy}, State) ->
-- 
cgit v1.2.1


From 3eac8ee3c4374b71450f2834fc1ff7ed7a09dcc0 Mon Sep 17 00:00:00 2001
From: Simon MacMullen <simon@rabbitmq.com>
Date: Tue, 25 Nov 2014 14:21:26 +0000
Subject: Mutual disconnect when partial partition detected. And don't allow a
 very short disconnection since Mnesia might not detect it.

---
 src/rabbit_node_monitor.erl | 24 +++++++++++++++++++++++-
 1 file changed, 23 insertions(+), 1 deletion(-)

diff --git a/src/rabbit_node_monitor.erl b/src/rabbit_node_monitor.erl
index fc82af0c..adc197e9 100644
--- a/src/rabbit_node_monitor.erl
+++ b/src/rabbit_node_monitor.erl
@@ -359,13 +359,22 @@ handle_cast({partial_partition, NotReallyDown, Proxy, MyGUID},
             rabbit_log:error(
               FmtBase ++ "We will therefore intentionally disconnect from ~s~n",
               ArgsBase ++ [Proxy]),
-            erlang:disconnect_node(Proxy),
+            cast(Proxy, {partial_partition_disconnect, node()}),
+            disconnect(Proxy),
             {noreply, State}
     end;
 
 handle_cast({partial_partition, _GUID, _Reporter, _Proxy}, State) ->
     {noreply, State};
 
+%% Sometimes it appears the Erlang VM does not give us nodedown
+%% messages reliably when another node disconnects from us. Therefore
+%% we are told just before the disconnection so we can reciprocate.
+handle_cast({partial_partition_disconnect, Other}, State) ->
+    rabbit_log:error("Partial partition disconnect from ~s~n", [Other]),
+    disconnect(Other),
+    {noreply, State};
+
 %% Note: when updating the status file, we can't simply write the
 %% mnesia information since the message can (and will) overtake the
 %% mnesia propagation.
@@ -646,6 +655,19 @@ del_node(Node, Nodes) -> Nodes -- [Node].
 
 cast(Node, Msg) -> gen_server:cast({?SERVER, Node}, Msg).
 
+%% When we call this, it's because we want to force Mnesia to detect a
+%% partition. But if we just disconnect_node/1 then Mnesia won't
+%% detect a very short partition. So we want to force a slightly
+%% longer disconnect. Unfortunately we don't have a way to blacklist
+%% individual nodes; the best we can do is turn off auto-connect
+%% altogether.
+disconnect(Node) ->
+    application:set_env(kernel, dist_auto_connect, never),
+    erlang:disconnect_node(Node),
+    timer:sleep(1000),
+    application:unset_env(kernel, dist_auto_connect),
+    ok.
+
 %%--------------------------------------------------------------------
 
 %% mnesia:system_info(db_nodes) (and hence
-- 
cgit v1.2.1


From 15aa5301e21e72330bfe3886593f46039a90cadb Mon Sep 17 00:00:00 2001
From: Simon MacMullen <simon@rabbitmq.com>
Date: Tue, 25 Nov 2014 14:21:48 +0000
Subject: Also add a tiny bit more logging, for symmetry.

---
 src/rabbit_node_monitor.erl | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/rabbit_node_monitor.erl b/src/rabbit_node_monitor.erl
index adc197e9..82a7a89b 100644
--- a/src/rabbit_node_monitor.erl
+++ b/src/rabbit_node_monitor.erl
@@ -453,6 +453,10 @@ handle_info({nodedown, Node, Info}, State = #state{guid       = MyGUID,
     end,
     {noreply, handle_dead_node(Node, State)};
 
+handle_info({nodeup, Node, _Info}, State) ->
+    rabbit_log:info("node ~p up~n", [Node]),
+    {noreply, State};
+
 handle_info({mnesia_system_event,
              {inconsistent_database, running_partitioned_network, Node}},
             State = #state{partitions = Partitions,
-- 
cgit v1.2.1