From 3eac8ee3c4374b71450f2834fc1ff7ed7a09dcc0 Mon Sep 17 00:00:00 2001 From: Simon MacMullen Date: Tue, 25 Nov 2014 14:21:26 +0000 Subject: Mutual disconnect when partial partition detected. And don't allow a very short disconnection since Mnesia might not detect it. --- src/rabbit_node_monitor.erl | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/src/rabbit_node_monitor.erl b/src/rabbit_node_monitor.erl index fc82af0c..adc197e9 100644 --- a/src/rabbit_node_monitor.erl +++ b/src/rabbit_node_monitor.erl @@ -359,13 +359,22 @@ handle_cast({partial_partition, NotReallyDown, Proxy, MyGUID}, rabbit_log:error( FmtBase ++ "We will therefore intentionally disconnect from ~s~n", ArgsBase ++ [Proxy]), - erlang:disconnect_node(Proxy), + cast(Proxy, {partial_partition_disconnect, node()}), + disconnect(Proxy), {noreply, State} end; handle_cast({partial_partition, _GUID, _Reporter, _Proxy}, State) -> {noreply, State}; +%% Sometimes it appears the Erlang VM does not give us nodedown +%% messages reliably when another node disconnects from us. Therefore +%% we are told just before the disconnection so we can reciprocate. +handle_cast({partial_partition_disconnect, Other}, State) -> + rabbit_log:error("Partial partition disconnect from ~s~n", [Other]), + disconnect(Other), + {noreply, State}; + %% Note: when updating the status file, we can't simply write the %% mnesia information since the message can (and will) overtake the %% mnesia propagation. @@ -646,6 +655,19 @@ del_node(Node, Nodes) -> Nodes -- [Node]. cast(Node, Msg) -> gen_server:cast({?SERVER, Node}, Msg). +%% When we call this, it's because we want to force Mnesia to detect a +%% partition. But if we just disconnect_node/1 then Mnesia won't +%% detect a very short partition. So we want to force a slightly +%% longer disconnect. Unfortunately we don't have a way to blacklist +%% individual nodes; the best we can do is turn off auto-connect +%% altogether. +disconnect(Node) -> + application:set_env(kernel, dist_auto_connect, never), + erlang:disconnect_node(Node), + timer:sleep(1000), + application:unset_env(kernel, dist_auto_connect), + ok. + %%-------------------------------------------------------------------- %% mnesia:system_info(db_nodes) (and hence -- cgit v1.2.1