Merged bug24624 into default

7 files changed, 124 insertions, 71 deletions

diff --git a/src/rabbit.erl b/src/rabbit.erl
index 3eb75217..0a0ca90a 100644
--- a/src/rabbit.erl
+++ b/src/rabbit.erl
@@ -366,10 +366,8 @@ rotate_logs(BinarySuffix) ->
 start(normal, []) ->
     case erts_version_check() of
         ok ->
-            ok = rabbit_mnesia:delete_previously_running_nodes(),
             {ok, SupPid} = rabbit_sup:start_link(),
             true = register(rabbit, self()),
-
             print_banner(),
             [ok = run_boot_step(Step) || Step <- boot_steps()],
             io:format("~nbroker running~n"),
@@ -507,6 +505,19 @@ sort_boot_steps(UnsortedSteps) ->
                end])
     end.
 
+boot_step_error({error, {timeout_waiting_for_tables, _}}, _Stacktrace) ->
+    {Err, Nodes} =
+        case rabbit_mnesia:read_previously_running_nodes() of
+            [] -> {"Timeout contacting cluster nodes. Since RabbitMQ was"
+                   " shut down forcefully~nit cannot determine which nodes"
+                   " are timing out. Details on all nodes will~nfollow.~n",
+                   rabbit_mnesia:all_clustered_nodes() -- [node()]};
+            Ns -> {rabbit_misc:format(
+                     "Timeout contacting cluster nodes: ~p.~n", [Ns]),
+                   Ns}
+        end,
+    boot_error(Err ++ rabbit_nodes:diagnostics(Nodes) ++ "~n~n", []);
+
 boot_step_error(Reason, Stacktrace) ->
     boot_error("Error description:~n   ~p~n~n"
                "Log files (may contain more information):~n   ~s~n   ~s~n~n"
@@ -668,7 +679,7 @@ print_banner() ->
                 {"app descriptor", app_location()},
                 {"home dir",       home_dir()},
                 {"config file(s)", config_files()},
-                {"cookie hash",    rabbit_misc:cookie_hash()},
+                {"cookie hash",    rabbit_nodes:cookie_hash()},
                 {"log",            log_location(kernel)},
                 {"sasl log",       log_location(sasl)},
                 {"database dir",   rabbit_mnesia:dir()},
diff --git a/src/rabbit_control.erl b/src/rabbit_control.erl
index d187a0b2..6a775adf 100644
--- a/src/rabbit_control.erl
+++ b/src/rabbit_control.erl
@@ -17,7 +17,7 @@
 -module(rabbit_control).
 -include("rabbit.hrl").
 
--export([start/0, stop/0, action/5, diagnostics/1]).
+-export([start/0, stop/0, action/5]).
 
 -define(RPC_TIMEOUT, infinity).
 -define(EXTERNAL_CHECK_INTERVAL, 1000).
@@ -49,7 +49,6 @@
         (atom(), node(), [string()], [{string(), any()}],
          fun ((string(), [any()]) -> 'ok'))
         -> 'ok').
--spec(diagnostics/1 :: (node()) -> [{string(), [any()]}]).
 -spec(usage/0 :: () -> no_return()).
 
 -endif.
@@ -67,7 +66,7 @@ start() ->
             CmdArgsAndOpts -> CmdArgsAndOpts
         end,
     Opts1 = [case K of
-                 ?NODE_OPT -> {?NODE_OPT, rabbit_misc:makenode(V)};
+                 ?NODE_OPT -> {?NODE_OPT, rabbit_nodes:make(V)};
                  _         -> {K, V}
              end || {K, V} <- Opts],
     Command = list_to_atom(Command0),
@@ -143,26 +142,7 @@ print_report0(Node, {Module, InfoFun, KeysFun}, VHostArg) ->
 print_error(Format, Args) -> fmt_stderr("Error: " ++ Format, Args).
 
 print_badrpc_diagnostics(Node) ->
-    [fmt_stderr(Fmt, Args) || {Fmt, Args} <- diagnostics(Node)].
-
-diagnostics(Node) ->
-    {_NodeName, NodeHost} = rabbit_misc:nodeparts(Node),
-    [{"diagnostics:", []},
-     case net_adm:names(NodeHost) of
-         {error, EpmdReason} ->
-             {"- unable to connect to epmd on ~s: ~w",
-              [NodeHost, EpmdReason]};
-         {ok, NamePorts} ->
-             {"- nodes and their ports on ~s: ~p",
-              [NodeHost, [{list_to_atom(Name), Port} ||
-                             {Name, Port} <- NamePorts]]}
-     end,
-     {"- current node: ~w", [node()]},
-     case init:get_argument(home) of
-         {ok, [[Home]]} -> {"- current node home dir: ~s", [Home]};
-         Other          -> {"- no current node home dir: ~p", [Other]}
-     end,
-     {"- current node cookie hash: ~s", [rabbit_misc:cookie_hash()]}].
+    fmt_stderr(rabbit_nodes:diagnostics([Node]), []).
 
 stop() ->
     ok.
diff --git a/src/rabbit_misc.erl b/src/rabbit_misc.erl
index 4a8cbace..798c21e6 100644
--- a/src/rabbit_misc.erl
+++ b/src/rabbit_misc.erl
@@ -34,7 +34,7 @@
 -export([execute_mnesia_transaction/2]).
 -export([execute_mnesia_tx_with_tail/1]).
 -export([ensure_ok/2]).
--export([makenode/1, nodeparts/1, cookie_hash/0, tcp_name/3]).
+-export([tcp_name/3]).
 -export([upmap/2, map_in_order/2]).
 -export([table_filter/3]).
 -export([dirty_read_all/1, dirty_foreach_key/2, dirty_dump_log/1]).
@@ -141,9 +141,6 @@
 -spec(execute_mnesia_tx_with_tail/1 ::
         (thunk(fun ((boolean()) -> B))) -> B | (fun ((boolean()) -> B))).
 -spec(ensure_ok/2 :: (ok_or_error(), atom()) -> 'ok').
--spec(makenode/1 :: ({string(), string()} | string()) -> node()).
--spec(nodeparts/1 :: (node() | string()) -> {string(), string()}).
--spec(cookie_hash/0 :: () -> string()).
 -spec(tcp_name/3 ::
         (atom(), inet:ip_address(), rabbit_networking:ip_port())
         -> atom()).
@@ -336,7 +333,7 @@ enable_cover(Dirs) ->
                 end, ok, Dirs).
 
 start_cover(NodesS) ->
-    {ok, _} = cover:start([makenode(N) || N <- NodesS]),
+    {ok, _} = cover:start([rabbit_nodes:make(N) || N <- NodesS]),
     ok.
 
 report_cover() -> report_cover(["."]).
@@ -453,23 +450,6 @@ execute_mnesia_tx_with_tail(TxFun) ->
 ensure_ok(ok, _) -> ok;
 ensure_ok({error, Reason}, ErrorTag) -> throw({error, {ErrorTag, Reason}}).
 
-makenode({Prefix, Suffix}) ->
-    list_to_atom(lists:append([Prefix, "@", Suffix]));
-makenode(NodeStr) ->
-    makenode(nodeparts(NodeStr)).
-
-nodeparts(Node) when is_atom(Node) ->
-    nodeparts(atom_to_list(Node));
-nodeparts(NodeStr) ->
-    case lists:splitwith(fun (E) -> E =/= $@ end, NodeStr) of
-        {Prefix, []}     -> {_, Suffix} = nodeparts(node()),
-                            {Prefix, Suffix};
-        {Prefix, Suffix} -> {Prefix, tl(Suffix)}
-    end.
-
-cookie_hash() ->
-    base64:encode_to_string(erlang:md5(atom_to_list(erlang:get_cookie()))).
-
 tcp_name(Prefix, IPAddress, Port)
   when is_atom(Prefix) andalso is_number(Port) ->
     list_to_atom(
diff --git a/src/rabbit_mnesia.erl b/src/rabbit_mnesia.erl
index 30b5478e..60dd0770 100644
--- a/src/rabbit_mnesia.erl
+++ b/src/rabbit_mnesia.erl
@@ -23,8 +23,8 @@
          empty_ram_only_tables/0, copy_db/1, wait_for_tables/1,
          create_cluster_nodes_config/1, read_cluster_nodes_config/0,
          record_running_nodes/0, read_previously_running_nodes/0,
-         delete_previously_running_nodes/0, running_nodes_filename/0,
-         is_disc_node/0, on_node_down/1, on_node_up/1]).
+         running_nodes_filename/0, is_disc_node/0, on_node_down/1,
+         on_node_up/1]).
 
 -export([table_names/0]).
 
@@ -64,7 +64,6 @@
 -spec(read_cluster_nodes_config/0 :: () ->  [node()]).
 -spec(record_running_nodes/0 :: () ->  'ok').
 -spec(read_previously_running_nodes/0 :: () ->  [node()]).
--spec(delete_previously_running_nodes/0 :: () ->  'ok').
 -spec(running_nodes_filename/0 :: () -> file:filename()).
 -spec(is_disc_node/0 :: () -> boolean()).
 -spec(on_node_up/1 :: (node()) -> 'ok').
@@ -104,6 +103,7 @@ init() ->
     %% Mnesia is up. In fact that's not guaranteed to be the case - let's
     %% make it so.
     ok = global:sync(),
+    ok = delete_previously_running_nodes(),
     ok.
 
 is_db_empty() ->
diff --git a/src/rabbit_nodes.erl b/src/rabbit_nodes.erl
new file mode 100644
index 00000000..ba695b8e
--- /dev/null
+++ b/src/rabbit_nodes.erl
@@ -0,0 +1,96 @@
+%% The contents of this file are subject to the Mozilla Public License
+%% Version 1.1 (the "License"); you may not use this file except in
+%% compliance with the License. You may obtain a copy of the License
+%% at http://www.mozilla.org/MPL/
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and
+%% limitations under the License.
+%%
+%% The Original Code is RabbitMQ.
+%%
+%% The Initial Developer of the Original Code is VMware, Inc.
+%% Copyright (c) 2007-2012 VMware, Inc.  All rights reserved.
+%%
+
+-module(rabbit_nodes).
+
+-export([names/1, diagnostics/1, make/1, parts/1, cookie_hash/0]).
+
+-define(EPMD_TIMEOUT, 30000).
+
+%%----------------------------------------------------------------------------
+%% Specs
+%%----------------------------------------------------------------------------
+
+-ifdef(use_specs).
+
+-spec(names/1 :: (string()) -> rabbit_types:ok_or_error2(
+                                 [{string(), integer()}], term())).
+-spec(diagnostics/1 :: ([node()]) -> string()).
+-spec(make/1 :: ({string(), string()} | string()) -> node()).
+-spec(parts/1 :: (node() | string()) -> {string(), string()}).
+-spec(cookie_hash/0 :: () -> string()).
+
+-endif.
+
+%%----------------------------------------------------------------------------
+
+names(Hostname) ->
+    Self = self(),
+    process_flag(trap_exit, true),
+    Pid = spawn_link(fun () -> Self ! {names, net_adm:names(Hostname)} end),
+    timer:exit_after(?EPMD_TIMEOUT, Pid, timeout),
+    Res = receive
+              {names, Names}        -> Names;
+              {'EXIT', Pid, Reason} -> {error, Reason}
+          end,
+    process_flag(trap_exit, false),
+    Res.
+
+diagnostics(Nodes) ->
+    Hosts = lists:usort([element(2, parts(Node)) || Node <- Nodes]),
+    NodeDiags = [{"~nDIAGNOSTICS~n===========~n~n"
+                  "nodes in question: ~p~n~n"
+                  "hosts, their running nodes and ports:", [Nodes]}] ++
+        [diagnostics_host(Host) || Host <- Hosts] ++
+        diagnostics0(),
+    lists:flatten([io_lib:format(F ++ "~n", A) || NodeDiag <- NodeDiags,
+                                                  {F, A}   <- [NodeDiag]]).
+
+diagnostics0() ->
+    [{"~ncurrent node details:~n- node name: ~w", [node()]},
+     case init:get_argument(home) of
+         {ok, [[Home]]} -> {"- home dir: ~s", [Home]};
+         Other          -> {"- no home dir: ~p", [Other]}
+     end,
+     {"- cookie hash: ~s", [cookie_hash()]}].
+
+diagnostics_host(Host) ->
+    case names(Host) of
+        {error, EpmdReason} ->
+            {"- unable to connect to epmd on ~s: ~w",
+             [Host, EpmdReason]};
+        {ok, NamePorts} ->
+            {"- ~s: ~p",
+             [Host, [{list_to_atom(Name), Port} ||
+                        {Name, Port} <- NamePorts]]}
+    end.
+
+make({Prefix, Suffix}) ->
+    list_to_atom(lists:append([Prefix, "@", Suffix]));
+make(NodeStr) ->
+    make(parts(NodeStr)).
+
+parts(Node) when is_atom(Node) ->
+    parts(atom_to_list(Node));
+parts(NodeStr) ->
+    case lists:splitwith(fun (E) -> E =/= $@ end, NodeStr) of
+        {Prefix, []}     -> {_, Suffix} = parts(node()),
+                            {Prefix, Suffix};
+        {Prefix, Suffix} -> {Prefix, tl(Suffix)}
+    end.
+
+cookie_hash() ->
+    base64:encode_to_string(erlang:md5(atom_to_list(erlang:get_cookie()))).
diff --git a/src/rabbit_prelaunch.erl b/src/rabbit_prelaunch.erl
index 11b4d776..162d44f1 100644
--- a/src/rabbit_prelaunch.erl
+++ b/src/rabbit_prelaunch.erl
@@ -22,7 +22,6 @@
 
 -define(BaseApps, [rabbit]).
 -define(ERROR_CODE, 1).
--define(EPMD_TIMEOUT, 30000).
 
 %%----------------------------------------------------------------------------
 %% Specs
@@ -244,16 +243,15 @@ duplicate_node_check([]) ->
     %% Ignore running node while installing windows service
     ok;
 duplicate_node_check(NodeStr) ->
-    Node = rabbit_misc:makenode(NodeStr),
-    {NodeName, NodeHost} = rabbit_misc:nodeparts(Node),
-    case names(NodeHost) of
+    Node = rabbit_nodes:make(NodeStr),
+    {NodeName, NodeHost} = rabbit_nodes:parts(Node),
+    case rabbit_nodes:names(NodeHost) of
         {ok, NamePorts}  ->
             case proplists:is_defined(NodeName, NamePorts) of
                 true -> io:format("node with name ~p "
                                   "already running on ~p~n",
                                   [NodeName, NodeHost]),
-                        [io:format(Fmt ++ "~n", Args) ||
-                            {Fmt, Args} <- rabbit_control:diagnostics(Node)],
+                        io:format(rabbit_nodes:diagnostics([Node]) ++ "~n"),
                         terminate(?ERROR_CODE);
                 false -> ok
             end;
@@ -279,15 +277,3 @@ terminate(Status) ->
                       after infinity -> ok
                       end
     end.
-
-names(Hostname) ->
-    Self = self(),
-    process_flag(trap_exit, true),
-    Pid = spawn_link(fun () -> Self ! {names, net_adm:names(Hostname)} end),
-    timer:exit_after(?EPMD_TIMEOUT, Pid, timeout),
-    Res = receive
-              {names, Names}        -> Names;
-              {'EXIT', Pid, Reason} -> {error, Reason}
-          end,
-    process_flag(trap_exit, false),
-    Res.
diff --git a/src/rabbit_tests.erl b/src/rabbit_tests.erl
index b9c0edfe..c645a436 100644
--- a/src/rabbit_tests.erl
+++ b/src/rabbit_tests.erl
@@ -59,7 +59,7 @@ all_tests() ->
     passed.
 
 maybe_run_cluster_dependent_tests() ->
-    SecondaryNode = rabbit_misc:makenode("hare"),
+    SecondaryNode = rabbit_nodes:make("hare"),
 
     case net_adm:ping(SecondaryNode) of
         pong -> passed = run_cluster_dependent_tests(SecondaryNode);
@@ -859,7 +859,7 @@ test_cluster_management() ->
                                         "invalid2@invalid"]),
     ok = assert_ram_node(),
 
-    SecondaryNode = rabbit_misc:makenode("hare"),
+    SecondaryNode = rabbit_nodes:make("hare"),
     case net_adm:ping(SecondaryNode) of
         pong -> passed = test_cluster_management2(SecondaryNode);
         pang -> io:format("Skipping clustering tests with node ~p~n",