rabbit: Run plugins' boot steps during rabbit start/2run-plugins-boot-steps-during-rabbit-start

This restores the behavior prior the commit making `rabbit` closer to a
standard Erlang application.

Plugins are still actually started after rabbit is started (because they
depend on the `rabbit` application). Only the execution of their boot
steps was moved earlier.

With the behavior restored, it also means that a plugin's dependencies
are not started yet when its boot steps are executed.

V2: Move the maintenance mode reset before the plugin boot steps run.

V3: Add a `core_started` boot state. That state is reached at the end of
    the `rabbit` app start function. It indicates when the RabbitMQ core
    is started but the full service is not yet ready.

    We now use this state in direct connection code to determine if
    clients can open a direct connection. We have to do that because
    some plugins open a direct connection as part of their own startup
    (i.e. they can't wait for the `ready` boot state which comes later).

3 files changed, 71 insertions, 50 deletions

diff --git a/deps/rabbit/apps/rabbitmq_prelaunch/src/rabbit_boot_state.erl b/deps/rabbit/apps/rabbitmq_prelaunch/src/rabbit_boot_state.erl
index c76824e7be..12a0e1f192 100644
--- a/deps/rabbit/apps/rabbitmq_prelaunch/src/rabbit_boot_state.erl
+++ b/deps/rabbit/apps/rabbitmq_prelaunch/src/rabbit_boot_state.erl
@@ -12,11 +12,13 @@
 
 -export([get/0,
          set/1,
-         wait_for/2]).
+         wait_for/2,
+         has_reached/1,
+         has_reached_and_is_active/1]).
 
 -define(PT_KEY_BOOT_STATE,    {?MODULE, boot_state}).
 
--type boot_state() :: 'stopped' | 'booting' | 'ready' | 'stopping'.
+-type boot_state() :: 'stopped' | 'booting' | 'core_started' | 'ready' | 'stopping'.
 
 -export_type([boot_state/0]).
 
@@ -36,7 +38,7 @@ set(BootState) ->
 
 -spec wait_for(boot_state(), timeout()) -> ok | {error, timeout}.
 wait_for(BootState, infinity) ->
-    case is_reached(BootState) of
+    case has_reached(BootState) of
         true  -> ok;
         false -> Wait = 200,
                  timer:sleep(Wait),
@@ -44,7 +46,7 @@ wait_for(BootState, infinity) ->
     end;
 wait_for(BootState, Timeout)
   when is_integer(Timeout) andalso Timeout >= 0 ->
-    case is_reached(BootState) of
+    case has_reached(BootState) of
         true  -> ok;
         false -> Wait = 200,
                  timer:sleep(Wait),
@@ -53,24 +55,35 @@ wait_for(BootState, Timeout)
 wait_for(_, _) ->
     {error, timeout}.
 
-boot_state_idx(stopped)  -> 0;
-boot_state_idx(booting)  -> 1;
-boot_state_idx(ready)    -> 2;
-boot_state_idx(stopping) -> 3.
+boot_state_idx(stopped)      -> 0;
+boot_state_idx(booting)      -> 1;
+boot_state_idx(core_started) -> 2;
+boot_state_idx(ready)        -> 3;
+boot_state_idx(stopping)     -> 4.
 
 is_valid(BootState) ->
     is_integer(boot_state_idx(BootState)).
 
-is_reached(TargetBootState) ->
-    is_reached(?MODULE:get(), TargetBootState).
+has_reached(TargetBootState) ->
+    has_reached(?MODULE:get(), TargetBootState).
 
-is_reached(CurrentBootState, CurrentBootState) ->
+has_reached(CurrentBootState, CurrentBootState) ->
     true;
-is_reached(stopping, stopped) ->
+has_reached(stopping, stopped) ->
     false;
-is_reached(_CurrentBootState, stopped) ->
+has_reached(_CurrentBootState, stopped) ->
     true;
-is_reached(stopped, _TargetBootState) ->
+has_reached(stopped, _TargetBootState) ->
     true;
-is_reached(CurrentBootState, TargetBootState) ->
+has_reached(CurrentBootState, TargetBootState) ->
     boot_state_idx(TargetBootState) =< boot_state_idx(CurrentBootState).
+
+has_reached_and_is_active(TargetBootState) ->
+    case ?MODULE:get() of
+        stopped ->
+            false;
+        CurrentBootState ->
+            has_reached(CurrentBootState, TargetBootState)
+            andalso
+            not has_reached(CurrentBootState, stopping)
+    end.
diff --git a/deps/rabbit/src/rabbit.erl b/deps/rabbit/src/rabbit.erl
index 9248c945dc..61e1149c11 100644
--- a/deps/rabbit/src/rabbit.erl
+++ b/deps/rabbit/src/rabbit.erl
@@ -542,10 +542,9 @@ handle_app_error(Term) ->
 is_booting() -> is_booting(node()).
 
 is_booting(Node) when Node =:= node() ->
-    case rabbit_boot_state:get() of
-        booting           -> true;
-        _                 -> false
-    end;
+    rabbit_boot_state:has_reached_and_is_active(booting)
+    andalso
+    not rabbit_boot_state:has_reached(ready);
 is_booting(Node) ->
     case rpc:call(Node, rabbit, is_booting, []) of
         {badrpc, _} = Err -> Err;
@@ -870,11 +869,30 @@ start(normal, []) ->
         log_banner(),
         warn_if_kernel_config_dubious(),
         warn_if_disc_io_options_dubious(),
-        %% We run `rabbit` boot steps only for now. Plugins boot steps
-        %% will be executed as part of the postlaunch phase after they
-        %% are started.
-        rabbit_boot_steps:run_boot_steps([rabbit]),
-        run_postlaunch_phase(),
+
+        rabbit_log_prelaunch:debug(""),
+        rabbit_log_prelaunch:debug("== Plugins (prelaunch phase) =="),
+
+        rabbit_log_prelaunch:debug("Setting plugins up"),
+        %% `Plugins` contains all the enabled plugins, plus their
+        %% dependencies. The order is important: dependencies appear
+        %% before plugin which depend on them.
+        Plugins = rabbit_plugins:setup(),
+        rabbit_log_prelaunch:debug(
+          "Loading the following plugins: ~p", [Plugins]),
+        %% We can load all plugins and refresh their feature flags at
+        %% once, because it does not involve running code from the
+        %% plugins.
+        ok = app_utils:load_applications(Plugins),
+        ok = rabbit_feature_flags:refresh_feature_flags_after_app_load(
+               Plugins),
+
+        rabbit_log_prelaunch:debug(""),
+        rabbit_log_prelaunch:debug("== Boot steps =="),
+
+        ok = rabbit_boot_steps:run_boot_steps([rabbit | Plugins]),
+        run_postlaunch_phase(Plugins),
+        rabbit_boot_state:set(core_started),
         {ok, SupPid}
     catch
         throw:{error, _} = Error ->
@@ -893,50 +911,39 @@ start(normal, []) ->
             Error
     end.
 
-run_postlaunch_phase() ->
-    spawn(fun() -> do_run_postlaunch_phase() end).
+run_postlaunch_phase(Plugins) ->
+    spawn(fun() -> do_run_postlaunch_phase(Plugins) end).
 
-do_run_postlaunch_phase() ->
+do_run_postlaunch_phase(Plugins) ->
     %% Once RabbitMQ itself is started, we need to run a few more steps,
     %% in particular start plugins.
     rabbit_log_prelaunch:debug(""),
     rabbit_log_prelaunch:debug("== Postlaunch phase =="),
 
     try
+        %% Successful boot resets node maintenance state.
+        rabbit_log_prelaunch:debug(""),
+        rabbit_log_prelaunch:info("Resetting node maintenance status"),
+        _ = rabbit_maintenance:unmark_as_being_drained(),
+
         rabbit_log_prelaunch:debug(""),
-        rabbit_log_prelaunch:debug("== Plugins =="),
+        rabbit_log_prelaunch:debug("== Plugins (postlaunch phase) =="),
 
-        rabbit_log_prelaunch:debug("Setting plugins up"),
-        %% `Plugins` contains all the enabled plugins, plus their
-        %% dependencies. The order is important: dependencies appear
-        %% before plugin which depend on them.
-        Plugins = rabbit_plugins:setup(),
-        rabbit_log_prelaunch:debug(
-          "Starting the following plugins: ~p", [Plugins]),
-        %% We can load all plugins and refresh their feature flags at
-        %% once, because it does not involve running code from the
-        %% plugins.
-        app_utils:load_applications(Plugins),
-        ok = rabbit_feature_flags:refresh_feature_flags_after_app_load(
-               Plugins),
         %% However, we want to run their boot steps and actually start
         %% them one by one, to ensure a dependency is fully started
         %% before a plugin which depends on it gets a chance to start.
+        rabbit_log_prelaunch:debug(
+          "Starting the following plugins: ~p", [Plugins]),
         lists:foreach(
           fun(Plugin) ->
-                  ok = rabbit_boot_steps:run_boot_steps([Plugin]),
                   case application:ensure_all_started(Plugin) of
                       {ok, _} -> ok;
                       Error   -> throw(Error)
                   end
           end, Plugins),
 
-        %% Successful boot resets node maintenance state.
-        rabbit_log_prelaunch:info("Resetting node maintenance status"),
-        _ = rabbit_maintenance:unmark_as_being_drained(),
-
         %% Export definitions after all plugins have been enabled,
-        %% see rabbitmq/rabbitmq-server#2384
+        %% see rabbitmq/rabbitmq-server#2384.
         case rabbit_definitions:maybe_load_definitions() of
             ok           -> ok;
             DefLoadError -> throw(DefLoadError)
@@ -951,8 +958,9 @@ do_run_postlaunch_phase() ->
         %% The node is ready: mark it as such and log it.
         %% NOTE: PLEASE DO NOT ADD CRITICAL NODE STARTUP CODE AFTER THIS.
         ok = rabbit_lager:broker_is_started(),
-        ok = log_broker_started(
-               rabbit_plugins:strictly_plugins(rabbit_plugins:active())),
+        ActivePlugins = rabbit_plugins:active(),
+        StrictlyPlugins = rabbit_plugins:strictly_plugins(ActivePlugins),
+        ok = log_broker_started(StrictlyPlugins),
 
         rabbit_log_prelaunch:debug("Marking ~s as running", [product_name()]),
         rabbit_boot_state:set(ready)
diff --git a/deps/rabbit/src/rabbit_direct.erl b/deps/rabbit/src/rabbit_direct.erl
index 3fc2d75908..0506e1bf6c 100644
--- a/deps/rabbit/src/rabbit_direct.erl
+++ b/deps/rabbit/src/rabbit_direct.erl
@@ -75,7 +75,7 @@ auth_fun({Username, Password}, VHost, ExtraAuthProps) ->
 connect(Creds, VHost, Protocol, Pid, Infos) ->
     ExtraAuthProps = extract_extra_auth_props(Creds, VHost, Pid, Infos),
     AuthFun = auth_fun(Creds, VHost, ExtraAuthProps),
-    case rabbit:is_running() of
+    case rabbit_boot_state:has_reached_and_is_active(core_started) of
         true  ->
             case whereis(rabbit_direct_client_sup) of
                 undefined ->