Merge pull request #2656 from rabbitmq/run-plugins-boot-steps-during-rabbit-startv3.8.10-rc.5

rabbit: Run plugins' boot steps during rabbit start/2
(cherry picked from commit 67624cdadcd24d6f3724020503cf68111d7e2c61)

3 files changed, 71 insertions, 46 deletions

diff --git a/deps/rabbit/apps/rabbitmq_prelaunch/src/rabbit_boot_state.erl b/deps/rabbit/apps/rabbitmq_prelaunch/src/rabbit_boot_state.erl
index 2610ba887e..eafb1ccdcd 100644
--- a/deps/rabbit/apps/rabbitmq_prelaunch/src/rabbit_boot_state.erl
+++ b/deps/rabbit/apps/rabbitmq_prelaunch/src/rabbit_boot_state.erl
@@ -20,11 +20,13 @@
 
 -export([get/0,
          set/1,
-         wait_for/2]).
+         wait_for/2,
+         has_reached/1,
+         has_reached_and_is_active/1]).
 
 -define(PT_KEY_BOOT_STATE,    {?MODULE, boot_state}).
 
--type boot_state() :: 'stopped' | 'booting' | 'ready' | 'stopping'.
+-type boot_state() :: 'stopped' | 'booting' | 'core_started' | 'ready' | 'stopping'.
 
 -export_type([boot_state/0]).
 
@@ -44,7 +46,7 @@ set(BootState) ->
 
 -spec wait_for(boot_state(), timeout()) -> ok | {error, timeout}.
 wait_for(BootState, infinity) ->
-    case is_reached(BootState) of
+    case has_reached(BootState) of
         true  -> ok;
         false -> Wait = 200,
                  timer:sleep(Wait),
@@ -52,7 +54,7 @@ wait_for(BootState, infinity) ->
     end;
 wait_for(BootState, Timeout)
   when is_integer(Timeout) andalso Timeout >= 0 ->
-    case is_reached(BootState) of
+    case has_reached(BootState) of
         true  -> ok;
         false -> Wait = 200,
                  timer:sleep(Wait),
@@ -61,24 +63,35 @@ wait_for(BootState, Timeout)
 wait_for(_, _) ->
     {error, timeout}.
 
-boot_state_idx(stopped)  -> 0;
-boot_state_idx(booting)  -> 1;
-boot_state_idx(ready)    -> 2;
-boot_state_idx(stopping) -> 3.
+boot_state_idx(stopped)      -> 0;
+boot_state_idx(booting)      -> 1;
+boot_state_idx(core_started) -> 2;
+boot_state_idx(ready)        -> 3;
+boot_state_idx(stopping)     -> 4.
 
 is_valid(BootState) ->
     is_integer(boot_state_idx(BootState)).
 
-is_reached(TargetBootState) ->
-    is_reached(?MODULE:get(), TargetBootState).
+has_reached(TargetBootState) ->
+    has_reached(?MODULE:get(), TargetBootState).
 
-is_reached(CurrentBootState, CurrentBootState) ->
+has_reached(CurrentBootState, CurrentBootState) ->
     true;
-is_reached(stopping, stopped) ->
+has_reached(stopping, stopped) ->
     false;
-is_reached(_CurrentBootState, stopped) ->
+has_reached(_CurrentBootState, stopped) ->
     true;
-is_reached(stopped, _TargetBootState) ->
+has_reached(stopped, _TargetBootState) ->
     true;
-is_reached(CurrentBootState, TargetBootState) ->
+has_reached(CurrentBootState, TargetBootState) ->
     boot_state_idx(TargetBootState) =< boot_state_idx(CurrentBootState).
+
+has_reached_and_is_active(TargetBootState) ->
+    case ?MODULE:get() of
+        stopped ->
+            false;
+        CurrentBootState ->
+            has_reached(CurrentBootState, TargetBootState)
+            andalso
+            not has_reached(CurrentBootState, stopping)
+    end.
diff --git a/deps/rabbit/src/rabbit.erl b/deps/rabbit/src/rabbit.erl
index f8b2ea7024..5fafcf1528 100644
--- a/deps/rabbit/src/rabbit.erl
+++ b/deps/rabbit/src/rabbit.erl
@@ -528,10 +528,9 @@ handle_app_error(Term) ->
 is_booting() -> is_booting(node()).
 
 is_booting(Node) when Node =:= node() ->
-    case rabbit_boot_state:get() of
-        booting           -> true;
-        _                 -> false
-    end;
+    rabbit_boot_state:has_reached_and_is_active(booting)
+    andalso
+    not rabbit_boot_state:has_reached(ready);
 is_booting(Node) ->
     case rpc:call(Node, rabbit, is_booting, []) of
         {badrpc, _} = Err -> Err;
@@ -856,11 +855,30 @@ start(normal, []) ->
         log_banner(),
         warn_if_kernel_config_dubious(),
         warn_if_disc_io_options_dubious(),
-        %% We run `rabbit` boot steps only for now. Plugins boot steps
-        %% will be executed as part of the postlaunch phase after they
-        %% are started.
-        rabbit_boot_steps:run_boot_steps([rabbit]),
-        run_postlaunch_phase(),
+
+        rabbit_log_prelaunch:debug(""),
+        rabbit_log_prelaunch:debug("== Plugins (prelaunch phase) =="),
+
+        rabbit_log_prelaunch:debug("Setting plugins up"),
+        %% `Plugins` contains all the enabled plugins, plus their
+        %% dependencies. The order is important: dependencies appear
+        %% before plugin which depend on them.
+        Plugins = rabbit_plugins:setup(),
+        rabbit_log_prelaunch:debug(
+          "Loading the following plugins: ~p", [Plugins]),
+        %% We can load all plugins and refresh their feature flags at
+        %% once, because it does not involve running code from the
+        %% plugins.
+        ok = app_utils:load_applications(Plugins),
+        ok = rabbit_feature_flags:refresh_feature_flags_after_app_load(
+               Plugins),
+
+        rabbit_log_prelaunch:debug(""),
+        rabbit_log_prelaunch:debug("== Boot steps =="),
+
+        ok = rabbit_boot_steps:run_boot_steps([rabbit | Plugins]),
+        run_postlaunch_phase(Plugins),
+        rabbit_boot_state:set(core_started),
         {ok, SupPid}
     catch
         throw:{error, _} = Error ->
@@ -879,38 +897,31 @@ start(normal, []) ->
             Error
     end.
 
-run_postlaunch_phase() ->
-    spawn(fun() -> do_run_postlaunch_phase() end).
+run_postlaunch_phase(Plugins) ->
+    spawn(fun() -> do_run_postlaunch_phase(Plugins) end).
 
-do_run_postlaunch_phase() ->
+do_run_postlaunch_phase(Plugins) ->
     %% Once RabbitMQ itself is started, we need to run a few more steps,
     %% in particular start plugins.
     rabbit_log_prelaunch:debug(""),
     rabbit_log_prelaunch:debug("== Postlaunch phase =="),
 
     try
+        %% Successful boot resets node maintenance state.
         rabbit_log_prelaunch:debug(""),
-        rabbit_log_prelaunch:debug("== Plugins =="),
+        rabbit_log_prelaunch:info("Resetting node maintenance status"),
+        _ = rabbit_maintenance:unmark_as_being_drained(),
+
+        rabbit_log_prelaunch:debug(""),
+        rabbit_log_prelaunch:debug("== Plugins (postlaunch phase) =="),
 
-        rabbit_log_prelaunch:debug("Setting plugins up"),
-        %% `Plugins` contains all the enabled plugins, plus their
-        %% dependencies. The order is important: dependencies appear
-        %% before plugin which depend on them.
-        Plugins = rabbit_plugins:setup(),
-        rabbit_log_prelaunch:debug(
-          "Starting the following plugins: ~p", [Plugins]),
-        %% We can load all plugins and refresh their feature flags at
-        %% once, because it does not involve running code from the
-        %% plugins.
-        app_utils:load_applications(Plugins),
-        ok = rabbit_feature_flags:refresh_feature_flags_after_app_load(
-               Plugins),
         %% However, we want to run their boot steps and actually start
         %% them one by one, to ensure a dependency is fully started
         %% before a plugin which depends on it gets a chance to start.
+        rabbit_log_prelaunch:debug(
+          "Starting the following plugins: ~p", [Plugins]),
         lists:foreach(
           fun(Plugin) ->
-                  ok = rabbit_boot_steps:run_boot_steps([Plugin]),
                   case application:ensure_all_started(Plugin) of
                       {ok, _} -> ok;
                       Error   -> throw(Error)
@@ -918,7 +929,7 @@ do_run_postlaunch_phase() ->
           end, Plugins),
 
         %% Export definitions after all plugins have been enabled,
-        %% see rabbitmq/rabbitmq-server#2384
+        %% see rabbitmq/rabbitmq-server#2384.
         case rabbit_definitions:maybe_load_definitions() of
             ok           -> ok;
             DefLoadError -> throw(DefLoadError)
@@ -933,8 +944,9 @@ do_run_postlaunch_phase() ->
         %% The node is ready: mark it as such and log it.
         %% NOTE: PLEASE DO NOT ADD CRITICAL NODE STARTUP CODE AFTER THIS.
         ok = rabbit_lager:broker_is_started(),
-        ok = log_broker_started(
-               rabbit_plugins:strictly_plugins(rabbit_plugins:active())),
+        ActivePlugins = rabbit_plugins:active(),
+        StrictlyPlugins = rabbit_plugins:strictly_plugins(ActivePlugins),
+        ok = log_broker_started(StrictlyPlugins),
 
         rabbit_log_prelaunch:info("Resetting node maintenance status"),
         %% successful boot resets node maintenance state
diff --git a/deps/rabbit/src/rabbit_direct.erl b/deps/rabbit/src/rabbit_direct.erl
index 3fc2d75908..0506e1bf6c 100644
--- a/deps/rabbit/src/rabbit_direct.erl
+++ b/deps/rabbit/src/rabbit_direct.erl
@@ -75,7 +75,7 @@ auth_fun({Username, Password}, VHost, ExtraAuthProps) ->
 connect(Creds, VHost, Protocol, Pid, Infos) ->
     ExtraAuthProps = extract_extra_auth_props(Creds, VHost, Pid, Infos),
     AuthFun = auth_fun(Creds, VHost, ExtraAuthProps),
-    case rabbit:is_running() of
+    case rabbit_boot_state:has_reached_and_is_active(core_started) of
         true  ->
             case whereis(rabbit_direct_client_sup) of
                 undefined ->