Start children applications concurrently

We build a DAG with all application dependencies
and start leaves recursively. As each application
starts, we further traverse the graph looking for
more leaves.

In order to maximize this new feature, we also
allow a list of applications to be given to
application:ensure_all_started/3.

4 files changed, 175 insertions, 51 deletions

diff --git a/lib/kernel/doc/src/application.xml b/lib/kernel/doc/src/application.xml
index 2a807e62bb..e62ac2606e 100644
--- a/lib/kernel/doc/src/application.xml
+++ b/lib/kernel/doc/src/application.xml
@@ -69,12 +69,21 @@
     <func>
       <name name="ensure_all_started" arity="1" since="OTP R16B02"/>
       <name name="ensure_all_started" arity="2" since="OTP R16B02"/>
-      <fsummary>Load and start an application and its dependencies, recursively.</fsummary>
+      <name name="ensure_all_started" arity="3" since="OTP 26.0"/>
+      <fsummary>Loads and starts all applications and their dependencies, recursively.</fsummary>
       <desc>
-        <p>Equivalent to calling
+        <p><c><anno>Applications</anno></c> is either an an <c>atom()</c> or a list
+            of <c>atom()</c> representing multiple applications.</p>
+        <p>This function is equivalent to calling
         <seemfa marker="#start/1"><c>start/1,2</c></seemfa>
-            repeatedly on all dependencies that are not yet started for an application.
+            repeatedly on all dependencies that are not yet started of each application.
             Optional dependencies will also be loaded and started if they are available.</p>
+        <p>The <c><anno>Mode</anno></c> argument controls if the applications should
+            be started in <c>serial</c> mode (one at a time) or <c>concurrent</c> mode.
+            In concurrent mode, a dependency graph is built and the leaves of the graph
+            are started concurrently and recursively. In both modes, no assertion can be
+            made about the order the applications are started. If not supplied, it defaults
+            to <c>serial</c>.</p>
         <p>Returns <c>{ok, AppNames}</c> for a successful start or for an already started
             application (which is, however, omitted from the <c>AppNames</c> list).</p>
         <p>The function reports <c>{error, {AppName,Reason}}</c> for errors, where
diff --git a/lib/kernel/src/application.erl b/lib/kernel/src/application.erl
index 1730e1e822..ad62a6e832 100644
--- a/lib/kernel/src/application.erl
+++ b/lib/kernel/src/application.erl
@@ -19,7 +19,8 @@
 %%
 -module(application).
 
--export([ensure_all_started/1, ensure_all_started/2, start/1, start/2,
+-export([ensure_all_started/1, ensure_all_started/2, ensure_all_started/3,
+	 start/1, start/2,
 	 start_boot/1, start_boot/2, stop/1, 
 	 load/1, load/2, unload/1, takeover/2,
 	 which_applications/0, which_applications/1,
@@ -117,28 +118,50 @@ unload(Application) ->
     application_controller:unload_application(Application).
 
 
--spec ensure_all_started(Application) -> {'ok', Started} | {'error', Reason} when
-      Application :: atom(),
+-spec ensure_all_started(Applications) -> {'ok', Started} | {'error', Reason} when
+      Applications :: atom() | [atom()],
       Started :: [atom()],
       Reason :: term().
 ensure_all_started(Application) ->
-    ensure_all_started(Application, temporary).
+    ensure_all_started(Application, temporary, serial).
 
--spec ensure_all_started(Application, Type) -> {'ok', Started} | {'error', Reason} when
-      Application :: atom(),
+-spec ensure_all_started(Applications, Type) -> {'ok', Started} | {'error', AppReason} when
+      Applications :: atom() | [atom()],
       Type :: restart_type(),
       Started :: [atom()],
-      Reason :: term().
+      AppReason :: {atom(), term()}.
 ensure_all_started(Application, Type) ->
-    case ensure_all_started([Application], [], Type, []) of
-	{ok, Started} ->
-	    {ok, lists:reverse(Started)};
-	{error, Reason, Started} ->
-	    _ = [stop(App) || App <- Started],
-	    {error, Reason}
+    ensure_all_started(Application, Type, serial).
+
+-spec ensure_all_started(Applications, Type, Mode) -> {'ok', Started} | {'error', AppReason} when
+      Applications :: atom() | [atom()],
+      Type :: restart_type(),
+      Mode :: serial | concurrent,
+      Started :: [atom()],
+      AppReason :: {atom(), term()}.
+ensure_all_started(Application, Type, Mode) when is_atom(Application) ->
+    ensure_all_started([Application], Type, Mode);
+ensure_all_started(Applications, Type, Mode) when is_list(Applications) ->
+    case ensure_all_enqueued(Applications, [], #{}, []) of
+        {ok, DAG, _Pending} ->
+            ForTraversal = maps:to_list(DAG),
+            case Mode of
+                concurrent ->
+                    ReqIDs = gen_server:reqids_new(),
+                    concurrent_dag_start(ForTraversal, ReqIDs, [], [], Type);
+                serial ->
+                    serial_dag_start(ForTraversal, [], [], [], Type)
+            end;
+        ErrorAppReason ->
+            ErrorAppReason
     end.
 
-ensure_all_started([App | Apps], OptionalApps, Type, Started) ->
+ensure_all_enqueued([App | Apps], Optional, DAG, Pending)
+  when is_map_key(App, DAG) ->
+    %% We already traversed the application, so only add it as pending
+    ensure_all_enqueued(Apps, Optional, DAG, [App | Pending]);
+
+ensure_all_enqueued([App | Apps], Optional, DAG, Pending) when is_atom(App) ->
     %% In case the app is already running, we just skip it instead
     %% of attempting to start all of its children - which would
     %% have already been loaded and started anyway.
@@ -146,44 +169,114 @@ ensure_all_started([App | Apps], OptionalApps, Type, Started) ->
         false ->
             case ensure_loaded(App) of
                 {ok, Name} ->
-                    case ensure_started(Name, App, Type, Started) of
-                        {ok, NewStarted} ->
-                            ensure_all_started(Apps, OptionalApps, Type, NewStarted);
-                        Error ->
-                            Error
+                    case enqueue_app(Name, App, DAG) of
+                        {ok, NewDAG} ->
+                            NewPending = [App | Pending],
+                            ensure_all_enqueued(Apps, Optional, NewDAG, NewPending);
+                        ErrorAppReason ->
+                            ErrorAppReason
                     end;
                 {error, {"no such file or directory", _} = Reason} ->
-                    case lists:member(App, OptionalApps) of
+                    case lists:member(App, Optional) of
                         true ->
-                            ensure_all_started(Apps, OptionalApps, Type, Started);
+                            ensure_all_enqueued(Apps, Optional, DAG, Pending);
                         false ->
-                            {error, {App, Reason}, Started}
+                            {error, {App, Reason}}
                     end;
                 {error, Reason} ->
-                    {error, {App, Reason}, Started}
+                    {error, {App, Reason}}
             end;
         true ->
-            ensure_all_started(Apps, OptionalApps, Type, Started)
+            ensure_all_enqueued(Apps, Optional, DAG, Pending)
     end;
-ensure_all_started([], _OptionalApps, _Type, Started) ->
-    {ok, Started}.
+ensure_all_enqueued([], _Optional, DAG, Pending) ->
+    {ok, DAG, Pending}.
 
-ensure_started(Name, App, Type, Started) ->
+enqueue_app(Name, App, DAG) ->
     {ok, ChildApps} = get_key(Name, applications),
     {ok, OptionalApps} = get_key(Name, optional_applications),
 
-    case ensure_all_started(ChildApps, OptionalApps, Type, Started) of
-	{ok, NewStarted} ->
-	    case application_controller:start_application(Name, Type) of
-		ok ->
-		    {ok, [App | NewStarted]};
-		{error, {already_started, App}} ->
-		    {ok, NewStarted};
-		{error, Reason} ->
-		    {error, {App, Reason}, NewStarted}
-	    end;
-	Error ->
-	    Error
+    case ensure_all_enqueued(ChildApps, OptionalApps, DAG, []) of
+        {ok, NewDAG, Pending} ->
+            {ok, NewDAG#{App => Pending}};
+        ErrorAppReason ->
+            ErrorAppReason
+    end.
+
+serial_dag_start([{App, Children} | Rest], Acc, Done, Started, Type) ->
+    case Children -- Done of
+        [] ->
+            case application_controller:start_application(App, Type) of
+                ok ->
+                    serial_dag_start(Rest, Acc, [App | Done], [App | Started], Type);
+                {error, {already_started, App}} ->
+                    serial_dag_start(Rest, Acc, [App | Done], Started, Type);
+                {error, Reason} ->
+                    _ = [stop(Name) || Name <- Started],
+                    {error, {App, Reason}}
+            end;
+        NewChildren ->
+            NewAcc = [{App, NewChildren} | Acc],
+            serial_dag_start(Rest, NewAcc, Done, Started, Type)
+    end;
+serial_dag_start([], [], _Done, Started, _Type) ->
+    {ok, lists:reverse(Started)};
+serial_dag_start([], Acc, Done, Started, Type) ->
+    serial_dag_start(Acc, [], Done, Started, Type).
+
+concurrent_dag_start([], ReqIDs, _Done, Started, _Type) ->
+    wait_all_enqueued(ReqIDs, Started, false);
+concurrent_dag_start(Pending0, ReqIDs0, Done, Started0, Type) ->
+    {Pending1, ReqIDs1} = enqueue_dag_leaves(Pending0, ReqIDs0, [], Done, Type),
+
+    case wait_one_enqueued(ReqIDs1, Started0) of
+        {ok, App, ReqIDs2, Started1} ->
+            concurrent_dag_start(Pending1, ReqIDs2, [App], Started1, Type);
+        {error, AppReason, ReqIDs2} ->
+            wait_all_enqueued(ReqIDs2, Started0, AppReason)
+    end.
+
+enqueue_dag_leaves([{App, Children} | Rest], ReqIDs, Acc, Done, Type) ->
+    case Children -- Done of
+        [] ->
+            Req = application_controller:start_application_request(App, Type),
+            NewReqIDs = gen_server:reqids_add(Req, App, ReqIDs),
+            enqueue_dag_leaves(Rest, NewReqIDs, Acc, Done, Type);
+        NewChildren ->
+            NewAcc = [{App, NewChildren} | Acc],
+            enqueue_dag_leaves(Rest, ReqIDs, NewAcc, Done, Type)
+    end;
+enqueue_dag_leaves([], ReqIDs, Acc, _Done, _Type) ->
+    {Acc, ReqIDs}.
+
+wait_one_enqueued(ReqIDs0, Started) ->
+    case gen_server:wait_response(ReqIDs0, infinity, true) of
+        {{reply, ok}, App, ReqIDs1} ->
+            {ok, App, ReqIDs1, [App | Started]};
+        {{reply, {error, {already_started, App}}}, App, ReqIDs1} ->
+            {ok, App, ReqIDs1, Started};
+        {{reply, {error, Reason}}, App, ReqIDs1} ->
+            {error, {App, Reason}, ReqIDs1};
+        {{error, {Reason, _Ref}}, _App, _ReqIDs1} ->
+            exit(Reason);
+        no_request ->
+            exit(deadlock)
+    end.
+
+wait_all_enqueued(ReqIDs0, Started0, LastAppReason) ->
+    case gen_server:reqids_size(ReqIDs0) of
+        0 when LastAppReason =:= false ->
+            {ok, lists:reverse(Started0)};
+        0 ->
+            _ = [stop(App) || App <- Started0],
+            {error, LastAppReason};
+        _ ->
+            case wait_one_enqueued(ReqIDs0, Started0) of
+                {ok, _App, ReqIDs1, Started1} ->
+                    wait_all_enqueued(ReqIDs1, Started1, LastAppReason);
+                {error, NewAppReason, ReqIDs1} ->
+                    wait_all_enqueued(ReqIDs1, Started0, NewAppReason)
+            end
     end.
 
 -spec start(Application) -> 'ok' | {'error', Reason} when
diff --git a/lib/kernel/src/application_controller.erl b/lib/kernel/src/application_controller.erl
index c5b4445bb8..58ae10320f 100644
--- a/lib/kernel/src/application_controller.erl
+++ b/lib/kernel/src/application_controller.erl
@@ -22,7 +22,8 @@
 %% External exports
 -export([start/1, 
 	 load_application/1, unload_application/1, 
-	 start_application/2, start_boot_application/2, stop_application/1,
+	 start_application/2, start_application_request/2,
+	 start_boot_application/2, stop_application/1,
 	 control_application/1, is_running/1,
 	 change_application_data/2, prep_config_change/0, config_change/1,
 	 which_applications/0, which_applications/1,
@@ -237,6 +238,9 @@ unload_application(AppName) ->
 start_application(AppName, RestartType) ->
     gen_server:call(?AC, {start_application, AppName, RestartType}, infinity).
 
+start_application_request(AppName, RestartType) ->
+    gen_server:send_request(?AC, {start_application, AppName, RestartType}).
+
 %%-----------------------------------------------------------------
 %% Func: is_running/1
 %% Args: Application = atom()
diff --git a/lib/kernel/test/application_SUITE.erl b/lib/kernel/test/application_SUITE.erl
index ae69d9874f..893d368bb9 100644
--- a/lib/kernel/test/application_SUITE.erl
+++ b/lib/kernel/test/application_SUITE.erl
@@ -960,9 +960,13 @@ ensure_started(_Conf) ->
     ok = application:unload(app1),
     ok.
 
-%% Test application:ensure_all_started/1-2.
+%% Test application:ensure_all_started/1-2-3.
 ensure_all_started(_Conf) ->
+    do_ensure_all_started(serial),
+    do_ensure_all_started(concurrent),
+    ok.
 
+do_ensure_all_started(Mode) ->
     {ok, Fd1} = file:open("app1.app", [write]),
     w_app1(Fd1),
     file:close(Fd1),
@@ -981,9 +985,10 @@ ensure_all_started(_Conf) ->
 
     %% Single app start/stop
     false = lists:keyfind(app1, 1, application:which_applications()),
-    {ok, [app1]} = application:ensure_all_started(app1), % app1 started
+    {ok, [app1]} = application:ensure_all_started(app1, temporary, Mode), % app1 started
     {app1, _, _} = lists:keyfind(app1, 1, application:which_applications()),
-    {ok, []} = application:ensure_all_started(app1), % no start needed
+    {ok, []} = application:ensure_all_started(app1, temporary), % no start needed
+    {ok, []} = application:ensure_all_started(app1, permanent), % no start needed
     ok = application:stop(app1),
     false = lists:keyfind(app1, 1, application:which_applications()),
     ok = application:unload(app1),
@@ -995,13 +1000,26 @@ ensure_all_started(_Conf) ->
 
     %% Start dependencies.
     {error, {not_started, app9}} = application:start(app10),
-    {ok, [app9,app10]} = application:ensure_all_started(app10, temporary),
+    {ok, [app9,app10]} = application:ensure_all_started(app10, temporary, Mode),
     {app9, _, _} = lists:keyfind(app9, 1, application:which_applications()),
     {app10, _, _} = lists:keyfind(app10, 1, application:which_applications()),
     %% Only report apps/dependencies that actually needed to start
     ok = application:stop(app10),
     ok = application:unload(app10),
-    {ok, [app10]} = application:ensure_all_started(app10, temporary),
+    {ok, [app10]} = application:ensure_all_started(app10, temporary, Mode),
+    ok = application:stop(app9),
+    ok = application:unload(app9),
+    ok = application:stop(app10),
+    ok = application:unload(app10),
+
+    %% Starts several
+    {ok, StartedSeveral} = application:ensure_all_started([app1, app10], temporary, Mode),
+    [app1,app10,app9] = lists:sort(StartedSeveral),
+    {app1, _, _} = lists:keyfind(app1, 1, application:which_applications()),
+    {app9, _, _} = lists:keyfind(app9, 1, application:which_applications()),
+    {app10, _, _} = lists:keyfind(app10, 1, application:which_applications()),
+    ok = application:stop(app1),
+    ok = application:unload(app1),
     ok = application:stop(app9),
     ok = application:unload(app9),
     ok = application:stop(app10),
@@ -1016,16 +1034,16 @@ ensure_all_started(_Conf) ->
     %% nor app10 running after failing to start
     %% hopefully_not_an_existing_app
     {error, {hopefully_not_an_existing_app, {"no such file or directory", _}}}=
-	application:ensure_all_started(app_chain_error),
+	application:ensure_all_started(app_chain_error, temporary, Mode),
     false = lists:keyfind(app9, 1, application:which_applications()),
     false = lists:keyfind(app10, 1, application:which_applications()),
-    false = lists:keyfind(app_chain_error2,1,application:which_applications()),
+    false = lists:keyfind(app_chain_error2, 1, application:which_applications()),
     false = lists:keyfind(app_chain_error, 1, application:which_applications()),
     %% Here we will have app9 already running, and app10 should be
     %% able to boot fine.
     %% In this dependency failing, we expect app9 to still be running, but
     %% not app10 after failing to start hopefully_not_an_existing_app
-    {ok, [app9]} = application:ensure_all_started(app9, temporary),
+    {ok, [app9]} = application:ensure_all_started(app9, temporary, Mode),
     {error, {hopefully_not_an_existing_app, {"no such file or directory", _}}}=
 	application:ensure_all_started(app_chain_error),
     {app9, _, _} = lists:keyfind(app9, 1, application:which_applications()),