Merge pull request #6737 from josevalim/jv-concurrent-app-dag

Start children applications concurrently
OTP-18451

4 files changed, 175 insertions, 51 deletions

diff --git a/lib/kernel/doc/src/application.xml b/lib/kernel/doc/src/application.xml
index d4d7810531..f787d3f27c 100644
--- a/lib/kernel/doc/src/application.xml
+++ b/lib/kernel/doc/src/application.xml
@@ -69,12 +69,21 @@
     <func>
       <name name="ensure_all_started" arity="1" since="OTP R16B02"/>
       <name name="ensure_all_started" arity="2" since="OTP R16B02"/>
-      <fsummary>Load and start an application and its dependencies, recursively.</fsummary>
+      <name name="ensure_all_started" arity="3" since="OTP 26.0"/>
+      <fsummary>Loads and starts all applications and their dependencies, recursively.</fsummary>
       <desc>
-        <p>Equivalent to calling
+        <p><c><anno>Applications</anno></c> is either an an <c>atom()</c> or a list
+            of <c>atom()</c> representing multiple applications.</p>
+        <p>This function is equivalent to calling
         <seemfa marker="#start/1"><c>start/1,2</c></seemfa>
-            repeatedly on all dependencies that are not yet started for an application.
+            repeatedly on all dependencies that are not yet started of each application.
             Optional dependencies will also be loaded and started if they are available.</p>
+        <p>The <c><anno>Mode</anno></c> argument controls if the applications should
+            be started in <c>serial</c> mode (one at a time) or <c>concurrent</c> mode.
+            In concurrent mode, a dependency graph is built and the leaves of the graph
+            are started concurrently and recursively. In both modes, no assertion can be
+            made about the order the applications are started. If not supplied, it defaults
+            to <c>serial</c>.</p>
         <p>Returns <c>{ok, AppNames}</c> for a successful start or for an already started
             application (which is, however, omitted from the <c>AppNames</c> list).</p>
         <p>The function reports <c>{error, {AppName,Reason}}</c> for errors, where
diff --git a/lib/kernel/src/application.erl b/lib/kernel/src/application.erl
index 4155537f50..093cde5c65 100644
--- a/lib/kernel/src/application.erl
+++ b/lib/kernel/src/application.erl
@@ -19,7 +19,8 @@
 %%
 -module(application).
 
--export([ensure_all_started/1, ensure_all_started/2, start/1, start/2,
+-export([ensure_all_started/1, ensure_all_started/2, ensure_all_started/3,
+	 start/1, start/2,
 	 start_boot/1, start_boot/2, stop/1, 
 	 load/1, load/2, unload/1, takeover/2,
 	 which_applications/0, which_applications/1,
@@ -117,28 +118,50 @@ unload(Application) ->
     application_controller:unload_application(Application).
 
 
--spec ensure_all_started(Application) -> {'ok', Started} | {'error', Reason} when
-      Application :: atom(),
+-spec ensure_all_started(Applications) -> {'ok', Started} | {'error', Reason} when
+      Applications :: atom() | [atom()],
       Started :: [atom()],
       Reason :: term().
 ensure_all_started(Application) ->
-    ensure_all_started(Application, temporary).
+    ensure_all_started(Application, temporary, serial).
 
--spec ensure_all_started(Application, Type) -> {'ok', Started} | {'error', Reason} when
-      Application :: atom(),
+-spec ensure_all_started(Applications, Type) -> {'ok', Started} | {'error', AppReason} when
+      Applications :: atom() | [atom()],
       Type :: restart_type(),
       Started :: [atom()],
-      Reason :: term().
+      AppReason :: {atom(), term()}.
 ensure_all_started(Application, Type) ->
-    case ensure_all_started([Application], [], Type, []) of
-	{ok, Started} ->
-	    {ok, lists:reverse(Started)};
-	{error, Reason, Started} ->
-	    _ = [stop(App) || App <- Started],
-	    {error, Reason}
+    ensure_all_started(Application, Type, serial).
+
+-spec ensure_all_started(Applications, Type, Mode) -> {'ok', Started} | {'error', AppReason} when
+      Applications :: atom() | [atom()],
+      Type :: restart_type(),
+      Mode :: serial | concurrent,
+      Started :: [atom()],
+      AppReason :: {atom(), term()}.
+ensure_all_started(Application, Type, Mode) when is_atom(Application) ->
+    ensure_all_started([Application], Type, Mode);
+ensure_all_started(Applications, Type, Mode) when is_list(Applications) ->
+    case ensure_all_enqueued(Applications, [], #{}, []) of
+        {ok, DAG, _Pending} ->
+            ForTraversal = maps:to_list(DAG),
+            case Mode of
+                concurrent ->
+                    ReqIDs = gen_server:reqids_new(),
+                    concurrent_dag_start(ForTraversal, ReqIDs, [], [], Type);
+                serial ->
+                    serial_dag_start(ForTraversal, [], [], [], Type)
+            end;
+        ErrorAppReason ->
+            ErrorAppReason
     end.
 
-ensure_all_started([App | Apps], OptionalApps, Type, Started) ->
+ensure_all_enqueued([App | Apps], Optional, DAG, Pending)
+  when is_map_key(App, DAG) ->
+    %% We already traversed the application, so only add it as pending
+    ensure_all_enqueued(Apps, Optional, DAG, [App | Pending]);
+
+ensure_all_enqueued([App | Apps], Optional, DAG, Pending) when is_atom(App) ->
     %% In case the app is already running, we just skip it instead
     %% of attempting to start all of its children - which would
     %% have already been loaded and started anyway.
@@ -146,44 +169,114 @@ ensure_all_started([App | Apps], OptionalApps, Type, Started) ->
         false ->
             case ensure_loaded(App) of
                 {ok, Name} ->
-                    case ensure_started(Name, App, Type, Started) of
-                        {ok, NewStarted} ->
-                            ensure_all_started(Apps, OptionalApps, Type, NewStarted);
-                        Error ->
-                            Error
+                    case enqueue_app(Name, App, DAG) of
+                        {ok, NewDAG} ->
+                            NewPending = [App | Pending],
+                            ensure_all_enqueued(Apps, Optional, NewDAG, NewPending);
+                        ErrorAppReason ->
+                            ErrorAppReason
                     end;
                 {error, {"no such file or directory", _} = Reason} ->
-                    case lists:member(App, OptionalApps) of
+                    case lists:member(App, Optional) of
                         true ->
-                            ensure_all_started(Apps, OptionalApps, Type, Started);
+                            ensure_all_enqueued(Apps, Optional, DAG, Pending);
                         false ->
-                            {error, {App, Reason}, Started}
+                            {error, {App, Reason}}
                     end;
                 {error, Reason} ->
-                    {error, {App, Reason}, Started}
+                    {error, {App, Reason}}
             end;
         true ->
-            ensure_all_started(Apps, OptionalApps, Type, Started)
+            ensure_all_enqueued(Apps, Optional, DAG, Pending)
     end;
-ensure_all_started([], _OptionalApps, _Type, Started) ->
-    {ok, Started}.
+ensure_all_enqueued([], _Optional, DAG, Pending) ->
+    {ok, DAG, Pending}.
 
-ensure_started(Name, App, Type, Started) ->
+enqueue_app(Name, App, DAG) ->
     {ok, ChildApps} = get_key(Name, applications),
     {ok, OptionalApps} = get_key(Name, optional_applications),
 
-    case ensure_all_started(ChildApps, OptionalApps, Type, Started) of
-	{ok, NewStarted} ->
-	    case application_controller:start_application(Name, Type) of
-		ok ->
-		    {ok, [App | NewStarted]};
-		{error, {already_started, App}} ->
-		    {ok, NewStarted};
-		{error, Reason} ->
-		    {error, {App, Reason}, NewStarted}
-	    end;
-	Error ->
-	    Error
+    case ensure_all_enqueued(ChildApps, OptionalApps, DAG, []) of
+        {ok, NewDAG, Pending} ->
+            {ok, NewDAG#{App => Pending}};
+        ErrorAppReason ->
+            ErrorAppReason
+    end.
+
+serial_dag_start([{App, Children} | Rest], Acc, Done, Started, Type) ->
+    case Children -- Done of
+        [] ->
+            case application_controller:start_application(App, Type) of
+                ok ->
+                    serial_dag_start(Rest, Acc, [App | Done], [App | Started], Type);
+                {error, {already_started, App}} ->
+                    serial_dag_start(Rest, Acc, [App | Done], Started, Type);
+                {error, Reason} ->
+                    _ = [stop(Name) || Name <- Started],
+                    {error, {App, Reason}}
+            end;
+        NewChildren ->
+            NewAcc = [{App, NewChildren} | Acc],
+            serial_dag_start(Rest, NewAcc, Done, Started, Type)
+    end;
+serial_dag_start([], [], _Done, Started, _Type) ->
+    {ok, lists:reverse(Started)};
+serial_dag_start([], Acc, Done, Started, Type) ->
+    serial_dag_start(Acc, [], Done, Started, Type).
+
+concurrent_dag_start([], ReqIDs, _Done, Started, _Type) ->
+    wait_all_enqueued(ReqIDs, Started, false);
+concurrent_dag_start(Pending0, ReqIDs0, Done, Started0, Type) ->
+    {Pending1, ReqIDs1} = enqueue_dag_leaves(Pending0, ReqIDs0, [], Done, Type),
+
+    case wait_one_enqueued(ReqIDs1, Started0) of
+        {ok, App, ReqIDs2, Started1} ->
+            concurrent_dag_start(Pending1, ReqIDs2, [App], Started1, Type);
+        {error, AppReason, ReqIDs2} ->
+            wait_all_enqueued(ReqIDs2, Started0, AppReason)
+    end.
+
+enqueue_dag_leaves([{App, Children} | Rest], ReqIDs, Acc, Done, Type) ->
+    case Children -- Done of
+        [] ->
+            Req = application_controller:start_application_request(App, Type),
+            NewReqIDs = gen_server:reqids_add(Req, App, ReqIDs),
+            enqueue_dag_leaves(Rest, NewReqIDs, Acc, Done, Type);
+        NewChildren ->
+            NewAcc = [{App, NewChildren} | Acc],
+            enqueue_dag_leaves(Rest, ReqIDs, NewAcc, Done, Type)
+    end;
+enqueue_dag_leaves([], ReqIDs, Acc, _Done, _Type) ->
+    {Acc, ReqIDs}.
+
+wait_one_enqueued(ReqIDs0, Started) ->
+    case gen_server:wait_response(ReqIDs0, infinity, true) of
+        {{reply, ok}, App, ReqIDs1} ->
+            {ok, App, ReqIDs1, [App | Started]};
+        {{reply, {error, {already_started, App}}}, App, ReqIDs1} ->
+            {ok, App, ReqIDs1, Started};
+        {{reply, {error, Reason}}, App, ReqIDs1} ->
+            {error, {App, Reason}, ReqIDs1};
+        {{error, {Reason, _Ref}}, _App, _ReqIDs1} ->
+            exit(Reason);
+        no_request ->
+            exit(deadlock)
+    end.
+
+wait_all_enqueued(ReqIDs0, Started0, LastAppReason) ->
+    case gen_server:reqids_size(ReqIDs0) of
+        0 when LastAppReason =:= false ->
+            {ok, lists:reverse(Started0)};
+        0 ->
+            _ = [stop(App) || App <- Started0],
+            {error, LastAppReason};
+        _ ->
+            case wait_one_enqueued(ReqIDs0, Started0) of
+                {ok, _App, ReqIDs1, Started1} ->
+                    wait_all_enqueued(ReqIDs1, Started1, LastAppReason);
+                {error, NewAppReason, ReqIDs1} ->
+                    wait_all_enqueued(ReqIDs1, Started0, NewAppReason)
+            end
     end.
 
 -spec start(Application) -> 'ok' | {'error', Reason} when
diff --git a/lib/kernel/src/application_controller.erl b/lib/kernel/src/application_controller.erl
index c5b4445bb8..58ae10320f 100644
--- a/lib/kernel/src/application_controller.erl
+++ b/lib/kernel/src/application_controller.erl
@@ -22,7 +22,8 @@
 %% External exports
 -export([start/1, 
 	 load_application/1, unload_application/1, 
-	 start_application/2, start_boot_application/2, stop_application/1,
+	 start_application/2, start_application_request/2,
+	 start_boot_application/2, stop_application/1,
 	 control_application/1, is_running/1,
 	 change_application_data/2, prep_config_change/0, config_change/1,
 	 which_applications/0, which_applications/1,
@@ -237,6 +238,9 @@ unload_application(AppName) ->
 start_application(AppName, RestartType) ->
     gen_server:call(?AC, {start_application, AppName, RestartType}, infinity).
 
+start_application_request(AppName, RestartType) ->
+    gen_server:send_request(?AC, {start_application, AppName, RestartType}).
+
 %%-----------------------------------------------------------------
 %% Func: is_running/1
 %% Args: Application = atom()
diff --git a/lib/kernel/test/application_SUITE.erl b/lib/kernel/test/application_SUITE.erl
index 18829d91ec..623e72bd4e 100644
--- a/lib/kernel/test/application_SUITE.erl
+++ b/lib/kernel/test/application_SUITE.erl
@@ -960,9 +960,13 @@ ensure_started(_Conf) ->
     ok = application:unload(app1),
     ok.
 
-%% Test application:ensure_all_started/1-2.
+%% Test application:ensure_all_started/1-2-3.
 ensure_all_started(_Conf) ->
+    do_ensure_all_started(serial),
+    do_ensure_all_started(concurrent),
+    ok.
 
+do_ensure_all_started(Mode) ->
     {ok, Fd1} = file:open("app1.app", [write]),
     w_app1(Fd1),
     file:close(Fd1),
@@ -981,9 +985,10 @@ ensure_all_started(_Conf) ->
 
     %% Single app start/stop
     false = lists:keyfind(app1, 1, application:which_applications()),
-    {ok, [app1]} = application:ensure_all_started(app1), % app1 started
+    {ok, [app1]} = application:ensure_all_started(app1, temporary, Mode), % app1 started
     {app1, _, _} = lists:keyfind(app1, 1, application:which_applications()),
-    {ok, []} = application:ensure_all_started(app1), % no start needed
+    {ok, []} = application:ensure_all_started(app1, temporary), % no start needed
+    {ok, []} = application:ensure_all_started(app1, permanent), % no start needed
     ok = application:stop(app1),
     false = lists:keyfind(app1, 1, application:which_applications()),
     ok = application:unload(app1),
@@ -995,13 +1000,26 @@ ensure_all_started(_Conf) ->
 
     %% Start dependencies.
     {error, {not_started, app9}} = application:start(app10),
-    {ok, [app9,app10]} = application:ensure_all_started(app10, temporary),
+    {ok, [app9,app10]} = application:ensure_all_started(app10, temporary, Mode),
     {app9, _, _} = lists:keyfind(app9, 1, application:which_applications()),
     {app10, _, _} = lists:keyfind(app10, 1, application:which_applications()),
     %% Only report apps/dependencies that actually needed to start
     ok = application:stop(app10),
     ok = application:unload(app10),
-    {ok, [app10]} = application:ensure_all_started(app10, temporary),
+    {ok, [app10]} = application:ensure_all_started(app10, temporary, Mode),
+    ok = application:stop(app9),
+    ok = application:unload(app9),
+    ok = application:stop(app10),
+    ok = application:unload(app10),
+
+    %% Starts several
+    {ok, StartedSeveral} = application:ensure_all_started([app1, app10], temporary, Mode),
+    [app1,app10,app9] = lists:sort(StartedSeveral),
+    {app1, _, _} = lists:keyfind(app1, 1, application:which_applications()),
+    {app9, _, _} = lists:keyfind(app9, 1, application:which_applications()),
+    {app10, _, _} = lists:keyfind(app10, 1, application:which_applications()),
+    ok = application:stop(app1),
+    ok = application:unload(app1),
     ok = application:stop(app9),
     ok = application:unload(app9),
     ok = application:stop(app10),
@@ -1016,16 +1034,16 @@ ensure_all_started(_Conf) ->
     %% nor app10 running after failing to start
     %% hopefully_not_an_existing_app
     {error, {hopefully_not_an_existing_app, {"no such file or directory", _}}}=
-	application:ensure_all_started(app_chain_error),
+	application:ensure_all_started(app_chain_error, temporary, Mode),
     false = lists:keyfind(app9, 1, application:which_applications()),
     false = lists:keyfind(app10, 1, application:which_applications()),
-    false = lists:keyfind(app_chain_error2,1,application:which_applications()),
+    false = lists:keyfind(app_chain_error2, 1, application:which_applications()),
     false = lists:keyfind(app_chain_error, 1, application:which_applications()),
     %% Here we will have app9 already running, and app10 should be
     %% able to boot fine.
     %% In this dependency failing, we expect app9 to still be running, but
     %% not app10 after failing to start hopefully_not_an_existing_app
-    {ok, [app9]} = application:ensure_all_started(app9, temporary),
+    {ok, [app9]} = application:ensure_all_started(app9, temporary, Mode),
     {error, {hopefully_not_an_existing_app, {"no such file or directory", _}}}=
 	application:ensure_all_started(app_chain_error),
     {app9, _, _} = lists:keyfind(app9, 1, application:which_applications()),