Automatically recompile dependencies if compile env changes

10 files changed, 143 insertions, 105 deletions

diff --git a/lib/elixir/lib/config/provider.ex b/lib/elixir/lib/config/provider.ex
index c8e27df09..cfd7d16b1 100644
--- a/lib/elixir/lib/config/provider.ex
+++ b/lib/elixir/lib/config/provider.ex
@@ -274,6 +274,17 @@ defmodule Config.Provider do
   end
 
   @doc false
+  def valid_compile_env?(compile_env) do
+    Enum.all?(compile_env, fn {app, [key | path], compile_return} ->
+      try do
+        traverse_env(Application.fetch_env(app, key), path) == compile_return
+      rescue
+        _ -> false
+      end
+    end)
+  end
+
+  @doc false
   def validate_compile_env(compile_env, ensure_loaded? \\ true)
 
   def validate_compile_env([{app, [key | path], compile_return} | compile_env], ensure_loaded?) do
diff --git a/lib/mix/lib/mix.ex b/lib/mix/lib/mix.ex
index dba6f114f..ae8c6c44b 100644
--- a/lib/mix/lib/mix.ex
+++ b/lib/mix/lib/mix.ex
@@ -593,7 +593,7 @@ defmodule Mix do
   """
   def ensure_application!(app) when is_atom(app) do
     case Mix.State.builtin_apps() do
-      %{^app => {:ebin, path}} ->
+      %{^app => path} ->
         Code.prepend_path(path, cache: true)
 
       %{} ->
diff --git a/lib/mix/lib/mix/app_loader.ex b/lib/mix/lib/mix/app_loader.ex
index 1c98473f1..fdb0fb61d 100644
--- a/lib/mix/lib/mix/app_loader.ex
+++ b/lib/mix/lib/mix/app_loader.ex
@@ -52,51 +52,43 @@ defmodule Mix.AppLoader do
   end
 
   @doc """
-  Loads the given application from `ebin_path`.
-
-  Returns either `{:ok, children}` or `{:error, message}`.
+  Loads the given app from path in an optimized format and returns its contents.
   """
-  def load_app(app, ebin_path, validate_compile_env?) do
-    if Application.spec(app, :vsn) do
-      {:ok, children(app)}
-    else
-      with true <- ebin_path != nil,
-           {:ok, bin} <- File.read(app_join(ebin_path, app, ~c".app")),
-           {:ok, {:application, _, properties} = application_data} <- consult_app_file(bin),
-           :ok <- :application.load(application_data) do
-        with [_ | _] = compile_env <- validate_compile_env? && properties[:compile_env],
-             {:error, message} <- Config.Provider.validate_compile_env(compile_env, false) do
-          {:error, message}
+  def load_app(app, app_path) do
+    case File.read(app_path) do
+      {:ok, bin} ->
+        with {:ok, tokens, _} <- :erl_scan.string(String.to_charlist(bin)),
+             {:ok, {:application, ^app, properties} = app_data} <- :erl_parse.parse_term(tokens),
+             :ok <- ensure_loaded(app_data) do
+          {:ok, properties}
         else
-          _ -> {:ok, children(app)}
+          _ -> :invalid
         end
-      else
-        # Optional applications won't be available
-        _ -> {:ok, []}
-      end
+
+      {:error, _} ->
+        :missing
+    end
+  end
+
+  defp ensure_loaded(app_data) do
+    case :application.load(app_data) do
+      :ok -> :ok
+      {:error, {:already_loaded, _}} -> :ok
+      {:error, error} -> {:error, error}
     end
   end
 
   @doc """
   Loads the given applications.
   """
-  def load_apps(apps, deps, config, validate_compile_env?, acc, fun) do
+  def load_apps(apps, deps, config, acc, fun) do
     lib_path = to_charlist(Path.join(Mix.Project.build_path(config), "lib"))
     deps_children = for dep <- deps, into: %{}, do: {dep.app, Enum.map(dep.deps, & &1.app)}
-    deps_paths = for dep <- deps, into: %{}, do: {dep.app, {:lib, lib_path}}
     builtin_paths = Mix.State.builtin_apps()
-    paths = Map.merge(builtin_paths, deps_paths)
-
-    ref = make_ref()
-    parent = self()
-    opts = [ordered: false, timeout: :infinity]
 
-    stream =
-      (extra_apps(config) ++ apps)
-      |> stream_apps(deps_children, paths, ref)
-      |> Task.async_stream(&load_stream_app(&1, ref, parent, validate_compile_env?), opts)
-
-    Enum.reduce(stream, acc, fn {:ok, res}, acc -> fun.(res, acc) end)
+    (extra_apps(config) ++ apps)
+    |> traverse_apps(%{}, deps_children, builtin_paths, lib_path)
+    |> Enum.reduce(acc, fun)
   end
 
   defp extra_apps(config) do
@@ -106,60 +98,47 @@ defmodule Mix.AppLoader do
     end
   end
 
-  defp load_stream_app({app, app_path}, ref, parent, validate_compile_env?) do
-    ebin_path = app_path_to_ebin_path(app, app_path)
-    send(parent, {ref, app, load_app(app, ebin_path, validate_compile_env?)})
-    {app, ebin_path}
-  end
-
-  defp stream_apps(initial, deps_children, paths, ref) do
-    Stream.unfold({initial, %{}, %{}, deps_children, paths, ref}, &stream_app/1)
-  end
-
   # We already processed this app, skip it.
-  defp stream_app({[app | apps], seen, done, deps_children, paths, ref})
+  defp traverse_apps([app | apps], seen, deps_children, builtin_paths, lib_path)
        when is_map_key(seen, app) do
-    stream_app({apps, seen, done, deps_children, paths, ref})
+    traverse_apps(apps, seen, deps_children, builtin_paths, lib_path)
   end
 
   # We haven't processed this app, emit it.
-  defp stream_app({[app | apps], seen, done, deps_children, paths, ref}) do
-    {{app, paths[app]}, {apps, Map.put(seen, app, true), done, deps_children, paths, ref}}
-  end
+  defp traverse_apps([app | apps], seen, deps_children, builtin_paths, lib_path) do
+    {ebin_path, dep_children} =
+      case deps_children do
+        %{^app => dep_children} -> {app_join(lib_path, app, ~c"/ebin"), dep_children}
+        _ -> {builtin_paths[app], []}
+      end
 
-  # We have processed all apps and all seen have been done.
-  defp stream_app({[], seen, done, _deps_children, _paths, _ref})
-       when map_size(seen) == map_size(done) do
-    nil
-  end
+    app_children =
+      if Application.spec(app, :vsn) do
+        app_children(app)
+      else
+        with true <- ebin_path != nil,
+             {:ok, _} <- load_app(app, app_join(ebin_path, app, ~c".app")) do
+          app_children(app)
+        else
+          # Optional applications won't be available
+          _ -> []
+        end
+      end
 
-  # We have processed all apps but there is work being done.
-  defp stream_app({[], seen, done, deps_children, paths, ref}) do
-    receive do
-      {^ref, app, {:ok, children}} ->
-        dep_children = Map.get(deps_children, app, [])
-        children = (dep_children -- children) ++ children
-        stream_app({children, seen, Map.put(done, app, true), deps_children, paths, ref})
+    children = (dep_children -- app_children) ++ app_children
+    seen = Map.put(seen, app, true)
+    apps = children ++ apps
+    [{app, ebin_path} | traverse_apps(apps, seen, deps_children, builtin_paths, lib_path)]
+  end
 
-      {^ref, _app, {:error, message}} ->
-        Mix.raise(message)
-    end
+  # We have processed all apps.
+  defp traverse_apps([], _seen, _deps_children, _builtin_paths, _lib_path) do
+    []
   end
 
-  defp children(app) do
+  defp app_children(app) do
     Application.spec(app, :applications) ++ Application.spec(app, :included_applications)
   end
 
-  defp app_path_to_ebin_path(app, {:lib, lib_path}), do: app_join(lib_path, app, ~c"/ebin")
-  defp app_path_to_ebin_path(_app, {:ebin, ebin_path}), do: ebin_path
-  defp app_path_to_ebin_path(_app, nil), do: nil
-
   defp app_join(path, app, suffix), do: path ++ ~c"/" ++ Atom.to_charlist(app) ++ suffix
-
-  defp consult_app_file(bin) do
-    # The path could be located in an .ez archive, so we use the prim loader.
-    with {:ok, tokens, _} <- :erl_scan.string(String.to_charlist(bin)) do
-      :erl_parse.parse_term(tokens)
-    end
-  end
 end
diff --git a/lib/mix/lib/mix/dep/converger.ex b/lib/mix/lib/mix/dep/converger.ex
index 7ae5f4928..87adb3206 100644
--- a/lib/mix/lib/mix/dep/converger.ex
+++ b/lib/mix/lib/mix/dep/converger.ex
@@ -108,7 +108,7 @@ defmodule Mix.Dep.Converger do
     use_remote? = !!remote and Enum.any?(deps, &remote.remote?/1)
 
     if not diverged? and use_remote? do
-      # Make sure there are no cycles before calling remote converge
+      # Make sure there are no cycles before calling the remote converger
       topological_sort(deps)
 
       # If there is a lock, it means we are doing a get/update
diff --git a/lib/mix/lib/mix/dep/loader.ex b/lib/mix/lib/mix/dep/loader.ex
index 11e707011..31aa9d503 100644
--- a/lib/mix/lib/mix/dep/loader.ex
+++ b/lib/mix/lib/mix/dep/loader.ex
@@ -411,14 +411,14 @@ defmodule Mix.Dep.Loader do
   end
 
   defp app_status(app_path, app, req) do
-    case :file.consult(app_path) do
-      {:ok, [{:application, ^app, config}]} ->
-        case List.keyfind(config, :vsn, 0) do
+    case Mix.AppLoader.load_app(app, app_path) do
+      {:ok, properties} ->
+        case List.keyfind(properties, :vsn, 0) do
           {:vsn, actual} when is_list(actual) ->
             actual = IO.iodata_to_binary(actual)
 
             case vsn_match(req, actual, app) do
-              {:ok, true} -> {:ok, actual}
+              {:ok, true} -> compile_env_status(actual, properties)
               {:ok, false} -> {:nomatchvsn, actual}
               {:error, error} -> {error, actual}
             end
@@ -430,14 +430,23 @@ defmodule Mix.Dep.Loader do
             {:invalidvsn, nil}
         end
 
-      {:ok, _} ->
+      :invalid ->
         {:invalidapp, app_path}
 
-      {:error, _} ->
+      :missing ->
         case Path.wildcard(Path.join(Path.dirname(app_path), "*.app")) do
           [other_app_path] -> {:noappfile, {app_path, other_app_path}}
           _ -> {:noappfile, {app_path, nil}}
         end
     end
   end
+
+  defp compile_env_status(vsn, properties) do
+    with [_ | _] = compile_env <- properties[:compile_env],
+         false <- Config.Provider.valid_compile_env?(compile_env) do
+      :compile
+    else
+      _ -> {:ok, vsn}
+    end
+  end
 end
diff --git a/lib/mix/lib/mix/state.ex b/lib/mix/lib/mix/state.ex
index f97a50148..19a18f987 100644
--- a/lib/mix/lib/mix/state.ex
+++ b/lib/mix/lib/mix/state.ex
@@ -107,7 +107,7 @@ defmodule Mix.State do
       builtin_apps =
         for path <- builtin_apps,
             app = app_from_code_path(path),
-            do: {app, {:ebin, path}},
+            do: {app, path},
             into: %{}
 
       {:reply, builtin_apps, %{state | builtin_apps: builtin_apps}}
diff --git a/lib/mix/lib/mix/tasks/compile.all.ex b/lib/mix/lib/mix/tasks/compile.all.ex
index ddc83bb19..d48e17dc3 100644
--- a/lib/mix/lib/mix/tasks/compile.all.ex
+++ b/lib/mix/lib/mix/tasks/compile.all.ex
@@ -27,14 +27,12 @@ defmodule Mix.Tasks.Compile.All do
     # from archives will be removed from the code path.
     deps = Mix.Dep.cached()
     apps = project_apps(config)
-    validate_compile_env? = "--no-validate-compile-env" not in args
 
     {loaded_paths, loaded_modules} =
-      Mix.AppLoader.load_apps(apps, deps, config, validate_compile_env?, {[], []}, fn
-        {app, path}, {paths, mods} ->
-          paths = if path, do: [path | paths], else: paths
-          mods = if app_cache, do: [{app, Application.spec(app, :modules)} | mods], else: mods
-          {paths, mods}
+      Mix.AppLoader.load_apps(apps, deps, config, {[], []}, fn {app, path}, {paths, mods} ->
+        paths = if path, do: [path | paths], else: paths
+        mods = if app_cache, do: [{app, Application.spec(app, :modules)} | mods], else: mods
+        {paths, mods}
       end)
 
     # We compute the diff as that will be more efficient
@@ -73,8 +71,12 @@ defmodule Mix.Tasks.Compile.All do
     _ = Code.prepend_path(compile_path)
 
     unless "--no-app-loading" in args do
-      with {:error, message} <-
-             Mix.AppLoader.load_app(config[:app], compile_path, validate_compile_env?) do
+      app = config[:app]
+
+      with {:ok, properties} <- Mix.AppLoader.load_app(app, "#{compile_path}/#{app}.app"),
+           false <- "--no-validate-compile-env" in args,
+           [_ | _] = compile_env <- properties[:compile_env],
+           {:error, message} <- Config.Provider.validate_compile_env(compile_env, false) do
         Mix.raise(message)
       end
     end
diff --git a/lib/mix/lib/mix/tasks/compile.ex b/lib/mix/lib/mix/tasks/compile.ex
index b884f07d7..758bed682 100644
--- a/lib/mix/lib/mix/tasks/compile.ex
+++ b/lib/mix/lib/mix/tasks/compile.ex
@@ -146,7 +146,7 @@ defmodule Mix.Tasks.Compile do
       loaded_paths =
         Mix.Project.apps_paths(config)
         |> Map.keys()
-        |> Mix.AppLoader.load_apps(Mix.Dep.cached(), config, false, [], fn
+        |> Mix.AppLoader.load_apps(Mix.Dep.cached(), config, [], fn
           {_app, path}, acc -> if path, do: [path | acc], else: acc
         end)
 
diff --git a/lib/mix/test/fixtures/deps_status/custom/raw_repo/lib/raw_repo.ex b/lib/mix/test/fixtures/deps_status/custom/raw_repo/lib/raw_repo.ex
index 806471ca9..b893d48bc 100644
--- a/lib/mix/test/fixtures/deps_status/custom/raw_repo/lib/raw_repo.ex
+++ b/lib/mix/test/fixtures/deps_status/custom/raw_repo/lib/raw_repo.ex
@@ -1,3 +1,5 @@
+Application.compile_env(:raw_repo, :compile_env)
+
 defmodule RawRepo do
   def hello do
     "world"
diff --git a/lib/mix/test/mix/tasks/deps_test.exs b/lib/mix/test/mix/tasks/deps_test.exs
index a16044409..200a978d9 100644
--- a/lib/mix/test/mix/tasks/deps_test.exs
+++ b/lib/mix/test/mix/tasks/deps_test.exs
@@ -57,6 +57,18 @@ defmodule Mix.Tasks.DepsTest do
     end
   end
 
+  defmodule RawRepoDep do
+    def project do
+      [
+        app: :raw_sample,
+        version: "0.1.0",
+        deps: [
+          {:raw_repo, "0.1.0", path: "custom/raw_repo"}
+        ]
+      ]
+    end
+  end
+
   ## deps
 
   test "prints list of dependencies and their status" do
@@ -414,18 +426,6 @@ defmodule Mix.Tasks.DepsTest do
 
   ## Deps environment
 
-  defmodule DepsEnvApp do
-    def project do
-      [
-        app: :raw_sample,
-        version: "0.1.0",
-        deps: [
-          {:raw_repo, "0.1.0", path: "custom/raw_repo"}
-        ]
-      ]
-    end
-  end
-
   defmodule CustomDepsEnvApp do
     def project do
       [
@@ -440,7 +440,7 @@ defmodule Mix.Tasks.DepsTest do
 
   test "sets deps env to prod by default" do
     in_fixture("deps_status", fn ->
-      Mix.Project.push(DepsEnvApp)
+      Mix.Project.push(RawRepoDep)
 
       Mix.Tasks.Deps.Update.run(["--all"])
       assert_received {:mix_shell, :info, [":raw_repo env is prod"]}
@@ -775,6 +775,41 @@ defmodule Mix.Tasks.DepsTest do
     end)
   end
 
+  test "checks if compile env changed" do
+    in_fixture("deps_status", fn ->
+      Mix.Project.push(RawRepoDep)
+      Mix.Tasks.Deps.Loadpaths.run([])
+      assert_receive {:mix_shell, :info, ["Generated raw_repo app"]}
+      assert Application.spec(:raw_repo, :vsn)
+
+      File.mkdir_p!("config")
+
+      File.write!("config/config.exs", """
+      import Config
+      config :raw_repo, :compile_env, :new_value
+      """)
+
+      Application.unload(:raw_repo)
+      Mix.ProjectStack.pop()
+      Mix.Task.clear()
+      Mix.Project.push(RawRepoDep)
+      purge([RawRepo])
+      Mix.Tasks.Loadconfig.load_compile("config/config.exs")
+
+      Mix.Tasks.Deps.run([])
+
+      assert_receive {:mix_shell, :info,
+                      ["  the dependency build is outdated, please run \"mix deps.compile\""]}
+
+      Mix.Tasks.Deps.Loadpaths.run([])
+
+      assert_receive {:mix_shell, :info, ["Generated raw_repo app"]}
+      assert Application.spec(:raw_repo, :vsn)
+    end)
+  after
+    Application.delete_env(:raw_repo, :compile_env, persistent: true)
+  end
+
   defmodule NonCompilingDeps do
     def project do
       [