Changes from feedbackemj/parallel-compile-erlang

4 files changed, 48 insertions, 23 deletions

diff --git a/lib/mix/lib/mix/compilers/erlang.ex b/lib/mix/lib/mix/compilers/erlang.ex
index e7860e5da..57500a202 100644
--- a/lib/mix/lib/mix/compilers/erlang.ex
+++ b/lib/mix/lib/mix/compilers/erlang.ex
@@ -12,6 +12,15 @@ defmodule Mix.Compilers.Erlang do
 
   `mappings` should be a list of tuples in the form of `{src, dest}` paths.
 
+  ## Options
+
+    * `:force` - forces compilation regardless of modification times
+
+    * `:parallel` - if `true` all files will be compiled in parallel,
+      otherwise the given list of source file names will be compiled
+      in parallel, all other files are compiled serially before the
+      parallel files
+
   ## Examples
 
   For example, a simple compiler for Lisp Flavored Erlang
@@ -74,6 +83,16 @@ defmodule Mix.Compilers.Erlang do
   A `manifest` file and a `callback` to be invoked for each src/dest pair
   must be given. A src/dest pair where destination is `nil` is considered
   to be up to date and won't be (re-)compiled.
+
+  ## Options
+
+    * `:force` - forces compilation regardless of modification times
+
+    * `:parallel` - if `true` all files will be compiled in parallel,
+      otherwise the given list of source file names will be compiled
+      in parallel, all other files are compiled serially before the
+      parallel files
+
   """
   def compile(manifest, mappings, opts \\ [], callback) do
     compile(manifest, mappings, :erl, opts, callback)
@@ -119,16 +138,21 @@ defmodule Mix.Compilers.Erlang do
       # and what not).
       Code.prepend_path(Mix.Project.compile_path())
 
-      parallel_sources = opts[:parallel] || MapSet.new()
-
       {parallel, serial} =
-        Enum.split_with(stale, fn {source, _target} -> source in parallel_sources end)
+        case opts[:parallel] || false do
+          true -> {stale, []}
+          false -> {[], stale}
+          parallel -> Enum.split_with(stale, fn {source, _target} -> source in parallel end)
+        end
 
       serial_results = Enum.map(serial, &do_compile(&1, callback, timestamp, verbose))
 
       parallel_results =
         parallel
-        |> Task.async_stream(&do_compile(&1, callback, timestamp, verbose), timeout: :infinity)
+        |> Task.async_stream(&do_compile(&1, callback, timestamp, verbose),
+          timeout: :infinity,
+          ordered: false
+        )
         |> Enum.map(fn {:ok, result} -> result end)
 
       # Compile stale files and print the results
diff --git a/lib/mix/lib/mix/tasks/compile.erlang.ex b/lib/mix/lib/mix/tasks/compile.erlang.ex
index 5c257acdf..b93430b1c 100644
--- a/lib/mix/lib/mix/tasks/compile.erlang.ex
+++ b/lib/mix/lib/mix/tasks/compile.erlang.ex
@@ -74,15 +74,7 @@ defmodule Mix.Tasks.Compile.Erlang do
     end
 
     erlc_options =
-      erlc_options ++
-        [
-          :debug_info,
-          :return,
-          :report,
-          :no_spawn_compiler_process,
-          outdir: compile_path,
-          i: include_path
-        ]
+      erlc_options ++ [:debug_info, :return, :report, outdir: compile_path, i: include_path]
 
     erlc_options =
       Enum.map(erlc_options, fn
@@ -92,10 +84,10 @@ defmodule Mix.Tasks.Compile.Erlang do
 
     compile_path = Path.relative_to(compile_path, File.cwd!())
 
-    erls = scan_sources(files, include_path, source_paths)
-    tuples = Enum.map(erls, &annotate_target(&1, compile_path, opts[:force]))
-    parallel = Enum.map(find_parallel(erls), & &1.file)
-    opts = opts ++ [parallel: MapSet.new(parallel)]
+    {erls, tuples} =
+      Enum.unzip(scan_sources(files, include_path, source_paths, compile_path, opts))
+
+    opts = [parallel: MapSet.new(find_parallel(erls))] ++ opts
 
     Erlang.compile(manifest(), tuples, opts, fn input, _output ->
       # We're purging the module because a previous compiler (for example, Phoenix)
@@ -131,18 +123,18 @@ defmodule Mix.Tasks.Compile.Erlang do
 
   ## Internal helpers
 
-  defp scan_sources(files, include_path, source_paths) do
+  defp scan_sources(files, include_path, source_paths, compile_path, opts) do
     include_paths = [include_path | source_paths]
 
     files
-    |> Task.async_stream(&scan_source(&1, include_paths))
+    |> Task.async_stream(&scan_source(&1, include_paths, compile_path, opts))
     |> Enum.flat_map(fn
-      {:ok, {:ok, forms}} -> [forms]
+      {:ok, {:ok, erl_file, target_tuple}} -> [{erl_file, target_tuple}]
       {:ok, :error} -> []
     end)
   end
 
-  defp scan_source(file, include_paths) do
+  defp scan_source(file, include_paths, compile_path, opts) do
     erl_file = %{
       file: file,
       module: module_from_artifact(file),
@@ -154,7 +146,9 @@ defmodule Mix.Tasks.Compile.Erlang do
 
     case :epp.parse_file(Erlang.to_erl_file(file), include_paths, []) do
       {:ok, forms} ->
-        {:ok, List.foldl(tl(forms), erl_file, &do_form(file, &1, &2))}
+        erl_file = List.foldl(tl(forms), erl_file, &do_form(file, &1, &2))
+        target_tuple = annotate_target(erl_file, compile_path, opts[:force])
+        {:ok, erl_file, target_tuple}
 
       {:error, _error} ->
         :error
@@ -189,7 +183,10 @@ defmodule Mix.Tasks.Compile.Erlang do
 
   defp find_parallel(erls) do
     serial = MapSet.new(find_dependencies(erls))
-    Enum.reject(erls, &(&1.module in serial))
+
+    erls
+    |> Enum.reject(&(&1.module in serial))
+    |> Enum.map(& &1.file)
   end
 
   defp find_dependencies(erls) do
diff --git a/lib/mix/lib/mix/tasks/compile.leex.ex b/lib/mix/lib/mix/tasks/compile.leex.ex
index 955124b8d..e99adca9e 100644
--- a/lib/mix/lib/mix/tasks/compile.leex.ex
+++ b/lib/mix/lib/mix/tasks/compile.leex.ex
@@ -54,6 +54,8 @@ defmodule Mix.Tasks.Compile.Leex do
       Mix.raise(":leex_options should be a list of options, got: #{inspect(options)}")
     end
 
+    opts = [parallel: true] ++ opts
+
     Erlang.compile(manifest(), mappings, :xrl, :erl, opts, fn input, output ->
       Erlang.ensure_application!(:parsetools, input)
       options = options ++ @forced_opts ++ [scannerfile: Erlang.to_erl_file(output)]
diff --git a/lib/mix/lib/mix/tasks/compile.yecc.ex b/lib/mix/lib/mix/tasks/compile.yecc.ex
index 28968c916..94b43e18b 100644
--- a/lib/mix/lib/mix/tasks/compile.yecc.ex
+++ b/lib/mix/lib/mix/tasks/compile.yecc.ex
@@ -54,6 +54,8 @@ defmodule Mix.Tasks.Compile.Yecc do
       Mix.raise(":yecc_options should be a list of options, got: #{inspect(options)}")
     end
 
+    opts = [parallel: true] ++ opts
+
     Erlang.compile(manifest(), mappings, :yrl, :erl, opts, fn input, output ->
       Erlang.ensure_application!(:parsetools, input)
       options = options ++ @forced_opts ++ [parserfile: Erlang.to_erl_file(output)]