Cache deps and archive loadpaths in Erlang/OTP 26

13 files changed, 119 insertions, 37 deletions

diff --git a/lib/elixir/lib/code.ex b/lib/elixir/lib/code.ex
index e54b6b03b..f92da34e5 100644
--- a/lib/elixir/lib/code.ex
+++ b/lib/elixir/lib/code.ex
@@ -288,10 +288,16 @@ defmodule Code do
       Code.append_path("/does_not_exist")
       #=> false
 
+  ## Options
+
+    * `:cache` - (since v1.15.0) when true, the code path is cached
+      the first time it is traversed in order to reduce file system
+      operations. It requires Erlang/OTP 26, otherwise it is a no-op.
+
   """
-  @spec append_path(Path.t()) :: true | false
-  def append_path(path) do
-    :code.add_pathz(to_charlist(Path.expand(path))) == true
+  @spec append_path(Path.t(), cache: boolean()) :: true | false
+  def append_path(path, opts \\ []) do
+    apply(:code, :add_pathz, [to_charlist(Path.expand(path)) | cache(opts)]) == true
   end
 
   @doc """
@@ -313,10 +319,16 @@ defmodule Code do
       Code.prepend_path("/does_not_exist")
       #=> false
 
+  ## Options
+
+    * `:cache` - (since v1.15.0) when true, the code path is cached
+      the first time it is traversed in order to reduce file system
+      operations. It requires Erlang/OTP 26, otherwise it is a no-op.
+
   """
-  @spec prepend_path(Path.t()) :: boolean()
-  def prepend_path(path) do
-    :code.add_patha(to_charlist(Path.expand(path))) == true
+  @spec prepend_path(Path.t(), cache: boolean()) :: boolean()
+  def prepend_path(path, opts \\ []) do
+    apply(:code, :add_patha, [to_charlist(Path.expand(path)) | cache(opts)]) == true
   end
 
   @doc """
@@ -335,11 +347,17 @@ defmodule Code do
 
       Code.prepend_paths([".", "/does_not_exist"])
       #=> :ok
+
+  ## Options
+
+    * `:cache` - when true, the code path is cached the first time
+      it is traversed in order to reduce file system operations.
+      It requires Erlang/OTP 26, otherwise it is a no-op.
   """
   @doc since: "1.15.0"
-  @spec prepend_paths([Path.t()]) :: :ok
-  def prepend_paths(paths) when is_list(paths) do
-    :code.add_pathsa(Enum.map(paths, &to_charlist(Path.expand(&1))))
+  @spec prepend_paths([Path.t()], cache: boolean()) :: :ok
+  def prepend_paths(paths, opts \\ []) when is_list(paths) do
+    apply(:code, :add_pathsa, [Enum.map(paths, &to_charlist(Path.expand(&1))) | cache(opts)])
   end
 
   @doc """
@@ -358,11 +376,25 @@ defmodule Code do
 
       Code.append_paths([".", "/does_not_exist"])
       #=> :ok
+
+  ## Options
+
+    * `:cache` - when true, the code path is cached the first time
+      it is traversed in order to reduce file system operations.
+      It requires Erlang/OTP 26, otherwise it is a no-op.
   """
   @doc since: "1.15.0"
-  @spec append_paths([Path.t()]) :: :ok
-  def append_paths(paths) when is_list(paths) do
-    :code.add_pathsz(Enum.map(paths, &to_charlist(Path.expand(&1))))
+  @spec append_paths([Path.t()], cache: boolean()) :: :ok
+  def append_paths(paths, opts \\ []) when is_list(paths) do
+    apply(:code, :add_pathsz, [Enum.map(paths, &to_charlist(Path.expand(&1))) | cache(opts)])
+  end
+
+  defp cache(opts) do
+    if function_exported?(:code, :add_path, 2) do
+      if opts[:cache], do: [:cache], else: [:nocache]
+    else
+      []
+    end
   end
 
   @doc """
diff --git a/lib/elixir/src/elixir.erl b/lib/elixir/src/elixir.erl
index 4de9d056b..46bb07d2d 100644
--- a/lib/elixir/src/elixir.erl
+++ b/lib/elixir/src/elixir.erl
@@ -25,6 +25,12 @@
 
 %% OTP Application API
 
+-if(?OTP_RELEASE >= 26).
+load_paths(Paths) -> code:add_pathsa(Paths, cache).
+-else.
+load_paths(Paths) -> code:add_pathsa(Paths).
+-endif.
+
 start(_Type, _Args) ->
   _ = parse_otp_release(),
   preload_common_modules(),
@@ -33,7 +39,7 @@ start(_Type, _Args) ->
 
   case init:get_argument(elixir_root) of
     {ok, [[Root]]} ->
-      code:add_pathsa([
+      load_paths([
         Root ++ "/eex/ebin",
         Root ++ "/ex_unit/ebin",
         Root ++ "/iex/ebin",
diff --git a/lib/elixir/test/elixir/code_test.exs b/lib/elixir/test/elixir/code_test.exs
index d2c33bbe3..4b1e2e439 100644
--- a/lib/elixir/test/elixir/code_test.exs
+++ b/lib/elixir/test/elixir/code_test.exs
@@ -413,10 +413,36 @@ defmodule Code.SyncTest do
 
   import PathHelpers
 
+  if :erlang.system_info(:otp_release) >= ~c"26" do
+    defp assert_cached(path) do
+      assert find_path(path) != :nocache
+    end
+
+    defp refute_cached(path) do
+      assert find_path(path) == :nocache
+    end
+
+    defp find_path(path) do
+      {:status, _, {:module, :code_server}, [_, :running, _, _, state]} =
+        :sys.get_status(:code_server)
+
+      [:state, _, _otp_root, paths | _] = Tuple.to_list(state)
+      {_, value} = List.keyfind(paths, to_charlist(path), 0)
+      value
+    end
+  else
+    defp assert_cached(_path), do: :ok
+    defp refute_cached(_path), do: :ok
+  end
+
   test "prepend_path" do
     path = Path.join(__DIR__, "fixtures")
     true = Code.prepend_path(path)
     assert to_charlist(path) in :code.get_path()
+    refute_cached(path)
+
+    true = Code.prepend_path(path, cache: true)
+    assert_cached(path)
 
     Code.delete_path(path)
     refute to_charlist(path) in :code.get_path()
@@ -426,6 +452,10 @@ defmodule Code.SyncTest do
     path = Path.join(__DIR__, "fixtures")
     true = Code.append_path(path)
     assert to_charlist(path) in :code.get_path()
+    refute_cached(path)
+
+    true = Code.append_path(path, cache: true)
+    assert_cached(path)
 
     Code.delete_path(path)
     refute to_charlist(path) in :code.get_path()
@@ -435,6 +465,10 @@ defmodule Code.SyncTest do
     path = Path.join(__DIR__, "fixtures")
     :ok = Code.prepend_paths([path])
     assert to_charlist(path) in :code.get_path()
+    refute_cached(path)
+
+    :ok = Code.prepend_paths([path], cache: true)
+    assert_cached(path)
 
     Code.delete_paths([path])
     refute to_charlist(path) in :code.get_path()
@@ -444,6 +478,10 @@ defmodule Code.SyncTest do
     path = Path.join(__DIR__, "fixtures")
     :ok = Code.append_paths([path])
     assert to_charlist(path) in :code.get_path()
+    refute_cached(path)
+
+    :ok = Code.append_paths([path], cache: true)
+    assert_cached(path)
 
     Code.delete_paths([path])
     refute to_charlist(path) in :code.get_path()
diff --git a/lib/mix/lib/mix.ex b/lib/mix/lib/mix.ex
index f6747c19f..dba6f114f 100644
--- a/lib/mix/lib/mix.ex
+++ b/lib/mix/lib/mix.ex
@@ -594,7 +594,7 @@ defmodule Mix do
   def ensure_application!(app) when is_atom(app) do
     case Mix.State.builtin_apps() do
       %{^app => {:ebin, path}} ->
-        Code.prepend_path(path)
+        Code.prepend_path(path, cache: true)
 
       %{} ->
         Mix.raise(
diff --git a/lib/mix/lib/mix/compilers/elixir.ex b/lib/mix/lib/mix/compilers/elixir.ex
index fbeeee24c..f07f382a4 100644
--- a/lib/mix/lib/mix/compilers/elixir.ex
+++ b/lib/mix/lib/mix/compilers/elixir.ex
@@ -165,8 +165,9 @@ defmodule Mix.Compilers.Elixir do
 
       Mix.Utils.compiling_n(length(stale), :ex)
       Mix.Project.ensure_structure()
-      true = Code.prepend_path(dest)
 
+      # We don't want to cache this path as we will write to it
+      true = Code.prepend_path(dest)
       previous_opts = set_compiler_opts(opts)
 
       try do
diff --git a/lib/mix/lib/mix/compilers/erlang.ex b/lib/mix/lib/mix/compilers/erlang.ex
index e0d2514bb..d589cedcb 100644
--- a/lib/mix/lib/mix/compilers/erlang.ex
+++ b/lib/mix/lib/mix/compilers/erlang.ex
@@ -115,7 +115,8 @@ defmodule Mix.Compilers.Erlang do
 
       # Let's prepend the newly created path so compiled files
       # can be accessed still during compilation (for behaviours
-      # and what not).
+      # and what not). Note we don't want to cache this path as
+      # we will write to it.
       Code.prepend_path(Mix.Project.compile_path())
 
       {parallel, serial} =
diff --git a/lib/mix/lib/mix/local.ex b/lib/mix/lib/mix/local.ex
index a4b2dc338..6e4af8bf7 100644
--- a/lib/mix/lib/mix/local.ex
+++ b/lib/mix/lib/mix/local.ex
@@ -51,7 +51,7 @@ defmodule Mix.Local do
   def append_archives do
     for archive <- archives_ebins() do
       check_elixir_version_in_ebin(archive)
-      Code.append_path(archive)
+      Code.append_path(archive, cache: true)
     end
 
     :ok
@@ -70,9 +70,12 @@ defmodule Mix.Local do
 
   @doc """
   Appends Mix paths to the Erlang code path.
+
+  We don't cache them as they are rarely used and
+  they may be development paths.
   """
   def append_paths do
-    Enum.each(mix_paths(), &Code.append_path(&1))
+    Enum.each(mix_paths(), &Code.append_path/1)
   end
 
   defp mix_paths do
diff --git a/lib/mix/lib/mix/tasks/archive.install.ex b/lib/mix/lib/mix/tasks/archive.install.ex
index ab9f4671e..19c47dfb1 100644
--- a/lib/mix/lib/mix/tasks/archive.install.ex
+++ b/lib/mix/lib/mix/tasks/archive.install.ex
@@ -121,7 +121,7 @@ defmodule Mix.Tasks.Archive.Install do
 
     ebin = Mix.Local.archive_ebin(dir_dest)
     Mix.Local.check_elixir_version_in_ebin(ebin)
-    true = Code.append_path(ebin)
+    true = Code.append_path(ebin, cache: true)
     :ok
   end
 
diff --git a/lib/mix/lib/mix/tasks/compile.all.ex b/lib/mix/lib/mix/tasks/compile.all.ex
index 485d53e98..ddc83bb19 100644
--- a/lib/mix/lib/mix/tasks/compile.all.ex
+++ b/lib/mix/lib/mix/tasks/compile.all.ex
@@ -45,7 +45,7 @@ defmodule Mix.Tasks.Compile.All do
       Code.delete_paths(current_paths -- loaded_paths)
     end
 
-    Code.prepend_paths(loaded_paths -- current_paths)
+    Code.prepend_paths(loaded_paths -- current_paths, cache: true)
 
     result =
       if "--no-compile" in args do
@@ -66,6 +66,9 @@ defmodule Mix.Tasks.Compile.All do
       Mix.AppLoader.write_cache(app_cache, Map.new(loaded_modules))
     end
 
+    # Add the current compilation path. compile.elixir and compile.erlang
+    # will also add this path, but only if they run, so we always add it
+    # here too. Furthermore, we don't cache it as we may still write to it.
     compile_path = to_charlist(Mix.Project.compile_path())
     _ = Code.prepend_path(compile_path)
 
diff --git a/lib/mix/lib/mix/tasks/compile.ex b/lib/mix/lib/mix/tasks/compile.ex
index ad70e0d71..b884f07d7 100644
--- a/lib/mix/lib/mix/tasks/compile.ex
+++ b/lib/mix/lib/mix/tasks/compile.ex
@@ -126,8 +126,8 @@ defmodule Mix.Tasks.Compile do
   def run(args) do
     Mix.Project.get!()
 
-    # Don't bother setting up the load paths because compile.all
-    # will load applications and set them up anyway.
+    # We run loadpaths to perform checks but we don't bother setting
+    # up the load paths because compile.all will manage them anyway.
     Mix.Task.run("loadpaths", ["--no-deps-loading" | args])
 
     {opts, _, _} = OptionParser.parse(args, switches: [erl_config: :string])
@@ -150,6 +150,7 @@ defmodule Mix.Tasks.Compile do
           {_app, path}, acc -> if path, do: [path | acc], else: acc
         end)
 
+      # We don't cache umbrella paths as we may write to them
       Code.prepend_paths(loaded_paths -- :code.get_path())
     end
 
@@ -173,6 +174,7 @@ defmodule Mix.Tasks.Compile do
     with true <- consolidate_protocols?,
          path = Mix.Project.consolidation_path(config),
          {:ok, protocols} <- File.ls(path) do
+      # We don't cache consolidation path as we may write to it
       Code.prepend_path(path)
       Enum.each(protocols, &load_protocol/1)
     end
diff --git a/lib/mix/lib/mix/tasks/deps.compile.ex b/lib/mix/lib/mix/tasks/deps.compile.ex
index 6a1ea16f1..66734eb4b 100644
--- a/lib/mix/lib/mix/tasks/deps.compile.ex
+++ b/lib/mix/lib/mix/tasks/deps.compile.ex
@@ -165,13 +165,7 @@ defmodule Mix.Tasks.Deps.Compile do
       end
 
       try do
-        options = [
-          "--from-mix-deps-compile",
-          "--no-archives-check",
-          "--no-warnings-as-errors",
-          "--no-code-path-pruning"
-        ]
-
+        options = ["--from-mix-deps-compile", "--no-warnings-as-errors", "--no-code-path-pruning"]
         res = Mix.Task.run("compile", options)
         match?({:ok, _}, res)
       catch
@@ -225,7 +219,7 @@ defmodule Mix.Tasks.Deps.Compile do
       Mix.Utils.symlink_or_copy(Path.join(dep_path, dir), Path.join(build_path, dir))
     end
 
-    Code.prepend_path(Path.join(build_path, "ebin"))
+    Code.prepend_path(Path.join(build_path, "ebin"), cache: true)
     true
   end
 
@@ -319,7 +313,7 @@ defmodule Mix.Tasks.Deps.Compile do
   defp build_structure(%Mix.Dep{opts: opts}, config) do
     config = Keyword.put(config, :deps_app_path, opts[:build])
     Mix.Project.build_structure(config, symlink_ebin: true, source: opts[:dest])
-    Code.prepend_path(Path.join(opts[:build], "ebin"))
+    Code.prepend_path(Path.join(opts[:build], "ebin"), cache: true)
   end
 
   defp old_elixir_req(config) do
diff --git a/lib/mix/lib/mix/tasks/deps.loadpaths.ex b/lib/mix/lib/mix/tasks/deps.loadpaths.ex
index 6a6db807a..9b4679a33 100644
--- a/lib/mix/lib/mix/tasks/deps.loadpaths.ex
+++ b/lib/mix/lib/mix/tasks/deps.loadpaths.ex
@@ -14,7 +14,8 @@ defmodule Mix.Tasks.Deps.Loadpaths do
 
   ## Command line options
 
-    * `--no-compile` - does not compile dependencies
+    * `--no-archives-check` - does not check archives
+    * `--no-compile` - does not compile even if files require compilation
     * `--no-deps-check` - does not check or compile deps, only load available ones
     * `--no-deps-loading` - does not add deps loadpaths to the code path
     * `--no-elixir-version-check` - does not check Elixir version
@@ -24,6 +25,10 @@ defmodule Mix.Tasks.Deps.Loadpaths do
 
   @impl true
   def run(args) do
+    unless "--no-archives-check" in args do
+      Mix.Task.run("archive.check", args)
+    end
+
     all = Mix.Dep.load_and_cache()
 
     all =
@@ -44,7 +49,7 @@ defmodule Mix.Tasks.Deps.Loadpaths do
     end
 
     unless "--no-deps-loading" in args do
-      Code.prepend_paths(Enum.flat_map(all, &Mix.Dep.load_paths/1))
+      Code.prepend_paths(Enum.flat_map(all, &Mix.Dep.load_paths/1), cache: true)
     end
 
     :ok
diff --git a/lib/mix/lib/mix/tasks/loadpaths.ex b/lib/mix/lib/mix/tasks/loadpaths.ex
index a5dbd7a75..9a5833282 100644
--- a/lib/mix/lib/mix/tasks/loadpaths.ex
+++ b/lib/mix/lib/mix/tasks/loadpaths.ex
@@ -29,10 +29,6 @@ defmodule Mix.Tasks.Loadpaths do
   def run(args) do
     config = Mix.Project.config()
 
-    unless "--no-archives-check" in args do
-      Mix.Task.run("archive.check", args)
-    end
-
     # --from-mix-deps-compile is used only internally to avoid
     # recursively checking and loading dependencies that have
     # already been loaded. It has no purpose from Mix.CLI
@@ -62,6 +58,7 @@ defmodule Mix.Tasks.Loadpaths do
       _ -> :ok
     end
 
+    # We don't cache the current application as we may still write to it
     Code.prepend_path(Mix.Project.compile_path(config))
   end