Embed IEx parser into get_untiljv-get-until-parse

5 files changed, 85 insertions, 83 deletions

diff --git a/lib/iex/lib/iex.ex b/lib/iex/lib/iex.ex
index ce60ff38e..1d36daf72 100644
--- a/lib/iex/lib/iex.ex
+++ b/lib/iex/lib/iex.ex
@@ -500,11 +500,17 @@ defmodule IEx do
   The parser is a "mfargs", which is a tuple with three elements:
   the module name, the function name, and extra arguments to
   be appended. The parser receives at least three arguments, the
-  current input as a string, the parsing options as a keyword list,
-  and the buffer as a string. It must return `{:ok, expr, buffer}`
-  or `{:incomplete, buffer}`.
+  current input as a charlist, the parsing options as a keyword list,
+  and the state. The initial state is an empty charlist. It must
+  return `{:ok, expr, state}` or `{:incomplete, state}`.
 
-  If the parser raises, the buffer is reset to an empty string.
+  If the parser raises, the state is reset to an empty charlist.
+
+  > In earlier Elixir versions, the parser would receive the input
+  > and the initial buffer as strings. However, this behaviour
+  > changed when Erlang/OTP introduced multiline editing. If you
+  > support earlier Elixir versions, you can normalize the inputs
+  > by calling `to_charlist/1`.
   """
   @spec configure(keyword()) :: :ok
   def configure(options) do
diff --git a/lib/iex/lib/iex/evaluator.ex b/lib/iex/lib/iex/evaluator.ex
index 2cd11ec10..26345b537 100644
--- a/lib/iex/lib/iex/evaluator.ex
+++ b/lib/iex/lib/iex/evaluator.ex
@@ -49,26 +49,21 @@ defmodule IEx.Evaluator do
     end
   end
 
-  # If parsing fails, this might be a TokenMissingError which we treat in
-  # a special way (to allow for continuation of an expression on the next
-  # line in IEx).
-  #
-  # The first two clauses provide support for the break-trigger allowing to
-  # break out from a pending incomplete expression. See
-  # https://github.com/elixir-lang/elixir/issues/1089 for discussion.
-  @break_trigger "#iex:break\n"
+  @break_trigger ~c"#iex:break\n"
 
   @op_tokens [:or_op, :and_op, :comp_op, :rel_op, :arrow_op, :in_op] ++
                [:three_op, :concat_op, :mult_op]
 
-  @doc false
+  @doc """
+  Default parsing implementation with support for pipes and #iex:break.
+
+  If parsing fails, this might be a TokenMissingError which we treat in
+  a special way (to allow for continuation of an expression on the next
+  line in IEx).
+  """
   def parse(input, opts, parser_state)
 
-  def parse(input, opts, ""), do: parse(input, opts, {"", :other})
-
-  def parse(@break_trigger, _opts, {"", _} = parser_state) do
-    {:incomplete, parser_state}
-  end
+  def parse(input, opts, []), do: parse(input, opts, {[], :other})
 
   def parse(@break_trigger, opts, _parser_state) do
     :elixir_errors.parse_error(
@@ -81,14 +76,13 @@ defmodule IEx.Evaluator do
   end
 
   def parse(input, opts, {buffer, last_op}) do
-    input = buffer <> input
+    input = buffer ++ input
     file = Keyword.get(opts, :file, "nofile")
     line = Keyword.get(opts, :line, 1)
     column = Keyword.get(opts, :column, 1)
-    charlist = String.to_charlist(input)
 
     result =
-      with {:ok, tokens} <- :elixir.string_to_tokens(charlist, line, column, file, opts),
+      with {:ok, tokens} <- :elixir.string_to_tokens(input, line, column, file, opts),
            {:ok, adjusted_tokens} <- adjust_operator(tokens, line, column, file, opts, last_op),
            {:ok, forms} <- :elixir.tokens_to_quoted(adjusted_tokens, file, opts) do
         last_op =
@@ -102,7 +96,7 @@ defmodule IEx.Evaluator do
 
     case result do
       {:ok, forms, last_op} ->
-        {:ok, forms, {"", last_op}}
+        {:ok, forms, {[], last_op}}
 
       {:error, {_, _, ""}} ->
         {:incomplete, {input, last_op}}
@@ -113,7 +107,7 @@ defmodule IEx.Evaluator do
           file,
           error,
           token,
-          {charlist, line, column}
+          {input, line, column}
         )
     end
   end
@@ -183,9 +177,9 @@ defmodule IEx.Evaluator do
 
   defp loop(%{server: server, ref: ref} = state) do
     receive do
-      {:eval, ^server, code, counter, parser_state} ->
-        {status, parser_state, state} = parse_eval_inspect(code, counter, parser_state, state)
-        send(server, {:evaled, self(), status, parser_state})
+      {:eval, ^server, code, counter} ->
+        {status, state} = safe_eval_and_inspect(code, counter, state)
+        send(server, {:evaled, self(), status})
         loop(state)
 
       {:fields_from_env, ^server, ref, receiver, fields} ->
@@ -285,32 +279,19 @@ defmodule IEx.Evaluator do
     end
   end
 
-  defp parse_eval_inspect(code, counter, parser_state, state) do
-    try do
-      {parser_module, parser_fun, args} = IEx.Config.parser()
-      args = [code, [line: counter, file: "iex"], parser_state | args]
-      eval_and_inspect_parsed(apply(parser_module, parser_fun, args), counter, state)
-    catch
-      kind, error ->
-        print_error(kind, error, __STACKTRACE__)
-        {:error, "", state}
-    end
-  end
-
-  defp eval_and_inspect_parsed({:ok, forms, parser_state}, counter, state) do
+  defp safe_eval_and_inspect(forms, counter, state) do
     put_history(state)
     put_whereami(state)
-    state = eval_and_inspect(forms, counter, state)
-    {:ok, parser_state, state}
+    {:ok, eval_and_inspect(forms, counter, state)}
+  catch
+    kind, error ->
+      print_error(kind, error, __STACKTRACE__)
+      {:error, state}
   after
     Process.delete(:iex_history)
     Process.delete(:iex_whereami)
   end
 
-  defp eval_and_inspect_parsed({:incomplete, parser_state}, _counter, state) do
-    {:incomplete, parser_state, state}
-  end
-
   defp put_history(%{history: history}) do
     Process.put(:iex_history, history)
   end
diff --git a/lib/iex/lib/iex/server.ex b/lib/iex/lib/iex/server.ex
index bc76afd8c..dc89f65ed 100644
--- a/lib/iex/lib/iex/server.ex
+++ b/lib/iex/lib/iex/server.ex
@@ -11,7 +11,7 @@ defmodule IEx.Server do
   """
 
   @doc false
-  defstruct parser_state: "",
+  defstruct parser_state: [],
             counter: 1,
             prefix: "iex",
             on_eof: :stop_evaluator,
@@ -82,7 +82,7 @@ defmodule IEx.Server do
     )
 
     evaluator = start_evaluator(state.counter, Keyword.merge(state.evaluator_options, opts))
-    loop(state, :ok, evaluator, Process.monitor(evaluator), input)
+    loop(state, evaluator, Process.monitor(evaluator), input)
   end
 
   # Starts an evaluator using the provided options.
@@ -111,18 +111,19 @@ defmodule IEx.Server do
     run_without_registration(state, opts, input)
   end
 
-  defp loop(state, status, evaluator, evaluator_ref, input) do
-    :io.setopts(expand_fun: state.expand_fun)
-    input = input || io_get(prompt(status, state.prefix, state.counter))
+  defp loop(state, evaluator, evaluator_ref, input) do
+    %{counter: counter, expand_fun: expand_fun, prefix: prefix, parser_state: parser} = state
+    :io.setopts(expand_fun: expand_fun)
+    input = input || io_get(prompt(prefix, counter), counter, parser)
     wait_input(state, evaluator, evaluator_ref, input)
   end
 
   defp wait_input(state, evaluator, evaluator_ref, input) do
     receive do
-      {:io_reply, ^input, code} when is_binary(code) ->
+      {:io_reply, ^input, {:ok, code, parser_state}} ->
         :io.setopts(expand_fun: fn _ -> {:yes, [], []} end)
-        send(evaluator, {:eval, self(), code, state.counter, state.parser_state})
-        wait_eval(state, evaluator, evaluator_ref)
+        send(evaluator, {:eval, self(), code, state.counter})
+        wait_eval(%{state | parser_state: parser_state}, evaluator, evaluator_ref)
 
       {:io_reply, ^input, :eof} ->
         case state.on_eof do
@@ -130,15 +131,21 @@ defmodule IEx.Server do
           :stop_evaluator -> stop_evaluator(evaluator, evaluator_ref)
         end
 
+      {:io_reply, ^input, {:error, kind, error, stacktrace}} ->
+        banner = IEx.color(:eval_error, Exception.format_banner(kind, error, stacktrace))
+        stackdata = Exception.format_stacktrace(stacktrace)
+        IO.write(:stdio, [banner, ?\n, IEx.color(:stack_info, stackdata)])
+        loop(%{state | parser_state: []}, evaluator, evaluator_ref, nil)
+
       # Triggered by pressing "i" as the job control switch
       {:io_reply, ^input, {:error, :interrupted}} ->
         io_error("** (EXIT) interrupted")
-        loop(%{state | parser_state: ""}, :ok, evaluator, evaluator_ref, nil)
+        loop(%{state | parser_state: []}, evaluator, evaluator_ref, nil)
 
       # Unknown IO message
       {:io_reply, ^input, msg} ->
         io_error("** (EXIT) unknown IO message: #{inspect(msg)}")
-        loop(%{state | parser_state: ""}, :ok, evaluator, evaluator_ref, nil)
+        loop(%{state | parser_state: []}, evaluator, evaluator_ref, nil)
 
       # Triggered when IO dies while waiting for input
       {:DOWN, ^input, _, _, _} ->
@@ -153,10 +160,10 @@ defmodule IEx.Server do
 
   defp wait_eval(state, evaluator, evaluator_ref) do
     receive do
-      {:evaled, ^evaluator, status, parser_state} ->
+      {:evaled, ^evaluator, status} ->
         counter = if(status == :ok, do: state.counter + 1, else: state.counter)
-        state = %{state | counter: counter, parser_state: parser_state}
-        loop(state, status, evaluator, evaluator_ref, nil)
+        state = %{state | counter: counter}
+        loop(state, evaluator, evaluator_ref, nil)
 
       msg ->
         handle_take_over(msg, state, evaluator, evaluator_ref, nil, fn state ->
@@ -193,7 +200,7 @@ defmodule IEx.Server do
         if take_over?(take_pid, take_ref, state.counter + 1, true) do
           # Since we are in process, also bump the counter
           state = reset_state(bump_counter(state))
-          loop(state, :ok, evaluator, evaluator_ref, input)
+          loop(state, evaluator, evaluator_ref, input)
         else
           callback.(state)
         end
@@ -342,7 +349,7 @@ defmodule IEx.Server do
   # Once the rerunning session restarts, we keep the same evaluator_options
   # and rollback to a new evaluator.
   defp reset_state(state) do
-    %{state | parser_state: ""}
+    %{state | parser_state: []}
   end
 
   defp bump_counter(state) do
@@ -351,28 +358,42 @@ defmodule IEx.Server do
 
   ## IO
 
-  defp io_get(prompt) do
+  defp io_get(prompt, counter, parser_state) do
     gl = Process.group_leader()
     ref = Process.monitor(gl)
-    command = {:get_until, :unicode, prompt, __MODULE__, :__parse__, []}
+    command = {:get_until, :unicode, prompt, __MODULE__, :__parse__, [{counter, parser_state}]}
     send(gl, {:io_request, self(), ref, command})
     ref
   end
 
   @doc false
-  def __parse__([], :eof), do: {:done, :eof, []}
-  def __parse__([], chars), do: {:done, List.to_string(chars), []}
+  def __parse__(_, :eof, _parser_state), do: {:done, :eof, []}
+
+  def __parse__([], chars, {counter, parser_state} = to_be_unused) do
+    __parse__({counter, parser_state, IEx.Config.parser()}, chars, to_be_unused)
+  end
+
+  def __parse__({counter, parser_state, mfa}, chars, _unused) do
+    {parser_module, parser_fun, args} = mfa
+    args = [chars, [line: counter, file: "iex"], parser_state | args]
+
+    case apply(parser_module, parser_fun, args) do
+      {:ok, forms, parser_state} -> {:done, {:ok, forms, parser_state}, []}
+      # TODO: Return new prompt when supported in Erlang/OTP
+      {:incomplete, parser_state} -> {:more, {counter, parser_state, mfa}}
+    end
+  catch
+    kind, error ->
+      {:done, {:error, kind, error, __STACKTRACE__}, []}
+  end
 
-  defp prompt(status, prefix, counter) do
-    {mode, prefix} =
+  defp prompt(prefix, counter) do
+    prompt =
       if Node.alive?() do
-        {prompt_mode(status, :alive), default_prefix(status, prefix)}
+        IEx.Config.alive_prompt()
       else
-        {prompt_mode(status, :default), default_prefix(status, prefix)}
+        IEx.Config.default_prompt()
       end
-
-    prompt =
-      apply(IEx.Config, mode, [])
       |> String.replace("%counter", to_string(counter))
       |> String.replace("%prefix", to_string(prefix))
       |> String.replace("%node", to_string(node()))
@@ -380,14 +401,6 @@ defmodule IEx.Server do
     [prompt, " "]
   end
 
-  defp default_prefix(:incomplete, _prefix), do: "..."
-  defp default_prefix(_ok_or_error, prefix), do: prefix
-
-  defp prompt_mode(:incomplete, :default), do: :continuation_prompt
-  defp prompt_mode(:incomplete, :alive), do: :alive_continuation_prompt
-  defp prompt_mode(_ok_or_error, :default), do: :default_prompt
-  defp prompt_mode(_ok_or_error, :alive), do: :alive_prompt
-
   defp io_error(result) do
     IO.puts(:stdio, IEx.color(:eval_error, result))
   end
diff --git a/lib/iex/test/iex/autocomplete_test.exs b/lib/iex/test/iex/autocomplete_test.exs
index ed17448df..d5e0d3119 100644
--- a/lib/iex/test/iex/autocomplete_test.exs
+++ b/lib/iex/test/iex/autocomplete_test.exs
@@ -13,8 +13,8 @@ defmodule IEx.AutocompleteTest do
     ExUnit.CaptureIO.capture_io(fn ->
       evaluator = Process.get(:evaluator)
       Process.group_leader(evaluator, Process.group_leader())
-      send(evaluator, {:eval, self(), line <> "\n", 1, ""})
-      assert_receive {:evaled, _, _, _}
+      send(evaluator, {:eval, self(), Code.string_to_quoted!(line <> "\n"), 1})
+      assert_receive {:evaled, _, _}
     end)
   end
 
diff --git a/lib/iex/test/iex/interaction_test.exs b/lib/iex/test/iex/interaction_test.exs
index fc006e174..86528e140 100644
--- a/lib/iex/test/iex/interaction_test.exs
+++ b/lib/iex/test/iex/interaction_test.exs
@@ -107,6 +107,8 @@ defmodule IEx.InteractionTest do
     assert capture_iex("1\n", opts, [], true) == "prompt(1)> 1\nprompt(2)>"
   end
 
+  # TODO: Implement this based on Erlang/OTP version
+  @tag :skip
   test "continuation prompt" do
     opts = [default_prompt: "%prefix(%counter)>", continuation_prompt: "%prefix(%counter)>>>"]
     assert capture_iex("[\n1\n]\n", opts, [], true) == "iex(1)> ...(1)>>> ...(1)>>> [1]\niex(2)>"
@@ -191,9 +193,9 @@ defmodule IEx.InteractionTest do
       end
     end
 
-    assert capture_iex("foo", parser: {EchoParser, :parse, []}) == "\"foo\""
+    assert capture_iex("foo", parser: {EchoParser, :parse, []}) == "~c\"foo\""
   after
-    IEx.configure(parser: {IEx.Evaluator, :parse, []})
+    IEx.configure(parser: {IEx.Server, :parse, []})
   end
 
   ## .iex file loading