Add end of expression token information

This also renames pairing_metadata to token_metadata,
as it mostly contains information about related tokens
and not necessarily pairs.

7 files changed, 265 insertions, 215 deletions

diff --git a/lib/elixir/lib/code.ex b/lib/elixir/lib/code.ex
index 00125ee50..1ad871f33 100644
--- a/lib/elixir/lib/code.ex
+++ b/lib/elixir/lib/code.ex
@@ -665,9 +665,10 @@ defmodule Code do
       when non-existing atoms are found by the tokenizer.
       Defaults to `false`.
 
-    * `:pairing_metadata` - when `true`, includes metadata about `do/end` blocks,
-      end of line, and closing pairs. See `t:Macro.metadata/0`. Defaults
-      to `false`.
+    * `:token_metadata` - when `true`, includes token-related metadata
+      in the expression AST, such as metadata for `do` and `end` tokens,
+      for closing tokens, end of expressions, as well as delimiters for
+      sigils. See `t:Macro.metadata/0`. Defaults to `false`.
 
     * `:static_atom_encoder` - the static atom encoder function, see
       "The `:static_atom_encoder` function" section below. Note this
diff --git a/lib/elixir/lib/code/formatter.ex b/lib/elixir/lib/code/formatter.ex
index 9d6ab45f4..2bb193b46 100644
--- a/lib/elixir/lib/code/formatter.ex
+++ b/lib/elixir/lib/code/formatter.ex
@@ -206,8 +206,8 @@ defmodule Code.Formatter do
     ]
 
     parser_options = [
-      elixir_private_formatter_metadata: true,
-      pairing_metadata: true
+      elixir_private_wrap_literals: true,
+      token_metadata: true
     ]
 
     with {:ok, tokens} <- :elixir.string_to_tokens(charlist, line, file, tokenizer_options),
@@ -611,14 +611,14 @@ defmodule Code.Formatter do
   end
 
   defp block_args_to_algebra(args, min_line, max_line, state) do
-    quoted_to_algebra = fn {kind, meta, _} = arg, _args, doc_newlines, state ->
-      doc_newlines = Keyword.get(meta, :newlines, doc_newlines)
+    quoted_to_algebra = fn {kind, meta, _} = arg, _args, state ->
+      newlines = meta[:end_of_expression][:newlines] || 1
       {doc, state} = quoted_to_algebra(arg, :block, state)
-      {doc, block_next_line(kind), doc_newlines, state}
+      {{doc, block_next_line(kind), newlines}, state}
     end
 
     {args_docs, _comments?, state} =
-      quoted_to_algebra_with_comments(args, [], min_line, max_line, 2, state, quoted_to_algebra)
+      quoted_to_algebra_with_comments(args, [], min_line, max_line, state, quoted_to_algebra)
 
     case args_docs do
       [] -> {@empty, state}
@@ -706,14 +706,14 @@ defmodule Code.Formatter do
       unwrap_right(right_arg, op, meta, right_context, [{{:root, left_context}, left_arg}])
 
     operand_to_algebra = fn
-      {{:root, context}, arg}, _args, newlines, state ->
+      {{:root, context}, arg}, _args, state ->
         {doc, state} = binary_operand_to_algebra(arg, context, state, op, op_info, :left, 2)
-        {doc, @empty, newlines, state}
+        {{doc, @empty, 1}, state}
 
-      {{kind, context}, arg}, _args, newlines, state ->
+      {{kind, context}, arg}, _args, state ->
         {doc, state} = binary_operand_to_algebra(arg, context, state, op, op_info, kind, 0)
         doc = doc |> nest_by_length(op_string) |> force_keyword(arg)
-        {concat(op_string, doc), @empty, newlines, state}
+        {{concat(op_string, doc), @empty, 1}, state}
     end
 
     {doc, state} =
@@ -744,15 +744,15 @@ defmodule Code.Formatter do
       unwrap_pipes(left_arg, meta, left_context, [{{op, right_context}, right_arg}])
 
     operand_to_algebra = fn
-      {{:root, context}, arg}, _args, newlines, state ->
+      {{:root, context}, arg}, _args, state ->
         {doc, state} = binary_operand_to_algebra(arg, context, state, op, op_info, :left, 2)
-        {doc, @empty, newlines, state}
+        {{doc, @empty, 1}, state}
 
-      {{op, context}, arg}, _args, newlines, state ->
+      {{op, context}, arg}, _args, state ->
         op_info = Code.Identifier.binary_op(op)
         op_string = Atom.to_string(op) <> " "
         {doc, state} = binary_operand_to_algebra(arg, context, state, op, op_info, :right, 0)
-        {concat(op_string, doc), @empty, newlines, state}
+        {{concat(op_string, doc), @empty, 1}, state}
     end
 
     operand_to_algebra_with_comments(pipes, meta, min_line, max_line, state, operand_to_algebra)
@@ -885,7 +885,7 @@ defmodule Code.Formatter do
 
   defp operand_to_algebra_with_comments(operands, meta, min_line, max_line, state, fun) do
     {docs, comments?, state} =
-      quoted_to_algebra_with_comments(operands, [], min_line, max_line, 1, state, fun)
+      quoted_to_algebra_with_comments(operands, [], min_line, max_line, state, fun)
 
     if comments? or eol?(meta) do
       {docs |> Enum.reduce(&line(&2, &1)) |> force_unfit(), state}
@@ -1628,7 +1628,7 @@ defmodule Code.Formatter do
     min_line = line(meta)
     max_line = closing_line(meta)
 
-    arg_to_algebra = fn arg, args, newlines, state ->
+    arg_to_algebra = fn arg, args, state ->
       {doc, state} = fun.(arg, state)
 
       doc =
@@ -1639,7 +1639,7 @@ defmodule Code.Formatter do
           [] when last_arg_mode == :none -> doc
         end
 
-      {doc, @empty, newlines, state}
+      {{doc, @empty, 1}, state}
     end
 
     # If skipping parens, we cannot extract the comments of the first
@@ -1647,15 +1647,15 @@ defmodule Code.Formatter do
     {args, acc, state} =
       case args do
         [head | tail] when skip_parens? ->
-          {head, next_line, newlines, state} = arg_to_algebra.(head, tail, 1, state)
-          {tail, [{head, next_line, newlines}], state}
+          {doc_triplet, state} = arg_to_algebra.(head, tail, state)
+          {tail, [doc_triplet], state}
 
         _ ->
           {args, [], state}
       end
 
     {args_docs, comments?, state} =
-      quoted_to_algebra_with_comments(args, acc, min_line, max_line, 1, state, arg_to_algebra)
+      quoted_to_algebra_with_comments(args, acc, min_line, max_line, state, arg_to_algebra)
 
     cond do
       args_docs == [] ->
@@ -1859,13 +1859,13 @@ defmodule Code.Formatter do
   end
 
   defp clause_args_to_algebra(args, min_line, state) do
-    arg_to_algebra = fn arg, _args, newlines, state ->
+    arg_to_algebra = fn arg, _args, state ->
       {doc, state} = clause_args_to_algebra(arg, state)
-      {doc, @empty, newlines, state}
+      {{doc, @empty, 1}, state}
     end
 
     {args_docs, comments?, state} =
-      quoted_to_algebra_with_comments([args], [], min_line, @min_line, 1, state, arg_to_algebra)
+      quoted_to_algebra_with_comments([args], [], min_line, @min_line, state, arg_to_algebra)
 
     if comments? do
       {Enum.reduce(args_docs, &line(&2, &1)), state}
@@ -1893,63 +1893,65 @@ defmodule Code.Formatter do
 
   ## Quoted helpers for comments
 
-  defp quoted_to_algebra_with_comments(args, acc, min_line, max_line, newlines, state, fun) do
+  defp quoted_to_algebra_with_comments(args, acc, min_line, max_line, state, fun) do
     {pre_comments, state} =
       get_and_update_in(state.comments, fn comments ->
         Enum.split_while(comments, fn {line, _, _} -> line <= min_line end)
       end)
 
     {docs, comments?, state} =
-      each_quoted_to_algebra_with_comments(args, acc, max_line, newlines, state, false, fun)
+      each_quoted_to_algebra_with_comments(args, acc, max_line, state, false, fun)
 
     {docs, comments?, update_in(state.comments, &(pre_comments ++ &1))}
   end
 
-  defp each_quoted_to_algebra_with_comments([], acc, max_line, _newlines, state, comments?, _fun) do
-    %{comments: comments} = state
-    {current, comments} = Enum.split_with(comments, fn {line, _, _} -> line < max_line end)
-
-    extra = for {_, {previous, _}, doc} <- current, do: {doc, @empty, previous}
-    args_docs = merge_algebra_with_comments(Enum.reverse(acc, extra), @empty)
-    {args_docs, comments? or extra != [], %{state | comments: comments}}
+  defp each_quoted_to_algebra_with_comments([], acc, max_line, state, comments?, _fun) do
+    {acc, comments, comments?} = extract_comments_before(max_line, acc, state.comments, comments?)
+    args_docs = merge_algebra_with_comments(Enum.reverse(acc), @empty)
+    {args_docs, comments?, %{state | comments: comments}}
   end
 
-  defp each_quoted_to_algebra_with_comments(args, acc, max_line, newlines, state, comments?, fun) do
+  defp each_quoted_to_algebra_with_comments(args, acc, max_line, state, comments?, fun) do
     [arg | args] = args
     {doc_start, doc_end} = traverse_line(arg, {@max_line, @min_line})
 
-    {doc_newlines, acc, comments, comments?} =
-      extract_comments_before(doc_start, newlines, acc, state.comments, comments?)
+    {acc, comments, comments?} =
+      extract_comments_before(doc_start, acc, state.comments, comments?)
 
-    {doc, next_line, doc_newlines, state} =
-      fun.(arg, args, doc_newlines, %{state | comments: comments})
+    {doc_triplet, state} = fun.(arg, args, %{state | comments: comments})
 
-    {doc_newlines, acc, comments, comments?} =
-      extract_comments_trailing(doc_start, doc_end, doc_newlines, acc, state.comments, comments?)
+    {acc, comments, comments?} =
+      extract_comments_trailing(doc_start, doc_end, acc, state.comments, comments?)
 
-    acc = [{doc, next_line, doc_newlines} | acc]
+    acc = [doc_triplet | acc]
     state = %{state | comments: comments}
-    each_quoted_to_algebra_with_comments(args, acc, max_line, newlines, state, comments?, fun)
+    each_quoted_to_algebra_with_comments(args, acc, max_line, state, comments?, fun)
   end
 
-  defp extract_comments_before(max, _, acc, [{line, _, _} = comment | rest], _) when line < max do
+  defp extract_comments_before(max, acc, [{line, _, _} = comment | rest], _) when line < max do
     {_, {previous, next}, doc} = comment
-    acc = [{doc, @empty, previous} | acc]
-    extract_comments_before(max, next, acc, rest, true)
+    acc = [{doc, @empty, next} | add_previous_to_acc(acc, previous)]
+    extract_comments_before(max, acc, rest, true)
   end
 
-  defp extract_comments_before(_max, newlines, acc, rest, comments?) do
-    {newlines, acc, rest, comments?}
+  defp extract_comments_before(_max, acc, rest, comments?) do
+    {acc, rest, comments?}
   end
 
-  defp extract_comments_trailing(min, max, newlines, acc, [{line, _, doc_comment} | rest], _)
+  defp add_previous_to_acc([{doc, next_line, newlines} | acc], previous) when newlines < previous,
+    do: [{doc, next_line, previous} | acc]
+
+  defp add_previous_to_acc(acc, _previous),
+    do: acc
+
+  defp extract_comments_trailing(min, max, acc, [{line, _, doc_comment} | rest], _)
        when line >= min and line <= max do
-    acc = [{doc_comment, @empty, newlines} | acc]
-    extract_comments_trailing(min, max, 1, acc, rest, true)
+    acc = [{doc_comment, @empty, 1} | acc]
+    extract_comments_trailing(min, max, acc, rest, true)
   end
 
-  defp extract_comments_trailing(_min, _max, newlines, acc, rest, comments?) do
-    {newlines, acc, rest, comments?}
+  defp extract_comments_trailing(_min, _max, acc, rest, comments?) do
+    {acc, rest, comments?}
   end
 
   defp traverse_line({expr, meta, args}, {min, max}) do
@@ -1981,8 +1983,8 @@ defmodule Code.Formatter do
   #      (except for module attributes)
   #   3. empty lines are collapsed as to not exceed more than one
   #
-  defp merge_algebra_with_comments([{doc, next_line, _newlines} | docs], left) do
-    right = next_line_separator(docs, next_line)
+  defp merge_algebra_with_comments([{doc, next_line, newlines} | docs], left) do
+    right = if newlines >= @newlines, do: line(), else: next_line
 
     doc =
       if left != @empty do
@@ -2063,14 +2065,6 @@ defmodule Code.Formatter do
     end)
   end
 
-  defp next_line_separator([{_doc, _next_line, newlines} | _], next_line) do
-    if newlines >= @newlines, do: line(), else: next_line
-  end
-
-  defp next_line_separator([], _) do
-    line()
-  end
-
   defp module_attribute_read?({:@, _, [{var, _, var_context}]})
        when is_atom(var) and is_atom(var_context) do
     Code.Identifier.classify(var) == :callable_local
@@ -2235,7 +2229,7 @@ defmodule Code.Formatter do
   end
 
   defp eol?(meta) do
-    Keyword.get(meta, :eol, false)
+    Keyword.get(meta, :newlines, 0) > 0
   end
 
   defp meta?(meta, key) do
diff --git a/lib/elixir/lib/macro.ex b/lib/elixir/lib/macro.ex
index e4f2aa5a3..da7cc3a8c 100644
--- a/lib/elixir/lib/macro.ex
+++ b/lib/elixir/lib/macro.ex
@@ -158,18 +158,20 @@ defmodule Macro do
 
   The following metadata keys are enabled by `Code.string_to_quoted/2`:
 
-    * `:column` - the column number of the AST node (when `:columns` is true)
-    * `:do` - contains metadata about the `do` location in a function call with
-      `do/end` blocks (when `:pairing_metadata` is true)
-    * `:end` - contains metadata about the `end` location in a function call with
-      `do/end` blocks (when `:pairing_metadata` is true)
     * `:closing` - contains metadata about the closing pair, such as a `}`
       in a tuple or in a map, or such as the closing `)` in a function call
-      with parens (when `:pairing_metadata` is true)
-    * `:eol` - is set to true when the opening pair, such as `{` or `(`, are
-      followed by the end of the line (when `:pairing_metadata` is true)
+      with parens. The `:closing` does not delimit the end of expression if
+      there are `:do` and `:end` metadata  (when `:token_metadata` is true)
+    * `:column` - the column number of the AST node (when `:columns` is true)
     * `:delimiter` - contains the opening delimiter for sigils as a string
       (such as `"{"`, `"/"`, etc)
+    * `:do` - contains metadata about the `do` location in a function call with
+      `do/end` blocks (when `:token_metadata` is true)
+    * `:end` - contains metadata about the `end` location in a function call with
+      `do/end` blocks (when `:token_metadata` is true)
+    * `:end_of_expression` - denotes when the end of expression effectively
+      happens. Available for all expressions except the last one inside a
+      `__block__` (when `:token_metadata` is true)
 
   The following metadata keys are private:
 
diff --git a/lib/elixir/src/elixir.erl b/lib/elixir/src/elixir.erl
index 5b205f3bc..98c7b8d9f 100644
--- a/lib/elixir/src/elixir.erl
+++ b/lib/elixir/src/elixir.erl
@@ -332,8 +332,8 @@ tokens_to_quoted(Tokens, File, Opts) ->
   after
     erase(elixir_parser_file),
     erase(elixir_parser_columns),
-    erase(elixir_pairing_metadata),
-    erase(elixir_formatter_metadata)
+    erase(elixir_token_metadata),
+    erase(elixir_wrap_literals)
   end.
 
 parser_line({Line, _, _}) ->
@@ -361,10 +361,10 @@ to_binary(List) when is_list(List) -> elixir_utils:characters_to_binary(List);
 to_binary(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8).
 
 handle_parsing_opts(File, Opts) ->
-  FormatterMetadata = lists:keyfind(elixir_private_formatter_metadata, 1, Opts) == {elixir_private_formatter_metadata, true},
-  PairingMetadata = lists:keyfind(pairing_metadata, 1, Opts) == {pairing_metadata, true},
+  FormatterMetadata = lists:keyfind(elixir_private_wrap_literals, 1, Opts) == {elixir_private_wrap_literals, true},
+  PairingMetadata = lists:keyfind(token_metadata, 1, Opts) == {token_metadata, true},
   Columns = lists:keyfind(columns, 1, Opts) == {columns, true},
   put(elixir_parser_file, File),
   put(elixir_parser_columns, Columns),
-  put(elixir_pairing_metadata, PairingMetadata),
-  put(elixir_formatter_metadata, FormatterMetadata).
+  put(elixir_token_metadata, PairingMetadata),
+  put(elixir_wrap_literals, FormatterMetadata).
diff --git a/lib/elixir/src/elixir_parser.yrl b/lib/elixir/src/elixir_parser.yrl
index a7277e6cb..3ae5203c6 100644
--- a/lib/elixir/src/elixir_parser.yrl
+++ b/lib/elixir/src/elixir_parser.yrl
@@ -89,7 +89,7 @@ grammar -> '$empty' : {'__block__', [], []}.
 
 % Note expressions are on reverse order
 expr_list -> expr : ['$1'].
-expr_list -> expr_list eoe expr : [annotate_newlines('$2', '$3') | '$1'].
+expr_list -> expr_list eoe expr : ['$3' | annotate_eoe('$2', '$1')].
 
 expr -> matched_expr : '$1'.
 expr -> no_parens_expr : '$1'.
@@ -301,7 +301,7 @@ eoe -> ';' : '$1'.
 eoe -> eol ';' : '$1'.
 
 fn_eoe -> 'fn' : '$1'.
-fn_eoe -> 'fn' eoe : next_is_eol('$1').
+fn_eoe -> 'fn' eoe : next_is_eol('$1', '$2').
 
 do_eoe -> 'do' : '$1'.
 do_eoe -> 'do' eoe : '$1'.
@@ -313,7 +313,7 @@ block_eoe -> block_identifier : '$1'.
 block_eoe -> block_identifier eoe : '$1'.
 
 stab -> stab_expr : ['$1'].
-stab -> stab eoe stab_expr : [annotate_newlines('$2', '$3') | '$1'].
+stab -> stab eoe stab_expr : ['$3' | annotate_eoe('$2', '$1')].
 
 stab_eoe -> stab : '$1'.
 stab_eoe -> stab eoe : '$1'.
@@ -349,24 +349,24 @@ block_list -> block_item block_list : ['$1' | '$2'].
 %% Helpers
 
 open_paren -> '('      : '$1'.
-open_paren -> '(' eol  : next_is_eol('$1').
+open_paren -> '(' eol  : next_is_eol('$1', '$2').
 close_paren -> ')'     : '$1'.
 close_paren -> eol ')' : '$2'.
 
 empty_paren -> open_paren ')' : '$1'.
 
 open_bracket  -> '['     : '$1'.
-open_bracket  -> '[' eol : next_is_eol('$1').
+open_bracket  -> '[' eol : next_is_eol('$1', '$2').
 close_bracket -> ']'     : '$1'.
 close_bracket -> eol ']' : '$2'.
 
 open_bit  -> '<<'     : '$1'.
-open_bit  -> '<<' eol : next_is_eol('$1').
+open_bit  -> '<<' eol : next_is_eol('$1', '$2').
 close_bit -> '>>'     : '$1'.
 close_bit -> eol '>>' : '$2'.
 
 open_curly  -> '{'     : '$1'.
-open_curly  -> '{' eol : next_is_eol('$1').
+open_curly  -> '{' eol : next_is_eol('$1', '$2').
 close_curly -> '}'     : '$1'.
 close_curly -> eol '}' : '$2'.
 
@@ -387,49 +387,49 @@ match_op_eol -> match_op : '$1'.
 match_op_eol -> match_op eol : '$1'.
 
 dual_op_eol -> dual_op : '$1'.
-dual_op_eol -> dual_op eol : next_is_eol('$1').
+dual_op_eol -> dual_op eol : next_is_eol('$1', '$2').
 
 mult_op_eol -> mult_op : '$1'.
-mult_op_eol -> mult_op eol : next_is_eol('$1').
+mult_op_eol -> mult_op eol : next_is_eol('$1', '$2').
 
 two_op_eol -> two_op : '$1'.
-two_op_eol -> two_op eol : next_is_eol('$1').
+two_op_eol -> two_op eol : next_is_eol('$1', '$2').
 
 three_op_eol -> three_op : '$1'.
-three_op_eol -> three_op eol : next_is_eol('$1').
+three_op_eol -> three_op eol : next_is_eol('$1', '$2').
 
 pipe_op_eol -> pipe_op : '$1'.
-pipe_op_eol -> pipe_op eol : next_is_eol('$1').
+pipe_op_eol -> pipe_op eol : next_is_eol('$1', '$2').
 
 and_op_eol -> and_op : '$1'.
-and_op_eol -> and_op eol : next_is_eol('$1').
+and_op_eol -> and_op eol : next_is_eol('$1', '$2').
 
 or_op_eol -> or_op : '$1'.
-or_op_eol -> or_op eol : next_is_eol('$1').
+or_op_eol -> or_op eol : next_is_eol('$1', '$2').
 
 in_op_eol -> in_op : '$1'.
-in_op_eol -> in_op eol : next_is_eol('$1').
+in_op_eol -> in_op eol : next_is_eol('$1', '$2').
 
 in_match_op_eol -> in_match_op : '$1'.
-in_match_op_eol -> in_match_op eol : next_is_eol('$1').
+in_match_op_eol -> in_match_op eol : next_is_eol('$1', '$2').
 
 type_op_eol -> type_op : '$1'.
-type_op_eol -> type_op eol : next_is_eol('$1').
+type_op_eol -> type_op eol : next_is_eol('$1', '$2').
 
 when_op_eol -> when_op : '$1'.
-when_op_eol -> when_op eol : next_is_eol('$1').
+when_op_eol -> when_op eol : next_is_eol('$1', '$2').
 
 stab_op_eol -> stab_op : '$1'.
-stab_op_eol -> stab_op eol : next_is_eol('$1').
+stab_op_eol -> stab_op eol : next_is_eol('$1', '$2').
 
 comp_op_eol -> comp_op : '$1'.
-comp_op_eol -> comp_op eol : next_is_eol('$1').
+comp_op_eol -> comp_op eol : next_is_eol('$1', '$2').
 
 rel_op_eol -> rel_op : '$1'.
-rel_op_eol -> rel_op eol : next_is_eol('$1').
+rel_op_eol -> rel_op eol : next_is_eol('$1', '$2').
 
 arrow_op_eol -> arrow_op : '$1'.
-arrow_op_eol -> arrow_op eol : next_is_eol('$1').
+arrow_op_eol -> arrow_op eol : next_is_eol('$1', '$2').
 
 % Dot operator
 
@@ -442,7 +442,7 @@ dot_identifier -> matched_expr dot_op identifier : build_dot('$2', '$1', '$3').
 dot_alias -> alias : build_alias('$1').
 dot_alias -> matched_expr dot_op alias : build_dot_alias('$2', '$1', '$3').
 dot_alias -> matched_expr dot_op open_curly '}' : build_dot_container('$2', '$1', [], []).
-dot_alias -> matched_expr dot_op open_curly container_args close_curly : build_dot_container('$2', '$1', '$4', eol_pair('$3', '$5')).
+dot_alias -> matched_expr dot_op open_curly container_args close_curly : build_dot_container('$2', '$1', '$4', newlines_pair('$3', '$5')).
 
 dot_op_identifier -> op_identifier : '$1'.
 dot_op_identifier -> matched_expr dot_op op_identifier : build_dot('$2', '$1', '$3').
@@ -510,19 +510,19 @@ call_args_parens_base -> call_args_parens_expr : ['$1'].
 call_args_parens_base -> call_args_parens_base ',' call_args_parens_expr : ['$3' | '$1'].
 
 call_args_parens -> open_paren ')' :
-                      {eol_pair('$1', '$2'), []}.
+                      {newlines_pair('$1', '$2'), []}.
 call_args_parens -> open_paren no_parens_expr close_paren :
-                      {eol_pair('$1', '$3'), ['$2']}.
+                      {newlines_pair('$1', '$3'), ['$2']}.
 call_args_parens -> open_paren kw_base close_paren :
-                      {eol_pair('$1', '$3'), [reverse('$2')]}.
+                      {newlines_pair('$1', '$3'), [reverse('$2')]}.
 call_args_parens -> open_paren kw_base ',' close_paren :
-                      warn_trailing_comma('$3'), {eol_pair('$1', '$4'), [reverse('$2')]}.
+                      warn_trailing_comma('$3'), {newlines_pair('$1', '$4'), [reverse('$2')]}.
 call_args_parens -> open_paren call_args_parens_base close_paren :
-                      {eol_pair('$1', '$3'), reverse('$2')}.
+                      {newlines_pair('$1', '$3'), reverse('$2')}.
 call_args_parens -> open_paren call_args_parens_base ',' kw_base close_paren :
-                      {eol_pair('$1', '$5'), reverse([reverse('$4') | '$2'])}.
+                      {newlines_pair('$1', '$5'), reverse([reverse('$4') | '$2'])}.
 call_args_parens -> open_paren call_args_parens_base ',' kw_base ',' close_paren :
-                      warn_trailing_comma('$5'), {eol_pair('$1', '$6'), reverse([reverse('$4') | '$2'])}.
+                      warn_trailing_comma('$5'), {newlines_pair('$1', '$6'), reverse([reverse('$4') | '$2'])}.
 
 % KV
 
@@ -619,8 +619,8 @@ Erlang code.
 
 -define(file(), get(elixir_parser_file)).
 -define(columns(), get(elixir_parser_columns)).
--define(pairing_metadata(), get(elixir_pairing_metadata)).
--define(formatter_metadata(), get(elixir_formatter_metadata)).
+-define(token_metadata(), get(elixir_token_metadata)).
+-define(wrap_literals(), get(elixir_wrap_literals)).
 
 -define(id(Token), element(1, Token)).
 -define(location(Token), element(2, Token)).
@@ -644,10 +644,9 @@ meta_from_location({Line, Column, _}) ->
   end.
 
 line_from_location({Line, _Column, _}) -> Line.
-is_eol({_, _, Eol}) -> is_integer(Eol) and (Eol > 0).
 
 do_end_meta(Do, End) ->
-  case ?pairing_metadata() of
+  case ?token_metadata() of
     true ->
       [{do, meta_from_location(?location(Do))}, {'end', meta_from_location(?location(End))}];
     false ->
@@ -655,7 +654,7 @@ do_end_meta(Do, End) ->
   end.
 
 meta_from_token_with_closing(Begin, End) ->
-  case ?pairing_metadata() of
+  case ?token_metadata() of
     true ->
       [{closing, meta_from_location(?location(End))} | meta_from_token(Begin)];
     false ->
@@ -671,7 +670,7 @@ handle_literal(Literal, Token) ->
   handle_literal(Literal, Token, []).
 
 handle_literal(Literal, Token, ExtraMeta) ->
-  case ?formatter_metadata() of
+  case ?wrap_literals() of
     true -> {'__block__', ExtraMeta ++ meta_from_token(Token), [Literal]};
     false -> Literal
   end.
@@ -692,34 +691,34 @@ build_op({_Kind, Location, 'in'}, {UOp, _, [Left]}, Right) when ?rearrange_uop(U
 build_op({_Kind, Location, 'not in'}, Left, Right) ->
   InMeta = meta_from_location(Location),
   NotMeta =
-    case ?formatter_metadata() of
+    case ?wrap_literals() of
       true -> [{operator, 'not in'} | InMeta];
       false -> InMeta
     end,
   {'not', NotMeta, [{'in', InMeta, [Left, Right]}]};
 build_op({_Kind, Location, Op}, Left, Right) ->
-  {Op, eol_op(Location) ++ meta_from_location(Location), [Left, Right]}.
+  {Op, newlines_op(Location) ++ meta_from_location(Location), [Left, Right]}.
 
 build_unary_op({_Kind, Location, Op}, Expr) ->
   {Op, meta_from_location(Location), [Expr]}.
 
 build_list(Left, Args, Right) ->
-  {handle_literal(Args, Left, eol_pair(Left, Right)), ?location(Left)}.
+  {handle_literal(Args, Left, newlines_pair(Left, Right)), ?location(Left)}.
 
 build_tuple(Left, [Arg1, Arg2], Right) ->
-  handle_literal({Arg1, Arg2}, Left, eol_pair(Left, Right));
+  handle_literal({Arg1, Arg2}, Left, newlines_pair(Left, Right));
 build_tuple(Left, Args, Right) ->
-  {'{}', eol_pair(Left, Right) ++ meta_from_token(Left), Args}.
+  {'{}', newlines_pair(Left, Right) ++ meta_from_token(Left), Args}.
 
 build_bit(Left, Args, Right) ->
-  {'<<>>', eol_pair(Left, Right) ++ meta_from_token(Left), Args}.
+  {'<<>>', newlines_pair(Left, Right) ++ meta_from_token(Left), Args}.
 
 build_map(Left, Args, Right) ->
-  {'%{}', eol_pair(Left, Right) ++ meta_from_token(Left), Args}.
+  {'%{}', newlines_pair(Left, Right) ++ meta_from_token(Left), Args}.
 
 build_map_update(Left, {Pipe, Struct, Map}, Right, Extra) ->
   Op = build_op(Pipe, Struct, append_non_empty(Map, Extra)),
-  {'%{}', eol_pair(Left, Right) ++ meta_from_token(Left), [Op]}.
+  {'%{}', newlines_pair(Left, Right) ++ meta_from_token(Left), [Op]}.
 
 %% Blocks
 
@@ -732,36 +731,52 @@ build_block([Expr]) ->
 build_block(Exprs) ->
   {'__block__', [], Exprs}.
 
-%% End of line and newlines
+%% Newlines
 
-eol_pair(Left, Right) ->
-  case ?pairing_metadata() of
+newlines_pair(Left, Right) ->
+  case ?token_metadata() of
     true ->
-      [
-        {eol, is_eol(?location(Left))},
-        {closing, meta_from_location(?location(Right))}
-      ];
+      newlines(?location(Left), [{closing, meta_from_location(?location(Right))}]);
     false ->
       []
   end.
 
-eol_op(Location) ->
-  case ?formatter_metadata() and is_eol(Location) of
-    true -> [{eol, true}];
+newlines_op(Location) ->
+  case ?token_metadata() of
+    true -> newlines(Location, []);
     false -> []
   end.
 
-next_is_eol(Token) ->
+next_is_eol(Token, {_, {_, _, Count}}) ->
   {Line, Column, _} = ?location(Token),
-  setelement(2, Token, {Line, Column, 1}).
+  setelement(2, Token, {Line, Column, Count}).
 
-annotate_newlines({_, {_, _, Count}}, {Left, Meta, Right}) when is_integer(Count), is_list(Meta) ->
-  case ?formatter_metadata() of
-    true -> {Left, [{newlines, Count} | Meta], Right};
-    false -> {Left, Meta, Right}
-  end;
-annotate_newlines(_, Expr) ->
-  Expr.
+newlines({_, _, Count}, Meta) when is_integer(Count) and (Count > 0) ->
+  [{newlines, Count} | Meta];
+newlines(_, Meta) ->
+  Meta.
+
+annotate_eoe(Token, Stack) ->
+  case ?token_metadata() of
+    true ->
+      case {Token, Stack} of
+        {{_, Location}, [{'->', StabMeta, [StabArgs, {Left, Meta, Right}]} | Rest]} when is_list(Meta) ->
+          [{'->', StabMeta, [StabArgs, {Left, [{end_of_expression, end_of_expression(Location)} | Meta], Right}]} | Rest];
+
+        {{_, Location}, [{Left, Meta, Right} | Rest]} when is_list(Meta) ->
+          [{Left, [{end_of_expression, end_of_expression(Location)} | Meta], Right} | Rest];
+
+        _ ->
+          Stack
+      end;
+    false ->
+      Stack
+  end.
+
+end_of_expression({_, _, Count} = Location) when is_integer(Count) ->
+  [{newlines, Count} | meta_from_location(Location)];
+end_of_expression(Location) ->
+  meta_from_location(Location).
 
 %% Dots
 
@@ -823,7 +838,7 @@ build_identifier({_, Location, Identifier}, Args) ->
 build_fn(Fn, Stab, End) ->
   case check_stab(Stab, none) of
     stab ->
-      Meta = eol_op(?location(Fn)) ++ meta_from_token_with_closing(Fn, End),
+      Meta = newlines_op(?location(Fn)) ++ meta_from_token_with_closing(Fn, End),
       {fn, Meta, collect_stab(Stab, [], [])};
     block ->
       return_error(meta_from_token(Fn), "expected anonymous functions to be defined with -> inside: ", "'fn'")
@@ -839,7 +854,7 @@ build_access(Expr, {List, Location}) ->
 
 build_sigil({sigil, Location, Sigil, Parts, Modifiers, Delimiter}) ->
   Meta = meta_from_location(Location),
-  MetaWithDelimiter = case ?pairing_metadata() of
+  MetaWithDelimiter = case ?token_metadata() of
     true -> [{delimiter, Delimiter} | Meta];
     false -> Meta
   end,
@@ -857,7 +872,7 @@ build_bin_string({bin_string, _Location, [H]} = Token, ExtraMeta) when is_binary
   handle_literal(H, Token, ExtraMeta);
 build_bin_string({bin_string, Location, Args}, ExtraMeta) ->
   Meta =
-    case ?formatter_metadata() of
+    case ?wrap_literals() of
       true -> ExtraMeta ++ meta_from_location(Location);
       false -> meta_from_location(Location)
     end,
@@ -868,7 +883,7 @@ build_list_string({list_string, _Location, [H]} = Token, ExtraMeta) when is_bina
 build_list_string({list_string, Location, Args}, ExtraMeta) ->
   Meta = meta_from_location(Location),
   MetaWithExtra =
-    case ?formatter_metadata() of
+    case ?wrap_literals() of
       true -> ExtraMeta ++ Meta;
       false -> Meta
     end,
@@ -880,7 +895,7 @@ build_quoted_atom({_, _Location, [H]} = Token, Safe, ExtraMeta) when is_binary(H
 build_quoted_atom({_, Location, Args}, Safe, ExtraMeta) ->
   Meta = meta_from_location(Location),
   MetaWithExtra =
-    case ?formatter_metadata() of
+    case ?wrap_literals() of
       true -> ExtraMeta ++ Meta;
       false -> Meta
     end,
@@ -896,7 +911,7 @@ charlist_part(Binary) when is_binary(Binary) ->
 charlist_part({Begin, End, Tokens}) ->
   Form = string_tokens_parse(Tokens),
   Meta =
-    case ?formatter_metadata() of
+    case ?wrap_literals() of
       true -> [{closing, meta_from_location(End)} | meta_from_location(Begin)];
       false -> meta_from_location(Begin)
     end,
@@ -909,7 +924,7 @@ string_part(Binary) when is_binary(Binary) ->
 string_part({Begin, End, Tokens}) ->
   Form = string_tokens_parse(Tokens),
   Meta =
-    case ?formatter_metadata() of
+    case ?wrap_literals() of
       true -> [{closing, meta_from_location(End)} | meta_from_location(Begin)];
       false -> meta_from_location(Begin)
     end,
diff --git a/lib/elixir/test/elixir/code_formatter/integration_test.exs b/lib/elixir/test/elixir/code_formatter/integration_test.exs
index d428e941f..d168fb28f 100644
--- a/lib/elixir/test/elixir/code_formatter/integration_test.exs
+++ b/lib/elixir/test/elixir/code_formatter/integration_test.exs
@@ -473,6 +473,19 @@ defmodule Code.Formatter.IntegrationTest do
     """
   end
 
+  test "newline after stab" do
+    assert_same """
+    capture_io(":erl. mof*,,l", fn ->
+      assert :io.scan_erl_form('>') == {:ok, [{:":", 1}, {:atom, 1, :erl}, {:dot, 1}], 1}
+
+      expected_tokens = [{:atom, 1, :mof}, {:*, 1}, {:",", 1}, {:",", 1}, {:atom, 1, :l}]
+      assert :io.scan_erl_form('>') == {:ok, expected_tokens, 1}
+
+      assert :io.scan_erl_form('>') == {:eof, 1}
+    end)
+    """
+  end
+
   test "capture with operators" do
     assert_same """
     "this works" |> (&String.upcase/1) |> (&String.downcase/1)
diff --git a/lib/elixir/test/elixir/code_test.exs b/lib/elixir/test/elixir/code_test.exs
index 7ab91fd20..840ddf12c 100644
--- a/lib/elixir/test/elixir/code_test.exs
+++ b/lib/elixir/test/elixir/code_test.exs
@@ -215,102 +215,127 @@ defmodule CodeTest do
     end
   end
 
-  describe "string_to_quoted/2 with :pairing_metadata" do
-    test "adds delimiter information to sigils" do
-      string_to_quoted = &Code.string_to_quoted!(&1, pairing_metadata: true)
+  describe "string_to_quoted/2 with :token_metadata" do
+    test "adds end_of_expression information to blocks" do
+      file = """
+      one();two()
+      three()
 
-      assert string_to_quoted.("~r/foo/") ==
-               {:sigil_r, [delimiter: "/", line: 1], [{:<<>>, [line: 1], ["foo"]}, []]}
+      four()
 
-      assert string_to_quoted.("~r[foo]") ==
-               {:sigil_r, [delimiter: "[", line: 1], [{:<<>>, [line: 1], ["foo"]}, []]}
 
-      assert string_to_quoted.("~r\"foo\"") ==
-               {:sigil_r, [delimiter: "\"", line: 1], [{:<<>>, [line: 1], ["foo"]}, []]}
+      five()
+      """
 
-      meta = [delimiter: "\"\"\"", line: 1]
-      args = {:sigil_S, meta, [{:<<>>, [line: 1], ["sigil heredoc\n"]}, []]}
-      assert string_to_quoted.("~S\"\"\"\nsigil heredoc\n\"\"\"") == args
+      args = [
+        {:one,
+         [
+           end_of_expression: [newlines: 0, line: 1, column: 6],
+           closing: [line: 1, column: 5],
+           line: 1,
+           column: 1
+         ], []},
+        {:two,
+         [
+           end_of_expression: [newlines: 1, line: 1, column: 12],
+           closing: [line: 1, column: 11],
+           line: 1,
+           column: 7
+         ], []},
+        {:three,
+         [
+           end_of_expression: [newlines: 2, line: 2, column: 8],
+           closing: [line: 2, column: 7],
+           line: 2,
+           column: 1
+         ], []},
+        {:four,
+         [
+           end_of_expression: [newlines: 3, line: 4, column: 7],
+           closing: [line: 4, column: 6],
+           line: 4,
+           column: 1
+         ], []},
+        {:five, [closing: [line: 7, column: 6], line: 7, column: 1], []}
+      ]
 
-      meta = [delimiter: "'''", line: 1]
-      args = {:sigil_S, meta, [{:<<>>, [line: 1], ["sigil heredoc\n"]}, []]}
-      assert string_to_quoted.("~S'''\nsigil heredoc\n'''") == args
+      assert Code.string_to_quoted!(file, token_metadata: true, columns: true) ==
+               {:__block__, [], args}
     end
 
     test "adds pairing information" do
-      string_to_quoted = &Code.string_to_quoted!(&1, pairing_metadata: true)
+      string_to_quoted = &Code.string_to_quoted!(&1, token_metadata: true)
 
       assert string_to_quoted.("foo") == {:foo, [line: 1], nil}
-      assert string_to_quoted.("foo()") == {:foo, [eol: false, closing: [line: 1], line: 1], []}
-      assert string_to_quoted.("foo(\n)") == {:foo, [eol: true, closing: [line: 2], line: 1], []}
-      assert string_to_quoted.("%{\n}") == {:%{}, [eol: true, closing: [line: 2], line: 1], []}
+      assert string_to_quoted.("foo()") == {:foo, [closing: [line: 1], line: 1], []}
+
+      assert string_to_quoted.("foo(\n)") ==
+               {:foo, [newlines: 1, closing: [line: 2], line: 1], []}
+
+      assert string_to_quoted.("%{\n}") == {:%{}, [newlines: 1, closing: [line: 2], line: 1], []}
 
       assert string_to_quoted.("foo(\n) do\nend") ==
-               {:foo, [do: [line: 2], end: [line: 3], eol: true, closing: [line: 2], line: 1],
+               {:foo, [do: [line: 2], end: [line: 3], newlines: 1, closing: [line: 2], line: 1],
                 [[do: {:__block__, [], []}]]}
     end
-  end
 
-  describe "string_to_quoted/2 with :elixir_private_formatter_metadata" do
-    test "wraps literals in blocks" do
-      string_to_quoted = &Code.string_to_quoted!(&1, elixir_private_formatter_metadata: true)
+    test "adds delimiter information to sigils" do
+      string_to_quoted = &Code.string_to_quoted!(&1, token_metadata: true)
 
-      assert string_to_quoted.(~s("one")) == {:__block__, [format: :string, line: 1], ["one"]}
-      assert string_to_quoted.("'one'") == {:__block__, [format: :charlist, line: 1], ['one']}
-      assert string_to_quoted.("?é") == {:__block__, [original: '?é', line: 1], [233]}
-      assert string_to_quoted.("0b10") == {:__block__, [original: '0b10', line: 1], [2]}
-      assert string_to_quoted.("12") == {:__block__, [original: '12', line: 1], [12]}
-      assert string_to_quoted.("0o123") == {:__block__, [original: '0o123', line: 1], [83]}
-      assert string_to_quoted.("0xEF") == {:__block__, [original: '0xEF', line: 1], [239]}
-      assert string_to_quoted.("12.3") == {:__block__, [original: '12.3', line: 1], [12.3]}
-      assert string_to_quoted.("nil") == {:__block__, [line: 1], [nil]}
-      assert string_to_quoted.(":one") == {:__block__, [line: 1], [:one]}
+      assert string_to_quoted.("~r/foo/") ==
+               {:sigil_r, [delimiter: "/", line: 1], [{:<<>>, [line: 1], ["foo"]}, []]}
 
-      args = [[{:__block__, [original: '1', line: 1], [1]}]]
+      assert string_to_quoted.("~r[foo]") ==
+               {:sigil_r, [delimiter: "[", line: 1], [{:<<>>, [line: 1], ["foo"]}, []]}
 
-      assert string_to_quoted.("[1]") ==
-               {:__block__, [line: 1], args}
+      assert string_to_quoted.("~r\"foo\"") ==
+               {:sigil_r, [delimiter: "\"", line: 1], [{:<<>>, [line: 1], ["foo"]}, []]}
 
-      args = [{{:__block__, [line: 1], [:ok]}, {:__block__, [line: 1], [:test]}}]
+      meta = [delimiter: "\"\"\"", line: 1]
+      args = {:sigil_S, meta, [{:<<>>, [line: 1], ["sigil heredoc\n"]}, []]}
+      assert string_to_quoted.("~S\"\"\"\nsigil heredoc\n\"\"\"") == args
 
-      assert string_to_quoted.("{:ok, :test}") ==
-               {:__block__, [line: 1], args}
+      meta = [delimiter: "'''", line: 1]
+      args = {:sigil_S, meta, [{:<<>>, [line: 1], ["sigil heredoc\n"]}, []]}
+      assert string_to_quoted.("~S'''\nsigil heredoc\n'''") == args
+    end
+  end
 
-      assert string_to_quoted.(~s("""\nhello\n""")) ==
-               {:__block__, [format: :bin_heredoc, line: 1], ["hello\n"]}
+  test "string_to_quoted/2 with :elixir_private_wrap_literals wraps literals in blocks" do
+    string_to_quoted = &Code.string_to_quoted!(&1, elixir_private_wrap_literals: true)
 
-      assert string_to_quoted.("'''\nhello\n'''") ==
-               {:__block__, [format: :list_heredoc, line: 1], ['hello\n']}
+    assert string_to_quoted.(~s("one")) == {:__block__, [format: :string, line: 1], ["one"]}
+    assert string_to_quoted.("'one'") == {:__block__, [format: :charlist, line: 1], ['one']}
+    assert string_to_quoted.("?é") == {:__block__, [original: '?é', line: 1], [233]}
+    assert string_to_quoted.("0b10") == {:__block__, [original: '0b10', line: 1], [2]}
+    assert string_to_quoted.("12") == {:__block__, [original: '12', line: 1], [12]}
+    assert string_to_quoted.("0o123") == {:__block__, [original: '0o123', line: 1], [83]}
+    assert string_to_quoted.("0xEF") == {:__block__, [original: '0xEF', line: 1], [239]}
+    assert string_to_quoted.("12.3") == {:__block__, [original: '12.3', line: 1], [12.3]}
+    assert string_to_quoted.("nil") == {:__block__, [line: 1], [nil]}
+    assert string_to_quoted.(":one") == {:__block__, [line: 1], [:one]}
 
-      left = {:__block__, [original: '1', line: 1, line: 1], [1]}
-      right = {:__block__, [format: :string, line: 1], ["hello"]}
-      args = [{:->, [line: 1], [[left], right]}]
+    args = [[{:__block__, [original: '1', line: 1], [1]}]]
 
-      assert string_to_quoted.(~s[fn (1) -> "hello" end]) == {:fn, [line: 1], args}
-    end
+    assert string_to_quoted.("[1]") ==
+             {:__block__, [line: 1], args}
 
-    test "adds newlines information to blocks" do
-      file = """
-      one();two()
-      three()
+    args = [{{:__block__, [line: 1], [:ok]}, {:__block__, [line: 1], [:test]}}]
 
-      four()
+    assert string_to_quoted.("{:ok, :test}") ==
+             {:__block__, [line: 1], args}
 
+    assert string_to_quoted.(~s("""\nhello\n""")) ==
+             {:__block__, [format: :bin_heredoc, line: 1], ["hello\n"]}
 
-      five()
-      """
+    assert string_to_quoted.("'''\nhello\n'''") ==
+             {:__block__, [format: :list_heredoc, line: 1], ['hello\n']}
 
-      args = [
-        {:one, [line: 1], []},
-        {:two, [newlines: 0, line: 1], []},
-        {:three, [newlines: 1, line: 2], []},
-        {:four, [newlines: 2, line: 4], []},
-        {:five, [newlines: 3, line: 7], []}
-      ]
+    left = {:__block__, [original: '1', line: 1, line: 1], [1]}
+    right = {:__block__, [format: :string, line: 1], ["hello"]}
+    args = [{:->, [line: 1], [[left], right]}]
 
-      assert Code.string_to_quoted!(file, elixir_private_formatter_metadata: true) ==
-               {:__block__, [], args}
-    end
+    assert string_to_quoted.(~s[fn (1) -> "hello" end]) == {:fn, [line: 1], args}
   end
 
   test "compile source" do