Exposing pairing metadata (#9146)

This introduces five new metadata nodes:

  * `do` - contains metadata about do location in a function call with
    `do/end` blocks
  * `end` - contains metadata about end location in a function call with
    `do/end` blocks
  * `closing` - contains metadata about the closing pair, such as a `}`
    in a tuple or in a map, or such as the closing `)` in a function call
    with parens
  * `eol` - is set to true when the opening pair, such as `{` or `(`, are
    followed by the end of the line
  * `delimiter` - contains the opening for sigils (such as `"{"`, `"/"`,
    etc)

10 files changed, 172 insertions, 124 deletions

diff --git a/lib/elixir/lib/code.ex b/lib/elixir/lib/code.ex
index e9e91c0f3..00125ee50 100644
--- a/lib/elixir/lib/code.ex
+++ b/lib/elixir/lib/code.ex
@@ -665,11 +665,15 @@ defmodule Code do
       when non-existing atoms are found by the tokenizer.
       Defaults to `false`.
 
-    * `:static_atom_encoder` - The static atom encoder function, see
-      "The `:static_atom_encoder` function" section below. This option
-      overrides the `:existing_atoms_only` behaviour for static atoms
-      but `:existing_atoms_only` is still used for dynamic atoms, such
-      as atoms with interpolations.
+    * `:pairing_metadata` - when `true`, includes metadata about `do/end` blocks,
+      end of line, and closing pairs. See `t:Macro.metadata/0`. Defaults
+      to `false`.
+
+    * `:static_atom_encoder` - the static atom encoder function, see
+      "The `:static_atom_encoder` function" section below. Note this
+      option overrides the `:existing_atoms_only` behaviour for static
+      atoms but `:existing_atoms_only` is still used for dynamic atoms,
+      such as atoms with interpolations.
 
     * `:warn_on_unnecessary_quotes` - when `false`, does not warn
       when atoms, keywords or calls have unnecessary quotes on
diff --git a/lib/elixir/lib/code/formatter.ex b/lib/elixir/lib/code/formatter.ex
index 09a7d3462..9d6ab45f4 100644
--- a/lib/elixir/lib/code/formatter.ex
+++ b/lib/elixir/lib/code/formatter.ex
@@ -205,8 +205,13 @@ defmodule Code.Formatter do
       warn_on_unnecessary_quotes: false
     ]
 
+    parser_options = [
+      elixir_private_formatter_metadata: true,
+      pairing_metadata: true
+    ]
+
     with {:ok, tokens} <- :elixir.string_to_tokens(charlist, line, file, tokenizer_options),
-         {:ok, forms} <- :elixir.tokens_to_quoted(tokens, file, formatter_metadata: true) do
+         {:ok, forms} <- :elixir.tokens_to_quoted(tokens, file, parser_options) do
       state =
         Process.get(:code_formatter_comments)
         |> Enum.reverse()
@@ -479,7 +484,7 @@ defmodule Code.Formatter do
   end
 
   defp quoted_to_algebra({:__block__, meta, _} = block, _context, state) do
-    {block, state} = block_to_algebra(block, line(meta), end_line(meta), state)
+    {block, state} = block_to_algebra(block, line(meta), closing_line(meta), state)
     {surround("(", block, ")"), state}
   end
 
@@ -513,7 +518,7 @@ defmodule Code.Formatter do
   defp quoted_to_algebra({:not, meta, [{:in, _, [left, right]} = arg]}, context, state) do
     %{rename_deprecated_at: since} = state
 
-    # TODO: Remove since check on Elixir v2.0 and the OP arrangement is removed.
+    # TODO: Remove metadata and always rewrite to left not in right in Elixir v2.0.
     if meta[:operator] == :"not in" || (since && Version.match?(since, "~> 1.5")) do
       binary_op_to_algebra(:in, "not in", meta, left, right, context, state)
     else
@@ -522,7 +527,7 @@ defmodule Code.Formatter do
   end
 
   defp quoted_to_algebra({:fn, meta, [_ | _] = clauses}, _context, state) do
-    anon_fun_to_algebra(clauses, line(meta), end_line(meta), state, eol?(meta))
+    anon_fun_to_algebra(clauses, line(meta), closing_line(meta), state, eol?(meta))
   end
 
   defp quoted_to_algebra({fun, meta, args}, context, state) when is_atom(fun) and is_list(args) do
@@ -900,11 +905,11 @@ defmodule Code.Formatter do
 
   # @foo bar
   # @foo(bar)
-  defp module_attribute_to_algebra(meta, {name, _, [_] = args} = expr, context, state)
+  defp module_attribute_to_algebra(meta, {name, call_meta, [_] = args} = expr, context, state)
        when is_atom(name) and name not in [:__block__, :__aliases__] do
     if Code.Identifier.classify(name) == :callable_local do
       {{call_doc, state}, wrap_in_parens?} =
-        call_args_to_algebra(args, meta, context, :skip_unless_many_args, false, state)
+        call_args_to_algebra(args, call_meta, context, :skip_unless_many_args, false, state)
 
       doc =
         "@#{name}"
@@ -995,8 +1000,7 @@ defmodule Code.Formatter do
     fun = remote_fun_to_algebra(target, fun, length(args), state)
     remote_doc = target_doc |> concat(".") |> concat(string(fun))
 
-    if args == [] and not remote_target_is_a_module?(target) and
-         Keyword.get(meta, :no_parens, false) do
+    if args == [] and not remote_target_is_a_module?(target) and not meta?(meta, :closing) do
       {remote_doc, state}
     else
       {{call_doc, state}, wrap_in_parens?} =
@@ -1075,7 +1079,7 @@ defmodule Code.Formatter do
   defp local_to_algebra(fun, meta, args, context, state) when is_atom(fun) do
     skip_parens =
       cond do
-        not Keyword.get(meta, :no_parens, false) -> :required
+        meta?(meta, :closing) -> :required
         local_without_parens?(fun, args, state) -> :skip_unless_many_args
         true -> :skip_if_do_end
       end
@@ -1109,7 +1113,7 @@ defmodule Code.Formatter do
   defp call_args_to_algebra(args, meta, context, parens, list_to_keyword?, state) do
     {rest, last} = split_last(args)
 
-    if blocks = do_end_blocks(last, state) do
+    if blocks = do_end_blocks(meta, last, state) do
       {call_doc, state} =
         if rest == [] do
           {" do", state}
@@ -1263,15 +1267,15 @@ defmodule Code.Formatter do
     not Enum.any?(args, &match?({:<-, _, [_, _]}, &1))
   end
 
-  defp do_end_blocks([{{:__block__, meta, [:do]}, _} | rest] = blocks, state) do
-    if meta[:format] == :block or can_force_do_end_blocks?(rest, state) do
+  defp do_end_blocks(meta, [{{:__block__, _, [:do]}, _} | rest] = blocks, state) do
+    if meta?(meta, :do) or can_force_do_end_blocks?(rest, state) do
       blocks
       |> Enum.map(fn {{:__block__, meta, [key]}, value} -> {key, line(meta), value} end)
       |> do_end_blocks_with_range(end_line(meta))
     end
   end
 
-  defp do_end_blocks(_, _), do: nil
+  defp do_end_blocks(_, _, _), do: nil
 
   defp can_force_do_end_blocks?(rest, state) do
     state.force_do_end_blocks and
@@ -1336,7 +1340,7 @@ defmodule Code.Formatter do
 
   defp list_interpolation_to_algebra([entry | entries], escape, state, acc, last) do
     {{:., _, [Kernel, :to_string]}, meta, [quoted]} = entry
-    {doc, state} = block_to_algebra(quoted, line(meta), end_line(meta), state)
+    {doc, state} = block_to_algebra(quoted, line(meta), closing_line(meta), state)
     doc = surround("\#{", doc, "}")
     list_interpolation_to_algebra(entries, escape, state, concat(acc, doc), last)
   end
@@ -1353,7 +1357,7 @@ defmodule Code.Formatter do
 
   defp interpolation_to_algebra([entry | entries], escape, state, acc, last) do
     {:"::", _, [{{:., _, [Kernel, :to_string]}, meta, [quoted]}, {:binary, _, _}]} = entry
-    {doc, state} = block_to_algebra(quoted, line(meta), end_line(meta), state)
+    {doc, state} = block_to_algebra(quoted, line(meta), closing_line(meta), state)
     doc = surround("\#{", doc, "}")
     interpolation_to_algebra(entries, escape, state, concat(acc, doc), last)
   end
@@ -1367,22 +1371,22 @@ defmodule Code.Formatter do
   defp maybe_sigil_to_algebra(fun, meta, args, state) do
     with <<"sigil_", name>> <- Atom.to_string(fun),
          [{:<<>>, _, entries}, modifiers] when is_list(modifiers) <- args,
-         opening_terminator when not is_nil(opening_terminator) <- Keyword.get(meta, :terminator) do
-      doc = <<?~, name, opening_terminator::binary>>
+         opening_delimiter when not is_nil(opening_delimiter) <- meta[:delimiter] do
+      doc = <<?~, name, opening_delimiter::binary>>
 
-      if opening_terminator in [@double_heredoc, @single_heredoc] do
-        closing_terminator = concat(opening_terminator, List.to_string(modifiers))
+      if opening_delimiter in [@double_heredoc, @single_heredoc] do
+        closing_delimiter = concat(opening_delimiter, List.to_string(modifiers))
 
         {doc, state} =
           entries
           |> prepend_heredoc_line()
-          |> interpolation_to_algebra(:heredoc, state, doc, closing_terminator)
+          |> interpolation_to_algebra(:heredoc, state, doc, closing_delimiter)
 
         {force_unfit(doc), state}
       else
-        escape = closing_sigil_terminator(opening_terminator)
-        closing_terminator = concat(escape, List.to_string(modifiers))
-        interpolation_to_algebra(entries, escape, state, doc, closing_terminator)
+        escape = closing_sigil_delimiter(opening_delimiter)
+        closing_delimiter = concat(escape, List.to_string(modifiers))
+        interpolation_to_algebra(entries, escape, state, doc, closing_delimiter)
       end
     else
       _ ->
@@ -1390,11 +1394,11 @@ defmodule Code.Formatter do
     end
   end
 
-  defp closing_sigil_terminator("("), do: ")"
-  defp closing_sigil_terminator("["), do: "]"
-  defp closing_sigil_terminator("{"), do: "}"
-  defp closing_sigil_terminator("<"), do: ">"
-  defp closing_sigil_terminator(other) when other in ["\"", "'", "|", "/"], do: other
+  defp closing_sigil_delimiter("("), do: ")"
+  defp closing_sigil_delimiter("["), do: "]"
+  defp closing_sigil_delimiter("{"), do: "}"
+  defp closing_sigil_delimiter("<"), do: ">"
+  defp closing_sigil_delimiter(other) when other in ["\"", "'", "|", "/"], do: other
 
   ## Bitstrings
 
@@ -1622,7 +1626,7 @@ defmodule Code.Formatter do
 
   defp args_to_algebra_with_comments(args, meta, skip_parens?, last_arg_mode, join, state, fun) do
     min_line = line(meta)
-    max_line = end_line(meta)
+    max_line = closing_line(meta)
 
     arg_to_algebra = fn arg, args, newlines, state ->
       {doc, state} = fun.(arg, state)
@@ -1823,7 +1827,7 @@ defmodule Code.Formatter do
   end
 
   defp clause_to_algebra({:->, meta, [[], body]}, _min_line, state) do
-    {body_doc, state} = block_to_algebra(body, line(meta), end_line(meta), state)
+    {body_doc, state} = block_to_algebra(body, line(meta), closing_line(meta), state)
     {"() ->" |> glue(body_doc) |> nest(2), state}
   end
 
@@ -1834,7 +1838,7 @@ defmodule Code.Formatter do
     {args_doc, state} = clause_args_to_algebra(args, min_line, state)
 
     state = %{state | operand_nesting: nesting}
-    {body_doc, state} = block_to_algebra(body, min_line, end_line(meta), state)
+    {body_doc, state} = block_to_algebra(body, min_line, closing_line(meta), state)
 
     doc =
       args_doc
@@ -2146,7 +2150,7 @@ defmodule Code.Formatter do
   end
 
   defp next_break_fits?({fun, meta, args}, _state) when is_atom(fun) and is_list(args) do
-    meta[:terminator] in [@double_heredoc, @single_heredoc] and
+    meta[:delimiter] in [@double_heredoc, @single_heredoc] and
       fun |> Atom.to_string() |> String.starts_with?("sigil_")
   end
 
@@ -2162,7 +2166,7 @@ defmodule Code.Formatter do
     eol?(meta) or
       (
         min_line = line(meta)
-        max_line = end_line(meta)
+        max_line = closing_line(meta)
         Enum.any?(comments, fn {line, _, _} -> line > min_line and line < max_line end)
       )
   end
@@ -2234,11 +2238,19 @@ defmodule Code.Formatter do
     Keyword.get(meta, :eol, false)
   end
 
+  defp meta?(meta, key) do
+    is_list(meta[key])
+  end
+
   defp line(meta) do
     meta[:line] || @max_line
   end
 
   defp end_line(meta) do
+    meta[:end][:line] || @min_line
+  end
+
+  defp closing_line(meta) do
     meta[:closing][:line] || @min_line
   end
 
diff --git a/lib/elixir/lib/macro.ex b/lib/elixir/lib/macro.ex
index 934d02fee..e4f2aa5a3 100644
--- a/lib/elixir/lib/macro.ex
+++ b/lib/elixir/lib/macro.ex
@@ -142,8 +142,6 @@ defmodule Macro do
 
   The following metadata keys are public:
 
-    * `:column` - The column number of the AST node. It is included only
-      in the AST generated by `Code.string_to_quoted/2` with the option `columns: true`.
     * `:context` - Defines the context in which the AST was generated.
       For example, `quote/2` will include the module calling `quote/2`
       as the context. This is often used to distinguish regular code from code
@@ -158,6 +156,21 @@ defmodule Macro do
       the file and the line number of the quoted source.
     * `:line` - The line number of the AST node.
 
+  The following metadata keys are enabled by `Code.string_to_quoted/2`:
+
+    * `:column` - the column number of the AST node (when `:columns` is true)
+    * `:do` - contains metadata about the `do` location in a function call with
+      `do/end` blocks (when `:pairing_metadata` is true)
+    * `:end` - contains metadata about the `end` location in a function call with
+      `do/end` blocks (when `:pairing_metadata` is true)
+    * `:closing` - contains metadata about the closing pair, such as a `}`
+      in a tuple or in a map, or such as the closing `)` in a function call
+      with parens (when `:pairing_metadata` is true)
+    * `:eol` - is set to true when the opening pair, such as `{` or `(`, are
+      followed by the end of the line (when `:pairing_metadata` is true)
+    * `:delimiter` - contains the opening delimiter for sigils as a string
+      (such as `"{"`, `"/"`, etc)
+
   The following metadata keys are private:
 
     * `:alias` - Used for alias hygiene.
diff --git a/lib/elixir/src/elixir.erl b/lib/elixir/src/elixir.erl
index c5d67690a..5b205f3bc 100644
--- a/lib/elixir/src/elixir.erl
+++ b/lib/elixir/src/elixir.erl
@@ -332,6 +332,7 @@ tokens_to_quoted(Tokens, File, Opts) ->
   after
     erase(elixir_parser_file),
     erase(elixir_parser_columns),
+    erase(elixir_pairing_metadata),
     erase(elixir_formatter_metadata)
   end.
 
@@ -360,8 +361,10 @@ to_binary(List) when is_list(List) -> elixir_utils:characters_to_binary(List);
 to_binary(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8).
 
 handle_parsing_opts(File, Opts) ->
-  FormatterMetadata = lists:keyfind(formatter_metadata, 1, Opts) == {formatter_metadata, true},
+  FormatterMetadata = lists:keyfind(elixir_private_formatter_metadata, 1, Opts) == {elixir_private_formatter_metadata, true},
+  PairingMetadata = lists:keyfind(pairing_metadata, 1, Opts) == {pairing_metadata, true},
   Columns = lists:keyfind(columns, 1, Opts) == {columns, true},
   put(elixir_parser_file, File),
   put(elixir_parser_columns, Columns),
+  put(elixir_pairing_metadata, PairingMetadata),
   put(elixir_formatter_metadata, FormatterMetadata).
diff --git a/lib/elixir/src/elixir_parser.yrl b/lib/elixir/src/elixir_parser.yrl
index 4d8e89bf9..a7277e6cb 100644
--- a/lib/elixir/src/elixir_parser.yrl
+++ b/lib/elixir/src/elixir_parser.yrl
@@ -161,9 +161,9 @@ no_parens_expr -> no_parens_many_expr : '$1'.
 
 block_expr -> dot_call_identifier call_args_parens do_block : build_parens('$1', '$2', '$3').
 block_expr -> dot_call_identifier call_args_parens call_args_parens do_block : build_nested_parens('$1', '$2', '$3', '$4').
-block_expr -> dot_do_identifier do_block : build_no_parens('$1', '$2').
-block_expr -> dot_op_identifier call_args_no_parens_all do_block : build_no_parens('$1', '$2' ++ '$3').
-block_expr -> dot_identifier call_args_no_parens_all do_block : build_no_parens('$1', '$2' ++ '$3').
+block_expr -> dot_do_identifier do_block : build_no_parens_do_block('$1', [], '$2').
+block_expr -> dot_op_identifier call_args_no_parens_all do_block : build_no_parens_do_block('$1', '$2', '$3').
+block_expr -> dot_identifier call_args_no_parens_all do_block : build_no_parens_do_block('$1', '$2', '$3').
 
 matched_op_expr -> match_op_eol matched_expr : {'$1', '$2'}.
 matched_op_expr -> dual_op_eol matched_expr : {'$1', '$2'}.
@@ -265,14 +265,13 @@ access_expr -> atom_unsafe : build_quoted_atom('$1', false, []).
 access_expr -> dot_alias : '$1'.
 access_expr -> parens_call : '$1'.
 
-%% Augment integer literals with representation format if formatter_metadata option is true
 number -> int : handle_literal(number_value('$1'), '$1', [{original, ?exprs('$1')}]).
 number -> char : handle_literal(?exprs('$1'), '$1', [{original, number_value('$1')}]).
 number -> float : handle_literal(number_value('$1'), '$1', [{original, ?exprs('$1')}]).
 
 %% Also used by maps and structs
-parens_call -> dot_call_identifier call_args_parens : build_parens('$1', '$2', []).
-parens_call -> dot_call_identifier call_args_parens call_args_parens : build_nested_parens('$1', '$2', '$3', []).
+parens_call -> dot_call_identifier call_args_parens : build_parens('$1', '$2', {[], []}).
+parens_call -> dot_call_identifier call_args_parens call_args_parens : build_nested_parens('$1', '$2', '$3', {[], []}).
 
 bracket_arg -> open_bracket kw close_bracket : build_list('$1', '$2', '$3').
 bracket_arg -> open_bracket container_expr close_bracket : build_list('$1', '$2', '$3').
@@ -289,17 +288,13 @@ bracket_at_expr -> at_op_eol access_expr bracket_arg :
 %% Blocks
 
 do_block -> do_eoe 'end' :
-              [[{handle_literal(do, '$1', end_meta('$2')),
-                 {'__block__', [], []}}]].
+              {do_end_meta('$1', '$2'), [[{handle_literal(do, '$1'), {'__block__', [], []}}]]}.
 do_block -> do_eoe stab end_eoe :
-              [[{handle_literal(do, '$1', end_meta('$3')),
-                 build_stab('$2')}]].
+              {do_end_meta('$1', '$3'), [[{handle_literal(do, '$1'), build_stab('$2')}]]}.
 do_block -> do_eoe block_list 'end' :
-              [[{handle_literal(do, '$1', end_meta('$3')),
-                 {'__block__', [], []}} | '$2']].
+              {do_end_meta('$1', '$3'), [[{handle_literal(do, '$1'), {'__block__', [], []}} | '$2']]}.
 do_block -> do_eoe stab_eoe block_list 'end' :
-              [[{handle_literal(do, '$1', end_meta('$4')),
-                 build_stab('$2')} | '$3']].
+              {do_end_meta('$1', '$4'), [[{handle_literal(do, '$1'), build_stab('$2')} | '$3']]}.
 
 eoe -> eol : '$1'.
 eoe -> ';' : '$1'.
@@ -344,11 +339,9 @@ stab_op_eol_and_expr -> stab_op_eol expr : {'$1', '$2'}.
 stab_op_eol_and_expr -> stab_op_eol : warn_empty_stab_clause('$1'), {'$1', handle_literal(nil, '$1')}.
 
 block_item -> block_eoe stab_eoe :
-                {handle_literal(?exprs('$1'), '$1', [{format, block}]),
-                 build_stab('$2')}.
+                {handle_literal(?exprs('$1'), '$1'), build_stab('$2')}.
 block_item -> block_eoe :
-                {handle_literal(?exprs('$1'), '$1', [{format, block}]),
-                 {'__block__', [], []}}.
+                {handle_literal(?exprs('$1'), '$1'), {'__block__', [], []}}.
 
 block_list -> block_item : ['$1'].
 block_list -> block_item block_list : ['$1' | '$2'].
@@ -626,6 +619,7 @@ Erlang code.
 
 -define(file(), get(elixir_parser_file)).
 -define(columns(), get(elixir_parser_columns)).
+-define(pairing_metadata(), get(elixir_pairing_metadata)).
 -define(formatter_metadata(), get(elixir_formatter_metadata)).
 
 -define(id(Token), element(1, Token)).
@@ -652,11 +646,16 @@ meta_from_location({Line, Column, _}) ->
 line_from_location({Line, _Column, _}) -> Line.
 is_eol({_, _, Eol}) -> is_integer(Eol) and (Eol > 0).
 
-end_meta(Token) ->
-  [{format, block}, {closing, meta_from_location(?location(Token))}].
+do_end_meta(Do, End) ->
+  case ?pairing_metadata() of
+    true ->
+      [{do, meta_from_location(?location(Do))}, {'end', meta_from_location(?location(End))}];
+    false ->
+      []
+  end.
 
 meta_from_token_with_closing(Begin, End) ->
-  case ?formatter_metadata() of
+  case ?pairing_metadata() of
     true ->
       [{closing, meta_from_location(?location(End))} | meta_from_token(Begin)];
     false ->
@@ -736,10 +735,10 @@ build_block(Exprs) ->
 %% End of line and newlines
 
 eol_pair(Left, Right) ->
-  case ?formatter_metadata() of
+  case ?pairing_metadata() of
     true ->
       [
-        {eol, is_eol(?location(Left)) and is_eol(?location(Right))},
+        {eol, is_eol(?location(Left))},
         {closing, meta_from_location(?location(Right))}
       ];
     false ->
@@ -790,25 +789,21 @@ extract_identifier({Kind, _, Identifier}) when
 
 %% Identifiers
 
-build_nested_parens(Dot, Args1, {Extra, Args2}, Block) ->
-  Identifier = build_parens(Dot, Args1, []),
-  Meta = ?meta(Identifier),
-  {Identifier, Extra ++ Meta, append_non_empty(Args2, Block)}.
+build_nested_parens(Dot, Args1, {Args2Meta, Args2}, {BlockMeta, Block}) ->
+  Identifier = build_parens(Dot, Args1, {[], []}),
+  Meta = BlockMeta ++ Args2Meta ++ ?meta(Identifier),
+  {Identifier, Meta, append_non_empty(Args2, Block)}.
 
-build_parens(Expr, {[], Args}, Block) ->
-  build_identifier(Expr, append_non_empty(Args, Block));
-build_parens(Expr, {Extra, Args}, Block) ->
+build_parens(Expr, {ArgsMeta, Args}, {BlockMeta, Block}) ->
   {BuiltExpr, BuiltMeta, BuiltArgs} = build_identifier(Expr, append_non_empty(Args, Block)),
-  {BuiltExpr, Extra ++ BuiltMeta, BuiltArgs}.
+  {BuiltExpr, BlockMeta ++ ArgsMeta ++ BuiltMeta, BuiltArgs}.
+
+build_no_parens_do_block(Expr, Args, {BlockMeta, Block}) ->
+  {BuiltExpr, BuiltMeta, BuiltArgs} = build_no_parens(Expr, Args ++ Block),
+  {BuiltExpr, BlockMeta ++ BuiltMeta, BuiltArgs}.
 
 build_no_parens(Expr, Args) ->
-  case ?formatter_metadata() of
-    true ->
-      {BuiltExpr, BuiltMeta, BuiltArgs} = build_identifier(Expr, Args),
-      {BuiltExpr, [{no_parens, true} | BuiltMeta], BuiltArgs};
-    false ->
-      build_identifier(Expr, Args)
-  end.
+  build_identifier(Expr, Args).
 
 build_identifier({'.', Meta, _} = Dot, Args) ->
   FArgs = case Args of
@@ -842,14 +837,14 @@ build_access(Expr, {List, Location}) ->
 
 %% Interpolation aware
 
-build_sigil({sigil, Location, Sigil, Parts, Modifiers, Terminator}) ->
+build_sigil({sigil, Location, Sigil, Parts, Modifiers, Delimiter}) ->
   Meta = meta_from_location(Location),
-  MetaWithTerminator = case ?formatter_metadata() of
-    true -> [{terminator, Terminator} | Meta];
+  MetaWithDelimiter = case ?pairing_metadata() of
+    true -> [{delimiter, Delimiter} | Meta];
     false -> Meta
   end,
   {list_to_atom("sigil_" ++ [Sigil]),
-   MetaWithTerminator,
+   MetaWithDelimiter,
    [{'<<>>', Meta, string_parts(Parts)}, Modifiers]}.
 
 build_bin_heredoc({bin_heredoc, Location, Args}) ->
diff --git a/lib/elixir/test/elixir/code_formatter/calls_test.exs b/lib/elixir/test/elixir/code_formatter/calls_test.exs
index 76bd18b15..87f3a7d8c 100644
--- a/lib/elixir/test/elixir/code_formatter/calls_test.exs
+++ b/lib/elixir/test/elixir/code_formatter/calls_test.exs
@@ -523,8 +523,8 @@ defmodule Code.Formatter.CallsTest do
       end
       """
 
-      # Doesn't preserve this because only the beginning has a newline
-      assert_format "call(\nfoo, bar, baz)", "call(foo, bar, baz)"
+      # Doesn't preserve this because only the ending has a newline
+      assert_format "call(foo, bar, baz\n)", "call(foo, bar, baz)"
 
       # Doesn't preserve because there are no args
       bad = """
@@ -548,7 +548,6 @@ defmodule Code.Formatter.CallsTest do
       )
       """
 
-      # Doesn't preserve this because only the beginning has a newline
       assert_format bad, """
       call(%{
         key: :value
@@ -717,8 +716,8 @@ defmodule Code.Formatter.CallsTest do
       )
       """
 
-      # Doesn't preserve this because only the beginning has a newline
-      assert_format "Remote.call(\nfoo, bar, baz)", "Remote.call(foo, bar, baz)"
+      # Doesn't preserve this because only the ending has a newline
+      assert_format "Remote.call(foo, bar, baz\n)", "Remote.call(foo, bar, baz)"
 
       assert_same """
       Remote.call(
@@ -826,8 +825,8 @@ defmodule Code.Formatter.CallsTest do
       )
       """
 
-      # Doesn't preserve this because only the beginning has a newline
-      assert_format "call.(\nfoo, bar, baz)", "call.(foo, bar, baz)"
+      # Doesn't preserve this because only the ending has a newline
+      assert_format "call.(foo, bar, baz\n)", "call.(foo, bar, baz)"
     end
   end
 
@@ -1165,8 +1164,7 @@ defmodule Code.Formatter.CallsTest do
       }
       """
 
-      # Doesn't preserve this because only the beginning has a newline
-      assert_format "call.{\nfoo, bar, baz}", "call.{foo, bar, baz}"
+      assert_format "call.{foo, bar, baz\n}", "call.{foo, bar, baz}"
     end
   end
 
diff --git a/lib/elixir/test/elixir/code_formatter/comments_test.exs b/lib/elixir/test/elixir/code_formatter/comments_test.exs
index 0adadd454..1e4677fad 100644
--- a/lib/elixir/test/elixir/code_formatter/comments_test.exs
+++ b/lib/elixir/test/elixir/code_formatter/comments_test.exs
@@ -885,7 +885,11 @@ defmodule Code.Formatter.CommentsTest do
 
       assert_format ambiguous, ~S"""
       # comment
-      [one, two, three]
+      [
+        one,
+        two,
+        three
+      ]
       """
     end
 
diff --git a/lib/elixir/test/elixir/code_formatter/containers_test.exs b/lib/elixir/test/elixir/code_formatter/containers_test.exs
index a7849bdc7..aefab7eb3 100644
--- a/lib/elixir/test/elixir/code_formatter/containers_test.exs
+++ b/lib/elixir/test/elixir/code_formatter/containers_test.exs
@@ -59,8 +59,8 @@ defmodule Code.Formatter.ContainersTest do
       }
       """
 
-      # Doesn't preserve this because only the beginning has a newline
-      assert_format "{\nfoo, bar, baz}", "{foo, bar, baz}"
+      # Doesn't preserve this because only the ending has a newline
+      assert_format "{foo, bar, baz\n}", "{foo, bar, baz}"
     end
 
     test "preserves user choice even when it fits with trailing comma" do
@@ -214,8 +214,8 @@ defmodule Code.Formatter.ContainersTest do
       ]
       """
 
-      # Doesn't preserve this because only the beginning has a newline
-      assert_format "[\nfoo, bar, baz]", "[foo, bar, baz]"
+      # Doesn't preserve this because only the ending has a newline
+      assert_format "[foo, bar, baz\n]", "[foo, bar, baz]"
     end
 
     test "preserves user choice even when it fits with trailing comma" do
@@ -304,8 +304,8 @@ defmodule Code.Formatter.ContainersTest do
       >>
       """
 
-      # Doesn't preserve this because only the beginning has a newline
-      assert_format "<<\nfoo, bar, baz>>", "<<foo, bar, baz>>"
+      # Doesn't preserve this because only the ending has a newline
+      assert_format "<<foo, bar, baz\n>>", "<<foo, bar, baz>>"
     end
 
     test "preserves user choice even when it fits with trailing comma" do
@@ -421,8 +421,8 @@ defmodule Code.Formatter.ContainersTest do
       }
       """
 
-      # Doesn't preserve this because only the beginning has a newline
-      assert_format "%{\nfoo: 1, bar: 2}", "%{foo: 1, bar: 2}"
+      # Doesn't preserve this because only the ending has a newline
+      assert_format "%{foo: 1, bar: 2\n}", "%{foo: 1, bar: 2}"
     end
 
     test "preserves user choice even when it fits with trailing comma" do
diff --git a/lib/elixir/test/elixir/code_formatter/integration_test.exs b/lib/elixir/test/elixir/code_formatter/integration_test.exs
index 2ce24c0c2..d428e941f 100644
--- a/lib/elixir/test/elixir/code_formatter/integration_test.exs
+++ b/lib/elixir/test/elixir/code_formatter/integration_test.exs
@@ -420,7 +420,11 @@ defmodule Code.Formatter.IntegrationTest do
                        [
                          {"p", [], ["1"]},
                          {"p", [], ["2"]},
-                         {"div", [], [{"p", [], ["3"]}, {"p", [], ["4"]}]},
+                         {"div", [],
+                          [
+                            {"p", [], ["3"]},
+                            {"p", [], ["4"]}
+                          ]},
                          {"p", [], ["5"]}
                        ]}
                     ]}
diff --git a/lib/elixir/test/elixir/code_test.exs b/lib/elixir/test/elixir/code_test.exs
index 21870d4ab..7ab91fd20 100644
--- a/lib/elixir/test/elixir/code_test.exs
+++ b/lib/elixir/test/elixir/code_test.exs
@@ -215,30 +215,45 @@ defmodule CodeTest do
     end
   end
 
-  describe "string_to_quoted/2 with :formatter_metadata (private)" do
-    test "adds terminator information to sigils" do
-      string_to_quoted = &Code.string_to_quoted!(&1, formatter_metadata: true)
+  describe "string_to_quoted/2 with :pairing_metadata" do
+    test "adds delimiter information to sigils" do
+      string_to_quoted = &Code.string_to_quoted!(&1, pairing_metadata: true)
 
       assert string_to_quoted.("~r/foo/") ==
-               {:sigil_r, [terminator: "/", line: 1], [{:<<>>, [line: 1], ["foo"]}, []]}
+               {:sigil_r, [delimiter: "/", line: 1], [{:<<>>, [line: 1], ["foo"]}, []]}
 
       assert string_to_quoted.("~r[foo]") ==
-               {:sigil_r, [terminator: "[", line: 1], [{:<<>>, [line: 1], ["foo"]}, []]}
+               {:sigil_r, [delimiter: "[", line: 1], [{:<<>>, [line: 1], ["foo"]}, []]}
 
       assert string_to_quoted.("~r\"foo\"") ==
-               {:sigil_r, [terminator: "\"", line: 1], [{:<<>>, [line: 1], ["foo"]}, []]}
+               {:sigil_r, [delimiter: "\"", line: 1], [{:<<>>, [line: 1], ["foo"]}, []]}
 
-      meta = [terminator: "\"\"\"", line: 1]
+      meta = [delimiter: "\"\"\"", line: 1]
       args = {:sigil_S, meta, [{:<<>>, [line: 1], ["sigil heredoc\n"]}, []]}
       assert string_to_quoted.("~S\"\"\"\nsigil heredoc\n\"\"\"") == args
 
-      meta = [terminator: "'''", line: 1]
+      meta = [delimiter: "'''", line: 1]
       args = {:sigil_S, meta, [{:<<>>, [line: 1], ["sigil heredoc\n"]}, []]}
       assert string_to_quoted.("~S'''\nsigil heredoc\n'''") == args
     end
 
-    test "wraps literals in blocks when :formatter_metadata (private) is given" do
-      string_to_quoted = &Code.string_to_quoted!(&1, formatter_metadata: true)
+    test "adds pairing information" do
+      string_to_quoted = &Code.string_to_quoted!(&1, pairing_metadata: true)
+
+      assert string_to_quoted.("foo") == {:foo, [line: 1], nil}
+      assert string_to_quoted.("foo()") == {:foo, [eol: false, closing: [line: 1], line: 1], []}
+      assert string_to_quoted.("foo(\n)") == {:foo, [eol: true, closing: [line: 2], line: 1], []}
+      assert string_to_quoted.("%{\n}") == {:%{}, [eol: true, closing: [line: 2], line: 1], []}
+
+      assert string_to_quoted.("foo(\n) do\nend") ==
+               {:foo, [do: [line: 2], end: [line: 3], eol: true, closing: [line: 2], line: 1],
+                [[do: {:__block__, [], []}]]}
+    end
+  end
+
+  describe "string_to_quoted/2 with :elixir_private_formatter_metadata" do
+    test "wraps literals in blocks" do
+      string_to_quoted = &Code.string_to_quoted!(&1, elixir_private_formatter_metadata: true)
 
       assert string_to_quoted.(~s("one")) == {:__block__, [format: :string, line: 1], ["one"]}
       assert string_to_quoted.("'one'") == {:__block__, [format: :charlist, line: 1], ['one']}
@@ -254,12 +269,12 @@ defmodule CodeTest do
       args = [[{:__block__, [original: '1', line: 1], [1]}]]
 
       assert string_to_quoted.("[1]") ==
-               {:__block__, [eol: false, closing: [line: 1], line: 1], args}
+               {:__block__, [line: 1], args}
 
       args = [{{:__block__, [line: 1], [:ok]}, {:__block__, [line: 1], [:test]}}]
 
       assert string_to_quoted.("{:ok, :test}") ==
-               {:__block__, [eol: false, closing: [line: 1], line: 1], args}
+               {:__block__, [line: 1], args}
 
       assert string_to_quoted.(~s("""\nhello\n""")) ==
                {:__block__, [format: :bin_heredoc, line: 1], ["hello\n"]}
@@ -267,15 +282,14 @@ defmodule CodeTest do
       assert string_to_quoted.("'''\nhello\n'''") ==
                {:__block__, [format: :list_heredoc, line: 1], ['hello\n']}
 
-      left = {:__block__, [original: '1', line: 1, closing: [line: 1], line: 1], [1]}
+      left = {:__block__, [original: '1', line: 1, line: 1], [1]}
       right = {:__block__, [format: :string, line: 1], ["hello"]}
       args = [{:->, [line: 1], [[left], right]}]
 
-      assert string_to_quoted.(~s[fn (1) -> "hello" end]) ==
-               {:fn, [closing: [line: 1], line: 1], args}
+      assert string_to_quoted.(~s[fn (1) -> "hello" end]) == {:fn, [line: 1], args}
     end
 
-    test "adds newlines information to blocks when :formatter_metadata (private) is given" do
+    test "adds newlines information to blocks" do
       file = """
       one();two()
       three()
@@ -287,14 +301,15 @@ defmodule CodeTest do
       """
 
       args = [
-        {:one, [eol: false, closing: [line: 1], line: 1], []},
-        {:two, [newlines: 0, eol: false, closing: [line: 1], line: 1], []},
-        {:three, [newlines: 1, eol: false, closing: [line: 2], line: 2], []},
-        {:four, [newlines: 2, eol: false, closing: [line: 4], line: 4], []},
-        {:five, [newlines: 3, eol: false, closing: [line: 7], line: 7], []}
+        {:one, [line: 1], []},
+        {:two, [newlines: 0, line: 1], []},
+        {:three, [newlines: 1, line: 2], []},
+        {:four, [newlines: 2, line: 4], []},
+        {:five, [newlines: 3, line: 7], []}
       ]
 
-      assert Code.string_to_quoted!(file, formatter_metadata: true) == {:__block__, [], args}
+      assert Code.string_to_quoted!(file, elixir_private_formatter_metadata: true) ==
+               {:__block__, [], args}
     end
   end