Add literal encoder option to Code.string_to_quoted

9 files changed, 176 insertions, 128 deletions

diff --git a/lib/elixir/lib/code.ex b/lib/elixir/lib/code.ex
index 1ad871f33..2ebc59239 100644
--- a/lib/elixir/lib/code.ex
+++ b/lib/elixir/lib/code.ex
@@ -644,7 +644,7 @@ defmodule Code do
     end
   end
 
-  @doc """
+  @doc ~S"""
   Converts the given string to its quoted form.
 
   Returns `{:ok, quoted_form}` if it succeeds,
@@ -670,8 +670,15 @@ defmodule Code do
       for closing tokens, end of expressions, as well as delimiters for
       sigils. See `t:Macro.metadata/0`. Defaults to `false`.
 
-    * `:static_atom_encoder` - the static atom encoder function, see
-      "The `:static_atom_encoder` function" section below. Note this
+    * `:literal_encoder` - how to encode literals in the AST. It must
+      be a function that receives two arguments, the literal and its
+      metadata, and it must return `{:ok, ast :: Macro.t}` or
+      `{:error, reason :: binary}`. If you return anything than the literal
+      itself as the `term`, then the AST is no longer valid. This option
+      may still useful for textual analysis of the source code.
+
+    * `:static_atoms_encoder` - the static atom encoder function, see
+      "The `:static_atoms_encoder` function" section below. Note this
       option overrides the `:existing_atoms_only` behaviour for static
       atoms but `:existing_atoms_only` is still used for dynamic atoms,
       such as atoms with interpolations.
@@ -686,9 +693,9 @@ defmodule Code do
   `Macro.to_string/2`, which converts a quoted form to a string/binary
   representation.
 
-  ## The `:static_atom_encoder` function
+  ## The `:static_atoms_encoder` function
 
-  When `static_atom_encoder: &my_encoder/2` is passed as an argument,
+  When `static_atoms_encoder: &my_encoder/2` is passed as an argument,
   `my_encoder/2` is called every time the tokenizer needs to create a
   "static" atom. Static atoms are atoms in the AST that function as
   aliases, remote calls, local calls, variable names, regular atoms
@@ -713,7 +720,7 @@ defmodule Code do
 
     * syntax keywords (`fn`, `do`, `else`, and so on)
 
-    * atoms containing interpolation (`:"\#{1 + 1} is two"`), as these
+    * atoms containing interpolation (`:"#{1 + 1} is two"`), as these
       atoms are constructed at runtime.
 
   """
diff --git a/lib/elixir/lib/code/formatter.ex b/lib/elixir/lib/code/formatter.ex
index 2bb193b46..4cfdb8cb4 100644
--- a/lib/elixir/lib/code/formatter.ex
+++ b/lib/elixir/lib/code/formatter.ex
@@ -206,7 +206,7 @@ defmodule Code.Formatter do
     ]
 
     parser_options = [
-      elixir_private_wrap_literals: true,
+      literal_encoder: &{:ok, {:__block__, &2, [&1]}},
       token_metadata: true
     ]
 
@@ -353,7 +353,7 @@ defmodule Code.Formatter do
       not interpolated?(entries) ->
         bitstring_to_algebra(meta, entries, state)
 
-      meta[:format] == :bin_heredoc ->
+      meta[:delimiter] == ~s["""] ->
         {doc, state} =
           entries
           |> prepend_heredoc_line()
@@ -375,7 +375,7 @@ defmodule Code.Formatter do
       not list_interpolated?(entries) ->
         remote_to_algebra(quoted, context, state)
 
-      meta[:format] == :list_heredoc ->
+      meta[:delimiter] == ~s['''] ->
         {doc, state} =
           entries
           |> prepend_heredoc_line()
@@ -433,12 +433,12 @@ defmodule Code.Formatter do
   end
 
   defp quoted_to_algebra({:__block__, meta, [list]}, _context, state) when is_list(list) do
-    case meta[:format] do
-      :list_heredoc ->
+    case meta[:delimiter] do
+      ~s['''] ->
         string = list |> List.to_string() |> escape_heredoc()
         {@single_heredoc |> concat(string) |> concat(@single_heredoc) |> force_unfit(), state}
 
-      :charlist ->
+      ~s['] ->
         string = list |> List.to_string() |> escape_string(@single_quote)
         {@single_quote |> concat(string) |> concat(@single_quote), state}
 
@@ -448,7 +448,7 @@ defmodule Code.Formatter do
   end
 
   defp quoted_to_algebra({:__block__, meta, [string]}, _context, state) when is_binary(string) do
-    if meta[:format] == :bin_heredoc do
+    if meta[:delimiter] == ~s["""] do
       string = escape_heredoc(string)
       {@double_heredoc |> concat(string) |> concat(@double_heredoc) |> force_unfit(), state}
     else
@@ -463,11 +463,11 @@ defmodule Code.Formatter do
 
   defp quoted_to_algebra({:__block__, meta, [integer]}, _context, state)
        when is_integer(integer) do
-    {integer_to_algebra(Keyword.fetch!(meta, :original)), state}
+    {integer_to_algebra(Keyword.fetch!(meta, :token)), state}
   end
 
   defp quoted_to_algebra({:__block__, meta, [float]}, _context, state) when is_float(float) do
-    {float_to_algebra(Keyword.fetch!(meta, :original)), state}
+    {float_to_algebra(Keyword.fetch!(meta, :token)), state}
   end
 
   defp quoted_to_algebra(
@@ -1544,43 +1544,40 @@ defmodule Code.Formatter do
 
   defp integer_to_algebra(text) do
     case text do
-      [?0, ?x | rest] ->
-        "0x" <> String.upcase(List.to_string(rest))
+      <<?0, ?x, rest::binary>> ->
+        "0x" <> String.upcase(rest)
 
-      [?0, base | _rest] = digits when base in [?b, ?o] ->
-        List.to_string(digits)
+      <<?0, base, _::binary>> = digits when base in [?b, ?o] ->
+        digits
 
-      [?? | _rest] = char ->
-        List.to_string(char)
+      <<??, _::binary>> = char ->
+        char
 
       decimal ->
-        List.to_string(insert_underscores(decimal))
+        insert_underscores(decimal)
     end
   end
 
   defp float_to_algebra(text) do
-    {int_part, [?. | decimal_part]} = Enum.split_while(text, &(&1 != ?.))
-
-    decimal_part =
-      decimal_part
-      |> List.to_string()
-      |> String.downcase()
-
-    List.to_string(insert_underscores(int_part)) <> "." <> decimal_part
+    [int_part, decimal_part] = :binary.split(text, ".")
+    decimal_part = String.downcase(decimal_part)
+    insert_underscores(int_part) <> "." <> decimal_part
   end
 
   defp insert_underscores(digits) do
     cond do
-      ?_ in digits ->
+      digits =~ "_" ->
         digits
 
-      length(digits) >= 6 ->
+      byte_size(digits) >= 6 ->
         digits
+        |> String.to_charlist()
         |> Enum.reverse()
         |> Enum.chunk_every(3)
         |> Enum.intersperse('_')
         |> List.flatten()
         |> Enum.reverse()
+        |> List.to_string()
 
       true ->
         digits
@@ -2119,12 +2116,12 @@ defmodule Code.Formatter do
   end
 
   defp next_break_fits?({:<<>>, meta, [_ | _] = entries}, state) do
-    meta[:format] == :bin_heredoc or
+    meta[:delimiter] == ~s["""] or
       (not interpolated?(entries) and eol_or_comments?(meta, state))
   end
 
   defp next_break_fits?({{:., _, [List, :to_charlist]}, meta, [[_ | _]]}, _state) do
-    meta[:format] == :list_heredoc
+    meta[:delimiter] == ~s[''']
   end
 
   defp next_break_fits?({{:., _, [_left, :{}]}, _, _}, _state) do
@@ -2132,11 +2129,11 @@ defmodule Code.Formatter do
   end
 
   defp next_break_fits?({:__block__, meta, [string]}, _state) when is_binary(string) do
-    meta[:format] == :bin_heredoc
+    meta[:delimiter] == ~s["""]
   end
 
   defp next_break_fits?({:__block__, meta, [list]}, _state) when is_list(list) do
-    meta[:format] != :charlist
+    meta[:delimeter] != ~s[']
   end
 
   defp next_break_fits?({form, _, [_ | _]}, _state) when form in [:fn, :%{}, :%] do
@@ -2208,25 +2205,17 @@ defmodule Code.Formatter do
     if force_args?(arg), do: force_unfit(doc), else: doc
   end
 
-  defp keyword?([{key, _} | list]) do
-    keyword_key?(key) and keyword?(list)
-  end
-
-  defp keyword?(rest) do
-    rest == []
-  end
+  defp keyword?([{_, _} | list]), do: keyword?(list)
+  defp keyword?(rest), do: rest == []
 
-  defp keyword_key?({:__block__, meta, [_]}) do
-    meta[:format] == :keyword
-  end
+  defp keyword_key?({:__block__, meta, [atom]}) when is_atom(atom),
+    do: meta[:delimiter] != ":"
 
-  defp keyword_key?({{:., _, [:erlang, :binary_to_atom]}, _, [{:<<>>, meta, _}, :utf8]}) do
-    meta[:format] == :keyword
-  end
+  defp keyword_key?({{:., _, [:erlang, :binary_to_atom]}, meta, [{:<<>>, _, _}, :utf8]}),
+    do: meta[:delimiter] != ":"
 
-  defp keyword_key?(_) do
-    false
-  end
+  defp keyword_key?(_),
+    do: false
 
   defp eol?(meta) do
     Keyword.get(meta, :newlines, 0) > 0
diff --git a/lib/elixir/lib/macro.ex b/lib/elixir/lib/macro.ex
index da7cc3a8c..6b91354b1 100644
--- a/lib/elixir/lib/macro.ex
+++ b/lib/elixir/lib/macro.ex
@@ -163,8 +163,8 @@ defmodule Macro do
       with parens. The `:closing` does not delimit the end of expression if
       there are `:do` and `:end` metadata  (when `:token_metadata` is true)
     * `:column` - the column number of the AST node (when `:columns` is true)
-    * `:delimiter` - contains the opening delimiter for sigils as a string
-      (such as `"{"`, `"/"`, etc)
+    * `:delimiter` - contains the opening delimiter for sigils, strings, atoms,
+      and charlists as a string (such as `"{"`, `"/"`, `":"`, etc)
     * `:do` - contains metadata about the `do` location in a function call with
       `do/end` blocks (when `:token_metadata` is true)
     * `:end` - contains metadata about the `end` location in a function call with
diff --git a/lib/elixir/src/elixir.erl b/lib/elixir/src/elixir.erl
index 98c7b8d9f..50e1b95dd 100644
--- a/lib/elixir/src/elixir.erl
+++ b/lib/elixir/src/elixir.erl
@@ -361,10 +361,14 @@ to_binary(List) when is_list(List) -> elixir_utils:characters_to_binary(List);
 to_binary(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8).
 
 handle_parsing_opts(File, Opts) ->
-  FormatterMetadata = lists:keyfind(elixir_private_wrap_literals, 1, Opts) == {elixir_private_wrap_literals, true},
-  PairingMetadata = lists:keyfind(token_metadata, 1, Opts) == {token_metadata, true},
+  LiteralEncoder =
+    case lists:keyfind(literal_encoder, 1, Opts) of
+      {literal_encoder, Fun} -> Fun;
+      false -> false
+    end,
+  TokenMetadata = lists:keyfind(token_metadata, 1, Opts) == {token_metadata, true},
   Columns = lists:keyfind(columns, 1, Opts) == {columns, true},
   put(elixir_parser_file, File),
   put(elixir_parser_columns, Columns),
-  put(elixir_token_metadata, PairingMetadata),
-  put(elixir_wrap_literals, FormatterMetadata).
+  put(elixir_token_metadata, TokenMetadata),
+  put(elixir_literal_encoder, LiteralEncoder).
diff --git a/lib/elixir/src/elixir_parser.yrl b/lib/elixir/src/elixir_parser.yrl
index 3ae5203c6..4c23f1903 100644
--- a/lib/elixir/src/elixir_parser.yrl
+++ b/lib/elixir/src/elixir_parser.yrl
@@ -39,7 +39,7 @@ Terminals
   capture_op rel_op
   'true' 'false' 'nil' 'do' eol ';' ',' '.'
   '(' ')' '[' ']' '{' '}' '<<' '>>' '%{}' '%'
-  int float char
+  int flt char
   .
 
 Rootsymbol grammar.
@@ -253,21 +253,21 @@ access_expr -> tuple : '$1'.
 access_expr -> 'true' : handle_literal(?id('$1'), '$1').
 access_expr -> 'false' : handle_literal(?id('$1'), '$1').
 access_expr -> 'nil' : handle_literal(?id('$1'), '$1').
-access_expr -> bin_string : build_bin_string('$1', [{format, string}]).
-access_expr -> list_string : build_list_string('$1', [{format, charlist}]).
+access_expr -> bin_string : build_bin_string('$1', delimiter(<<$">>)).
+access_expr -> list_string : build_list_string('$1', delimiter(<<$'>>)).
 access_expr -> bin_heredoc : build_bin_heredoc('$1').
 access_expr -> list_heredoc : build_list_heredoc('$1').
 access_expr -> bit_string : '$1'.
 access_expr -> sigil : build_sigil('$1').
-access_expr -> atom : handle_literal(?exprs('$1'), '$1', []).
-access_expr -> atom_safe : build_quoted_atom('$1', true, []).
-access_expr -> atom_unsafe : build_quoted_atom('$1', false, []).
+access_expr -> atom : handle_literal(?exprs('$1'), '$1', delimiter(<<$:>>)).
+access_expr -> atom_safe : build_quoted_atom('$1', true, delimiter(<<$:>>)).
+access_expr -> atom_unsafe : build_quoted_atom('$1', false, delimiter(<<$:>>)).
 access_expr -> dot_alias : '$1'.
 access_expr -> parens_call : '$1'.
 
-number -> int : handle_literal(number_value('$1'), '$1', [{original, ?exprs('$1')}]).
-number -> char : handle_literal(?exprs('$1'), '$1', [{original, number_value('$1')}]).
-number -> float : handle_literal(number_value('$1'), '$1', [{original, ?exprs('$1')}]).
+number -> int : handle_number(number_value('$1'), '$1', ?exprs('$1')).
+number -> flt : handle_number(number_value('$1'), '$1', ?exprs('$1')).
+number -> char : handle_number(?exprs('$1'), '$1', number_value('$1')).
 
 %% Also used by maps and structs
 parens_call -> dot_call_identifier call_args_parens : build_parens('$1', '$2', {[], []}).
@@ -526,12 +526,12 @@ call_args_parens -> open_paren call_args_parens_base ',' kw_base ',' close_paren
 
 % KV
 
-kw_eol -> kw_identifier : handle_literal(?exprs('$1'), '$1', [{format, keyword}]).
-kw_eol -> kw_identifier eol : handle_literal(?exprs('$1'), '$1', [{format, keyword}]).
-kw_eol -> kw_identifier_safe : build_quoted_atom('$1', true, [{format, keyword}]).
-kw_eol -> kw_identifier_safe eol : build_quoted_atom('$1', true, [{format, keyword}]).
-kw_eol -> kw_identifier_unsafe : build_quoted_atom('$1', false, [{format, keyword}]).
-kw_eol -> kw_identifier_unsafe eol : build_quoted_atom('$1', false, [{format, keyword}]).
+kw_eol -> kw_identifier : handle_literal(?exprs('$1'), '$1').
+kw_eol -> kw_identifier eol : handle_literal(?exprs('$1'), '$1').
+kw_eol -> kw_identifier_safe : build_quoted_atom('$1', true, []).
+kw_eol -> kw_identifier_safe eol : build_quoted_atom('$1', true, []).
+kw_eol -> kw_identifier_unsafe : build_quoted_atom('$1', false, []).
+kw_eol -> kw_identifier_unsafe eol : build_quoted_atom('$1', false, []).
 
 kw_base -> kw_eol container_expr : [{'$1', '$2'}].
 kw_base -> kw_base ',' kw_eol container_expr : [{'$3', '$4'} | '$1'].
@@ -670,9 +670,24 @@ handle_literal(Literal, Token) ->
   handle_literal(Literal, Token, []).
 
 handle_literal(Literal, Token, ExtraMeta) ->
-  case ?wrap_literals() of
-    true -> {'__block__', ExtraMeta ++ meta_from_token(Token), [Literal]};
-    false -> Literal
+  case get(elixir_literal_encoder) of
+    false ->
+      Literal;
+
+    Fun ->
+      Meta = ExtraMeta ++ meta_from_token(Token),
+      case Fun(Literal, Meta) of
+        {ok, EncodedLiteral} ->
+          EncodedLiteral;
+        {error, Reason} ->
+          return_error(Meta, elixir_utils:characters_to_list(Reason) ++ [": "], "literal")
+      end
+  end.
+
+handle_number(Number, Token, Original) ->
+  case ?token_metadata() of
+    true -> handle_literal(Number, Token, [{token, elixir_utils:characters_to_binary(Original)}]);
+    false -> handle_literal(Number, Token, [])
   end.
 
 number_value({_, {_, _, Value}, _}) ->
@@ -691,7 +706,7 @@ build_op({_Kind, Location, 'in'}, {UOp, _, [Left]}, Right) when ?rearrange_uop(U
 build_op({_Kind, Location, 'not in'}, Left, Right) ->
   InMeta = meta_from_location(Location),
   NotMeta =
-    case ?wrap_literals() of
+    case ?token_metadata() of
       true -> [{operator, 'not in'} | InMeta];
       false -> InMeta
     end,
@@ -863,16 +878,16 @@ build_sigil({sigil, Location, Sigil, Parts, Modifiers, Delimiter}) ->
    [{'<<>>', Meta, string_parts(Parts)}, Modifiers]}.
 
 build_bin_heredoc({bin_heredoc, Location, Args}) ->
-  build_bin_string({bin_string, Location, Args}, [{format, bin_heredoc}]).
+  build_bin_string({bin_string, Location, Args}, delimiter(<<$", $", $">>)).
 
 build_list_heredoc({list_heredoc, Location, Args}) ->
-  build_list_string({list_string, Location, Args}, [{format, list_heredoc}]).
+  build_list_string({list_string, Location, Args}, delimiter(<<$', $', $'>>)).
 
 build_bin_string({bin_string, _Location, [H]} = Token, ExtraMeta) when is_binary(H) ->
   handle_literal(H, Token, ExtraMeta);
 build_bin_string({bin_string, Location, Args}, ExtraMeta) ->
   Meta =
-    case ?wrap_literals() of
+    case ?token_metadata() of
       true -> ExtraMeta ++ meta_from_location(Location);
       false -> meta_from_location(Location)
     end,
@@ -883,7 +898,7 @@ build_list_string({list_string, _Location, [H]} = Token, ExtraMeta) when is_bina
 build_list_string({list_string, Location, Args}, ExtraMeta) ->
   Meta = meta_from_location(Location),
   MetaWithExtra =
-    case ?wrap_literals() of
+    case ?token_metadata() of
       true -> ExtraMeta ++ Meta;
       false -> Meta
     end,
@@ -895,11 +910,11 @@ build_quoted_atom({_, _Location, [H]} = Token, Safe, ExtraMeta) when is_binary(H
 build_quoted_atom({_, Location, Args}, Safe, ExtraMeta) ->
   Meta = meta_from_location(Location),
   MetaWithExtra =
-    case ?wrap_literals() of
+    case ?token_metadata() of
       true -> ExtraMeta ++ Meta;
       false -> Meta
     end,
-  {{'.', Meta, [erlang, binary_to_atom_op(Safe)]}, Meta, [{'<<>>', MetaWithExtra, string_parts(Args)}, utf8]}.
+  {{'.', Meta, [erlang, binary_to_atom_op(Safe)]}, MetaWithExtra, [{'<<>>', Meta, string_parts(Args)}, utf8]}.
 
 binary_to_atom_op(true)  -> binary_to_existing_atom;
 binary_to_atom_op(false) -> binary_to_atom.
@@ -910,12 +925,13 @@ charlist_part(Binary) when is_binary(Binary) ->
   Binary;
 charlist_part({Begin, End, Tokens}) ->
   Form = string_tokens_parse(Tokens),
-  Meta =
-    case ?wrap_literals() of
-      true -> [{closing, meta_from_location(End)} | meta_from_location(Begin)];
-      false -> meta_from_location(Begin)
+  Meta = meta_from_location(Begin),
+  MetaWithExtra =
+    case ?token_metadata() of
+      true -> [{closing, meta_from_location(End)} | Meta];
+      false -> Meta
     end,
-  {{'.', Meta, ['Elixir.Kernel', to_string]}, Meta, [Form]}.
+  {{'.', Meta, ['Elixir.Kernel', to_string]}, MetaWithExtra, [Form]}.
 
 string_parts(Parts) ->
   [string_part(Part) || Part <- Parts].
@@ -923,12 +939,13 @@ string_part(Binary) when is_binary(Binary) ->
   Binary;
 string_part({Begin, End, Tokens}) ->
   Form = string_tokens_parse(Tokens),
-  Meta =
-    case ?wrap_literals() of
+  Meta = meta_from_location(Begin),
+  MetaWithExtra =
+    case ?token_metadata() of
       true -> [{closing, meta_from_location(End)} | meta_from_location(Begin)];
       false -> meta_from_location(Begin)
     end,
-  {'::', Meta, [{{'.', Meta, ['Elixir.Kernel', to_string]}, Meta, [Form]}, {binary, Meta, nil}]}.
+  {'::', Meta, [{{'.', Meta, ['Elixir.Kernel', to_string]}, MetaWithExtra, [Form]}, {binary, Meta, nil}]}.
 
 string_tokens_parse(Tokens) ->
   case parse(Tokens) of
@@ -936,6 +953,12 @@ string_tokens_parse(Tokens) ->
     {error, _} = Error -> throw(Error)
   end.
 
+delimiter(Delimiter) ->
+  case ?token_metadata() of
+    true -> [{delimiter, Delimiter}];
+    false -> []
+  end.
+
 %% Keywords
 
 check_stab([{'->', _, [_, _]}], _) -> stab;
diff --git a/lib/elixir/src/elixir_tokenizer.erl b/lib/elixir/src/elixir_tokenizer.erl
index 201d19fd9..b02250eaf 100644
--- a/lib/elixir/src/elixir_tokenizer.erl
+++ b/lib/elixir/src/elixir_tokenizer.erl
@@ -476,6 +476,8 @@ tokenize([$: | String] = Original, Line, Column, Scope, Tokens) ->
   end;
 
 % Integers and floats
+% We use int and flt otherwise elixir_parser won't format them
+% properly in case of errors.
 
 tokenize([H | T], Line, Column, Scope, Tokens) when ?is_digit(H) ->
   case tokenize_number(T, [H], 1, false) of
@@ -485,7 +487,7 @@ tokenize([H | T], Line, Column, Scope, Tokens) when ?is_digit(H) ->
       Token = {int, {Line, Column, Number}, Original},
       tokenize(Rest, Line, Column + Length, Scope, [Token | Tokens]);
     {Rest, Number, Original, Length} ->
-      Token = {float, {Line, Column, Number}, Original},
+      Token = {flt, {Line, Column, Number}, Original},
       tokenize(Rest, Line, Column + Length, Scope, [Token | Tokens])
   end;
 
diff --git a/lib/elixir/test/elixir/code_test.exs b/lib/elixir/test/elixir/code_test.exs
index 840ddf12c..3f3c8e9cb 100644
--- a/lib/elixir/test/elixir/code_test.exs
+++ b/lib/elixir/test/elixir/code_test.exs
@@ -301,41 +301,58 @@ defmodule CodeTest do
     end
   end
 
-  test "string_to_quoted/2 with :elixir_private_wrap_literals wraps literals in blocks" do
-    string_to_quoted = &Code.string_to_quoted!(&1, elixir_private_wrap_literals: true)
+  describe "string_to_quoted/2 with :literal_encoder" do
+    test "wraps literals in blocks" do
+      opts = [literal_encoder: &{:ok, {:__block__, &2, [&1]}}, token_metadata: true]
+      string_to_quoted = &Code.string_to_quoted!(&1, opts)
+
+      assert string_to_quoted.(~s("one")) == {:__block__, [delimiter: "\"", line: 1], ["one"]}
+      assert string_to_quoted.("'one'") == {:__block__, [delimiter: "'", line: 1], ['one']}
+      assert string_to_quoted.("?é") == {:__block__, [token: "?é", line: 1], [233]}
+      assert string_to_quoted.("0b10") == {:__block__, [token: "0b10", line: 1], [2]}
+      assert string_to_quoted.("12") == {:__block__, [token: "12", line: 1], [12]}
+      assert string_to_quoted.("0o123") == {:__block__, [token: "0o123", line: 1], [83]}
+      assert string_to_quoted.("0xEF") == {:__block__, [token: "0xEF", line: 1], [239]}
+      assert string_to_quoted.("12.3") == {:__block__, [token: "12.3", line: 1], [12.3]}
+      assert string_to_quoted.("nil") == {:__block__, [line: 1], [nil]}
+      assert string_to_quoted.(":one") == {:__block__, [delimiter: ":", line: 1], [:one]}
+
+      args = [[{:__block__, [token: "1", line: 1], [1]}]]
+
+      assert string_to_quoted.("[1]") ==
+               {:__block__, [closing: [line: 1], line: 1], args}
 
-    assert string_to_quoted.(~s("one")) == {:__block__, [format: :string, line: 1], ["one"]}
-    assert string_to_quoted.("'one'") == {:__block__, [format: :charlist, line: 1], ['one']}
-    assert string_to_quoted.("?é") == {:__block__, [original: '?é', line: 1], [233]}
-    assert string_to_quoted.("0b10") == {:__block__, [original: '0b10', line: 1], [2]}
-    assert string_to_quoted.("12") == {:__block__, [original: '12', line: 1], [12]}
-    assert string_to_quoted.("0o123") == {:__block__, [original: '0o123', line: 1], [83]}
-    assert string_to_quoted.("0xEF") == {:__block__, [original: '0xEF', line: 1], [239]}
-    assert string_to_quoted.("12.3") == {:__block__, [original: '12.3', line: 1], [12.3]}
-    assert string_to_quoted.("nil") == {:__block__, [line: 1], [nil]}
-    assert string_to_quoted.(":one") == {:__block__, [line: 1], [:one]}
-
-    args = [[{:__block__, [original: '1', line: 1], [1]}]]
-
-    assert string_to_quoted.("[1]") ==
-             {:__block__, [line: 1], args}
+      args = [
+        {{:__block__, [delimiter: ":", line: 1], [:ok]},
+         {:__block__, [delimiter: ":", line: 1], [:test]}}
+      ]
 
-    args = [{{:__block__, [line: 1], [:ok]}, {:__block__, [line: 1], [:test]}}]
+      assert string_to_quoted.("{:ok, :test}") ==
+               {:__block__, [closing: [line: 1], line: 1], args}
 
-    assert string_to_quoted.("{:ok, :test}") ==
-             {:__block__, [line: 1], args}
+      assert string_to_quoted.(~s("""\nhello\n""")) ==
+               {:__block__, [delimiter: ~s["""], line: 1], ["hello\n"]}
 
-    assert string_to_quoted.(~s("""\nhello\n""")) ==
-             {:__block__, [format: :bin_heredoc, line: 1], ["hello\n"]}
+      assert string_to_quoted.("'''\nhello\n'''") ==
+               {:__block__, [delimiter: ~s['''], line: 1], ['hello\n']}
 
-    assert string_to_quoted.("'''\nhello\n'''") ==
-             {:__block__, [format: :list_heredoc, line: 1], ['hello\n']}
+      args = [
+        {:->, [line: 1],
+         [
+           [{:__block__, [token: "1", line: 1, closing: [line: 1], line: 1], [1]}],
+           {:__block__, [delimiter: "\"", line: 1], ["hello"]}
+         ]}
+      ]
 
-    left = {:__block__, [original: '1', line: 1, line: 1], [1]}
-    right = {:__block__, [format: :string, line: 1], ["hello"]}
-    args = [{:->, [line: 1], [[left], right]}]
+      assert string_to_quoted.(~s[fn (1) -> "hello" end]) ==
+               {:fn, [closing: [line: 1], line: 1], args}
+    end
 
-    assert string_to_quoted.(~s[fn (1) -> "hello" end]) == {:fn, [line: 1], args}
+    test "raises on bad literal" do
+      assert_raise SyntaxError, "nofile:1: oops: literal", fn ->
+        Code.string_to_quoted!(":one", literal_encoder: fn _, _ -> {:error, "oops"} end)
+      end
+    end
   end
 
   test "compile source" do
diff --git a/lib/elixir/test/elixir/kernel/errors_test.exs b/lib/elixir/test/elixir/kernel/errors_test.exs
index f9cfaed12..4ef6159f1 100644
--- a/lib/elixir/test/elixir/kernel/errors_test.exs
+++ b/lib/elixir/test/elixir/kernel/errors_test.exs
@@ -1027,6 +1027,12 @@ defmodule Kernel.ErrorsTest do
     assert_eval_raise SyntaxError, "nofile:1: syntax error before: ?す", ':ok ?す'
   end
 
+  test "numbers are printed correctly in syntax errors" do
+    assert_eval_raise SyntaxError, "nofile:1: syntax error before: \"12\"", ':ok 12'
+    assert_eval_raise SyntaxError, "nofile:1: syntax error before: \"0b1\"", ':ok 0b1'
+    assert_eval_raise SyntaxError, "nofile:1: syntax error before: \"12.3\"", ':ok 12.3'
+  end
+
   test "invalid \"fn do expr end\"" do
     assert_eval_raise SyntaxError,
                       "nofile:1: unexpected token: do. Anonymous functions are written as:\n\n    fn pattern -> expression end",
diff --git a/lib/elixir/test/erlang/tokenizer_test.erl b/lib/elixir/test/erlang/tokenizer_test.erl
index e7d71cf03..2ce0f4929 100644
--- a/lib/elixir/test/erlang/tokenizer_test.erl
+++ b/lib/elixir/test/erlang/tokenizer_test.erl
@@ -36,9 +36,9 @@ op_kw_test() ->
    {atom, {1, 6, nil}, bar}] = tokenize(":foo+:bar").
 
 scientific_test() ->
-  [{float, {1, 1, 0.1}, "1.0e-1"}] = tokenize("1.0e-1"),
-  [{float, {1, 1, 0.1}, "1.0E-1"}] = tokenize("1.0E-1"),
-  [{float, {1, 1, 1.2345678e-7}, "1_234.567_8e-10"}] = tokenize("1_234.567_8e-10"),
+  [{flt, {1, 1, 0.1}, "1.0e-1"}] = tokenize("1.0e-1"),
+  [{flt, {1, 1, 0.1}, "1.0E-1"}] = tokenize("1.0E-1"),
+  [{flt, {1, 1, 1.2345678e-7}, "1_234.567_8e-10"}] = tokenize("1_234.567_8e-10"),
   {1, 1, "invalid float number ", "1.0e309"} = tokenize_error("1.0e309").
 
 hex_bin_octal_test() ->
@@ -87,11 +87,11 @@ int_test() ->
   [{int, {1, 1, 100000}, "0100000"}] = tokenize("0100000").
 
 float_test() ->
-  [{float, {1, 1, 12.3}, "12.3"}] = tokenize("12.3"),
-  [{float, {1, 1, 12.3}, "12.3"}, {';', {1, 5, 0}}] = tokenize("12.3;"),
-  [{eol, {1, 1, 2}}, {float, {3, 1, 12.3}, "12.3"}] = tokenize("\n\n12.3"),
-  [{float, {1, 3, 12.3}, "12.3"}, {float, {1, 9, 23.4}, "23.4"}] = tokenize("  12.3  23.4  "),
-  [{float, {1, 1, 12.3}, "00_12.3_00"}] = tokenize("00_12.3_00"),
+  [{flt, {1, 1, 12.3}, "12.3"}] = tokenize("12.3"),
+  [{flt, {1, 1, 12.3}, "12.3"}, {';', {1, 5, 0}}] = tokenize("12.3;"),
+  [{eol, {1, 1, 2}}, {flt, {3, 1, 12.3}, "12.3"}] = tokenize("\n\n12.3"),
+  [{flt, {1, 3, 12.3}, "12.3"}, {flt, {1, 9, 23.4}, "23.4"}] = tokenize("  12.3  23.4  "),
+  [{flt, {1, 1, 12.3}, "00_12.3_00"}] = tokenize("00_12.3_00"),
   OversizedFloat = string:copies("9", 310) ++ ".0",
   {1, 1, "invalid float number ", OversizedFloat} = tokenize_error(OversizedFloat).