compiler: Replace exception trampolines with {succeeded,body|guard}

This is less elegant and easy to use but much faster to compile,
making our compile speeds mostly comparable to OTP 22.

 * {succeeded,guard} means we're in a guard and it's safe to remove
   both the checked instruction and the check itself when we know
   whether it will succeed.
 * {succeeded,body} means we're in a function body and must retain
   the checked instruction when we know it will fail. It's safe to
   remove both when we know it will succeed.

9 files changed, 174 insertions, 175 deletions

diff --git a/lib/compiler/src/beam_kernel_to_ssa.erl b/lib/compiler/src/beam_kernel_to_ssa.erl
index 6b552ae38c..e2c24a8551 100644
--- a/lib/compiler/src/beam_kernel_to_ssa.erl
+++ b/lib/compiler/src/beam_kernel_to_ssa.erl
@@ -218,7 +218,7 @@ select_val_cg(k_atom, {succeeded,Dst}, Vls, _Tf, _Vf, Sis, St0) ->
             %% following the `peek_message` and `wait_timeout`
             %% instructions.
             {Bool,St} = new_ssa_var('@ssa_bool', St0),
-            Succeeded = #b_set{op=succeeded,dst=Bool,args=[Dst]},
+            Succeeded = #b_set{op={succeeded,guard},dst=Bool,args=[Dst]},
             Br = #b_br{bool=Bool,succ=Succ,fail=Fail},
             {[Succeeded,Br|Sis],St};
         #b_literal{val=true}=Bool ->
@@ -333,33 +333,37 @@ make_uncond_branch(Fail) ->
     #b_br{bool=#b_literal{val=true},succ=Fail,fail=Fail}.
 
 %%
-%% The 'succeeded' instruction needs special treatment in catch blocks to
-%% prevent the checked operation from being optimized away if a later pass
-%% determines that it always fails.
+%% Success checks need to be treated differently in bodies and guards; a check
+%% in a guard can be safely removed when we know it fails because we know
+%% there's never any side-effects, but in bodies the checked instruction may
+%% throw an exception and we need to ensure it isn't optimized away.
+%%
+%% Checks are expressed as {succeeded,guard} and {succeeded,body} respectively,
+%% where the latter has a side-effect (see beam_ssa:no_side_effect/1) and the
+%% former does not. This ensures that passes like ssa_opt_dead and ssa_opt_live
+%% won't optimize away pure operations that may throw an exception, since their
+%% result is used in {succeeded,body}.
+%%
+%% Other than the above details, the two variants are equivalent and most
+%% passes that care about them can simply match {succeeded,_}.
 %%
 
-make_succeeded(Var, {in_catch, CatchLbl}, St0) ->
-    {Bool, St1} = new_ssa_var('@ssa_bool', St0),
-    {Succ, St2} = new_label(St1),
-    {Fail, St} = new_label(St2),
+make_succeeded(Var, {guard, Fail}, St) ->
+    make_succeeded_1(Var, guard, Fail, St);
+make_succeeded(Var, {in_catch, CatchLbl}, St) ->
+    make_succeeded_1(Var, body, CatchLbl, St);
+make_succeeded(Var, {no_catch, Fail}, St) ->
+    #cg{ultimate_failure=Fail} = St,            %Assertion
+    make_succeeded_1(Var, body, Fail, St).
 
-    Check = [#b_set{op=succeeded,dst=Bool,args=[Var]},
-             #b_br{bool=Bool,succ=Succ,fail=Fail}],
+make_succeeded_1(Var, Kind, Fail, St0) ->
+    {Bool,St1} = new_ssa_var('@ssa_bool', St0),
+    {Succ,St} = new_label(St1),
 
-    %% Add a dummy block that references the checked variable, ensuring it
-    %% stays alive and that it won't be merged with the landing pad.
-    Trampoline = [{label,Fail},
-                  #b_set{op=exception_trampoline,args=[Var]},
-                  make_uncond_branch(CatchLbl)],
+    Check = [#b_set{op={succeeded,Kind},dst=Bool,args=[Var]},
+             #b_br{bool=Bool,succ=Succ,fail=Fail}],
 
-    {Check ++ Trampoline ++ [{label,Succ}], St};
-make_succeeded(Var, {no_catch, Fail}, St) ->
-    %% Ultimate failure raises an exception, so we must treat it as if it were
-    %% in a catch to keep it from being optimized out.
-    #cg{ultimate_failure=Fail} = St,            %Assertion
-    make_succeeded(Var, {in_catch, Fail}, St);
-make_succeeded(Var, {guard, Fail}, St) ->
-    make_cond_branch(succeeded, [Var], Fail, St).
+    {Check ++ [{label,Succ}], St}.
 
 %% Instructions for selection of binary segments.
 
@@ -391,11 +395,11 @@ select_bin_seg(#k_val_clause{val=#k_bin_int{size=Sz,unit=U,flags=Fs,
     {Bis,St} = match_cg(B, Fail, St1),
     Is = case Mis ++ Bis of
              [#b_set{op=bs_match,args=[#b_literal{val=string},OtherCtx1,Bin1]},
-              #b_set{op=succeeded,dst=Bool1},
+              #b_set{op={succeeded,guard},dst=Bool1},
               #b_br{bool=Bool1,succ=Succ,fail=Fail},
               {label,Succ},
               #b_set{op=bs_match,dst=Dst,args=[#b_literal{val=string},_OtherCtx2,Bin2]}|
-              [#b_set{op=succeeded,dst=Bool2},
+              [#b_set{op={succeeded,guard},dst=Bool2},
                #b_br{bool=Bool2,fail=Fail}|_]=Is0] ->
                  %% We used to do this optimization later, but it
                  %% turns out that in huge functions with many
@@ -680,12 +684,6 @@ call_cg(Func, As, [#k_var{name=R}|MoreRs]=Rs, Le, St0) ->
     end.
 
 enter_cg(Func, As0, Le, St0) ->
-    %% Adding a trampoline here would give us greater freedom in rewriting
-    %% calls, but doing so makes it difficult to tell tail calls apart from
-    %% body calls during code generation.
-    %%
-    %% We therefore skip the trampoline, reasoning that we've already left the
-    %% current function by the time an exception is thrown.
     As = ssa_args([Func|As0], St0),
     {Ret,St} = new_ssa_var('@ssa_ret', St0),
     Call = #b_set{anno=line_anno(Le),op=call,dst=Ret,args=As},
diff --git a/lib/compiler/src/beam_ssa.erl b/lib/compiler/src/beam_ssa.erl
index 9a8ae9b407..cd948e1dd8 100644
--- a/lib/compiler/src/beam_ssa.erl
+++ b/lib/compiler/src/beam_ssa.erl
@@ -82,7 +82,12 @@
 -type literal_value() :: atom() | integer() | float() | list() |
                          nil() | tuple() | map() | binary() | fun().
 
--type op()   :: {'bif',atom()} | {'float',float_op()} | prim_op() | cg_prim_op().
+-type op()   :: {'bif',atom()} |
+                {'float',float_op()} |
+                {'succeeded', 'guard' | 'body'} |
+                prim_op() |
+                cg_prim_op().
+
 -type anno() :: #{atom() := any()}.
 
 -type block_map() :: #{label():=b_blk()}.
@@ -102,7 +107,7 @@
                    'bs_match' | 'bs_put' | 'bs_start_match' | 'bs_test_tail' |
                    'bs_utf16_size' | 'bs_utf8_size' | 'build_stacktrace' |
                    'call' | 'catch_end' |
-                   'extract' | 'exception_trampoline' |
+                   'extract' |
                    'get_hd' | 'get_map_element' | 'get_tl' | 'get_tuple_element' |
                    'has_map_field' |
                    'is_nonempty_list' | 'is_tagged_tuple' |
@@ -111,7 +116,6 @@
                    'make_fun' | 'new_try_tag' |
                    'peek_message' | 'phi' | 'put_list' | 'put_map' | 'put_tuple' |
                    'raw_raise' | 'recv_next' | 'remove_message' | 'resume' |
-                   'succeeded' |
                    'timeout' |
                    'wait' | 'wait_timeout'.
 
@@ -123,7 +127,7 @@
                       'bs_restore' | 'bs_save' | 'bs_set_position' | 'bs_skip' |
                       'copy' | 'match_fail' | 'put_tuple_arity' |
                       'put_tuple_element' | 'put_tuple_elements' |
-                      'set_tuple_element'.
+                      'set_tuple_element' | 'succeeded'.
 
 -import(lists, [foldl/3,keyfind/3,mapfoldl/3,member/2,reverse/1,sort/1]).
 
@@ -215,7 +219,7 @@ no_side_effect(#b_set{op=Op}) ->
         put_map -> true;
         put_list -> true;
         put_tuple -> true;
-        succeeded -> true;
+        {succeeded,guard} -> true;
         _ -> false
     end.
 
diff --git a/lib/compiler/src/beam_ssa_bool.erl b/lib/compiler/src/beam_ssa_bool.erl
index 59c2346d44..1fb815e753 100644
--- a/lib/compiler/src/beam_ssa_bool.erl
+++ b/lib/compiler/src/beam_ssa_bool.erl
@@ -68,7 +68,7 @@
 %%   _2 = bif:is_integer _0
 %%   _3 = bif:is_atom _1
 %%   _7 = bif:'and' _2, _3
-%%   @ssa_bool = succeeded _7
+%%   @ssa_bool = succeeded:guard _7
 %%   br @ssa_bool, label 4, label 3
 %%
 %% 4:
@@ -330,7 +330,8 @@ pre_opt_is([#b_set{op=phi,dst=Dst,args=Args0}=I0|Is], Reached, Sub0, Acc) ->
                     pre_opt_is(Is, Reached, Sub0, [I|Acc])
             end
     end;
-pre_opt_is([#b_set{op=succeeded,dst=Dst,args=Args0}=I0|Is], Reached, Sub0, Acc) ->
+pre_opt_is([#b_set{op={succeeded,_},dst=Dst,args=Args0}=I0|Is],
+           Reached, Sub0, Acc) ->
     [Arg] = Args = sub_args(Args0, Sub0),
     I = I0#b_set{args=Args},
     case pre_is_safe_bool(Arg, Sub0) of
@@ -354,7 +355,7 @@ pre_opt_is([#b_set{dst=Dst,args=Args0}=I0|Is], Reached, Sub0, Acc) ->
                 #b_var{}=Var ->
                     %% We must remove the 'succeeded' instruction that
                     %% follows since the variable it checks is gone.
-                    [#b_set{op=succeeded,dst=SuccDst,args=[Dst]}] = Is,
+                    [#b_set{op={succeeded,_},dst=SuccDst,args=[Dst]}] = Is,
                     Sub = Sub0#{Dst=>Var,SuccDst=>#b_literal{val=true}},
                     pre_opt_is([], Reached, Sub, Acc);
                 #b_literal{}=Lit ->
@@ -707,7 +708,7 @@ split_dom_block(L, Blocks0) ->
     Blocks = Blocks0#{L:=Blk},
     {PreIs,Blocks}.
 
-split_dom_block_is([#b_set{},#b_set{op=succeeded}]=Is, PreAcc) ->
+split_dom_block_is([#b_set{},#b_set{op={succeeded,_}}]=Is, PreAcc) ->
     {reverse(PreAcc),Is};
 split_dom_block_is([#b_set{}=I|Is]=Is0, PreAcc) ->
     case is_bool_expr(I) of
@@ -973,7 +974,7 @@ convert_to_br_node(I, Target, G0, St) ->
 ensure_no_failing_instructions(First, Second, G, St) ->
     Vs0 = covered(get_vertex(First, St), get_vertex(Second, St), G),
     Vs = [{V,beam_digraph:vertex(G, V)} || V <- Vs0],
-    Failing = [P || {V,#b_set{op=succeeded}}=P <- Vs,
+    Failing = [P || {V,#b_set{op={succeeded,_}}}=P <- Vs,
                     not eaten_by_phi(V, G)],
     case Failing of
         [] -> ok;
diff --git a/lib/compiler/src/beam_ssa_bsm.erl b/lib/compiler/src/beam_ssa_bsm.erl
index 1d5a99a7a1..4c464448ff 100644
--- a/lib/compiler/src/beam_ssa_bsm.erl
+++ b/lib/compiler/src/beam_ssa_bsm.erl
@@ -471,7 +471,7 @@ combine_matches(#b_function{bs=Blocks0,cnt=Counter0}=F, ModInfo) ->
 cm_1([#b_set{ op=bs_start_match,
               dst=Ctx,
               args=[_,Src] },
-      #b_set{ op=succeeded,
+      #b_set{ op={succeeded,guard},
               dst=Bool,
               args=[Ctx] }]=MatchSeq, Acc0, Lbl, State0) ->
     Acc = reverse(Acc0),
diff --git a/lib/compiler/src/beam_ssa_dead.erl b/lib/compiler/src/beam_ssa_dead.erl
index 021b773419..af769e6d4b 100644
--- a/lib/compiler/src/beam_ssa_dead.erl
+++ b/lib/compiler/src/beam_ssa_dead.erl
@@ -440,7 +440,7 @@ eval_is([#b_set{op=phi,dst=Dst,args=Args}|Is], From, Bs0, St) ->
     Val = get_phi_arg(Args, From),
     Bs = bind_var(Dst, Val, Bs0),
     eval_is(Is, From, Bs, St);
-eval_is([#b_set{op=succeeded,dst=Dst,args=[Var]}], _From, Bs, _St) ->
+eval_is([#b_set{op={succeeded,guard},dst=Dst,args=[Var]}], _From, Bs, _St) ->
     case Bs of
         #{Var:=failed} ->
             bind_var(Dst, #b_literal{val=false}, Bs);
diff --git a/lib/compiler/src/beam_ssa_lint.erl b/lib/compiler/src/beam_ssa_lint.erl
index 224095d4c4..3521c116f1 100644
--- a/lib/compiler/src/beam_ssa_lint.erl
+++ b/lib/compiler/src/beam_ssa_lint.erl
@@ -236,16 +236,17 @@ vvars_block(Id, State0) ->
 vvars_block_1([], State) ->
     State;
 vvars_block_1([#b_set{dst=OpVar,args=OpArgs}=I,
-               #b_set{op=succeeded,args=[OpVar],dst=SuccVar}], State) ->
+               #b_set{op={succeeded,Kind},args=[OpVar],dst=SuccVar}], State) ->
+    true = Kind =:= guard orelse Kind =:= body, %Assertion.
     ok = vvars_assert_args(OpArgs, I, State),
     vvars_save_var(SuccVar, vvars_save_var(OpVar, State));
-vvars_block_1([#b_set{op=succeeded,args=Args}=I | [_|_]], State) ->
+vvars_block_1([#b_set{op={succeeded,guard},args=Args}=I | [_|_]], State) ->
     ok = vvars_assert_args(Args, I, State),
     %% 'succeeded' must be the last instruction in its block.
     throw({succeeded_not_last, I});
-vvars_block_1([#b_set{op=succeeded,args=Args}=I], State)->
+vvars_block_1([#b_set{op={succeeded,_},args=Args}=I], State)->
     ok = vvars_assert_args(Args, I, State),
-    %% 'succeeded' must be be directly preceded by the operation it checks.
+    %% 'succeeded' must be directly preceded by the operation it checks.
     throw({succeeded_not_preceded, I});
 vvars_block_1([#b_set{ dst = Dst, op = phi } | Is], State) ->
     %% We don't check phi node arguments at this point since we may not have
diff --git a/lib/compiler/src/beam_ssa_opt.erl b/lib/compiler/src/beam_ssa_opt.erl
index 00d57ca7a9..5ec6f26f37 100644
--- a/lib/compiler/src/beam_ssa_opt.erl
+++ b/lib/compiler/src/beam_ssa_opt.erl
@@ -743,7 +743,7 @@ collect_element_calls([{L,#b_blk{is=Is0,last=Last}}|Bs]) ->
     case {Is0,Last} of
         {[#b_set{op={bif,element},dst=Element,
                  args=[#b_literal{val=N},#b_var{}=Tuple]},
-          #b_set{op=succeeded,dst=Bool,args=[Element]}],
+          #b_set{op={succeeded,guard},dst=Bool,args=[Element]}],
          #b_br{bool=Bool,succ=Succ,fail=Fail}} ->
             Info = {L,Succ,{Tuple,Fail},N},
             [Info|collect_element_calls(Bs)];
@@ -906,7 +906,7 @@ cse([{L,#b_blk{is=Is0,last=Last0}=Blk}|Bs], Sub0, M0) ->
     [{L,Blk#b_blk{is=Is,last=Last}}|cse(Bs, Sub, M)];
 cse([], _, _) -> [].
 
-cse_successors([#b_set{op=succeeded,args=[Src]},Bif|_], Blk, EsSucc, M0) ->
+cse_successors([#b_set{op={succeeded,_},args=[Src]},Bif|_], Blk, EsSucc, M0) ->
     case cse_suitable(Bif) of
         true ->
             %% The previous instruction only has a valid value at the success branch.
@@ -944,7 +944,7 @@ cse_successors_1([L|Ls], Es0, M) ->
     end;
 cse_successors_1([], _, M) -> M.
 
-cse_is([#b_set{op=succeeded,dst=Bool,args=[Src]}=I0|Is], Es, Sub0, Acc) ->
+cse_is([#b_set{op={succeeded,_},dst=Bool,args=[Src]}=I0|Is], Es, Sub0, Acc) ->
     I = sub(I0, Sub0),
     case I of
         #b_set{args=[Src]} ->
@@ -1056,21 +1056,6 @@ float_can_optimize_blk(#b_blk{last=#b_br{bool=#b_var{},fail=F}},
 float_can_optimize_blk(#b_blk{}, #fs{}) ->
     false.
 
-float_opt([{L,#b_blk{is=[#b_set{op=exception_trampoline,args=[Var]}]}=Blk0} |
-           Bs0], Count0, Fs) ->
-    %% If we've replaced a BIF with float operations, we'll have a lot of extra
-    %% blocks that jump to the same failure block, which may have a trampoline
-    %% that refers to the original operation.
-    %%
-    %% Since the point of the trampoline is to keep the BIF from being removed
-    %% by liveness optimization, we can discard it as the liveness pass leaves
-    %% floats alone.
-    Blk = case cerl_sets:is_element(Var, Fs#fs.vars) of
-              true -> Blk0#b_blk{is=[]};
-              false -> Blk0
-          end,
-    {Bs, Count} = float_opt(Bs0, Count0, Fs),
-    {[{L,Blk}|Bs],Count};
 float_opt([{L,Blk}|Bs0], Count0, Fs) ->
     case float_can_optimize_blk(Blk, Fs) of
         true ->
@@ -1131,7 +1116,7 @@ float_conv([{L,#b_blk{is=Is0}=Blk0}|Bs0], Fail, Count0) ->
         [#b_set{op={float,convert}}=Conv] ->
             {Bool0,Count1} = new_reg('@ssa_bool', Count0),
             Bool = #b_var{name=Bool0},
-            Succeeded = #b_set{op=succeeded,dst=Bool,
+            Succeeded = #b_set{op={succeeded,body},dst=Bool,
                                args=[Conv#b_set.dst]},
             Is = [Conv,Succeeded],
             [{NextL,_}|_] = Bs0,
@@ -1193,7 +1178,7 @@ float_maybe_flush(Blk0, #fs{s=cleared,fail=Fail,bs=Blocks}=Fs0, Count0) ->
 float_maybe_flush(Blk, Fs, Count) ->
     {[],Blk,Fs,Count}.
 
-float_opt_is([#b_set{op=succeeded,args=[Src]}=I0],
+float_opt_is([#b_set{op={succeeded,_},args=[Src]}=I0],
              #fs{regs=Rs}=Fs, Count, Acc) ->
     case Rs of
         #{Src:=Fr} ->
@@ -1361,7 +1346,7 @@ live_opt_is([#b_set{op=phi,dst=Dst}=I|Is], Live, Acc) ->
         false ->
             live_opt_is(Is, Live, Acc)
     end;
-live_opt_is([#b_set{op=succeeded,dst=SuccDst,args=[MapDst]}=SuccI,
+live_opt_is([#b_set{op={succeeded,_},dst=SuccDst,args=[MapDst]}=SuccI,
              #b_set{op=get_map_element,dst=MapDst}=MapI | Is],
             Live0, Acc) ->
     case {gb_sets:is_member(SuccDst, Live0),
@@ -1485,13 +1470,12 @@ is_safe_without_try([#b_set{op=kill_try_tag}|Is], Acc) ->
 is_safe_without_try([#b_set{op=extract}|_], _Acc) ->
     %% The error reason is accessed.
     unsafe;
-is_safe_without_try([#b_set{op=exception_trampoline}|Is], Acc) ->
-    is_safe_without_try(Is, Acc);
 is_safe_without_try([#b_set{op=landingpad}|Is], Acc) ->
     is_safe_without_try(Is, Acc);
 is_safe_without_try([#b_set{op=Op}=I|Is], Acc) ->
     IsSafe = case Op of
                  phi -> true;
+                 {succeeded,_} -> true;
                  _ -> beam_ssa:no_side_effect(I)
              end,
     case IsSafe of
@@ -1573,7 +1557,7 @@ coalesce_skips_is([#b_set{op=bs_match,
                                 Ctx0,Type,Flags,
                                 #b_literal{val=Size0},
                                 #b_literal{val=Unit0}]}=Skip0,
-                   #b_set{op=succeeded}],
+                   #b_set{op={succeeded,guard}}],
                   #b_br{succ=L2,fail=Fail}=Br0,
                   Bs0) when is_integer(Size0) ->
     case Bs0 of
@@ -1582,7 +1566,7 @@ coalesce_skips_is([#b_set{op=bs_match,
                                args=[#b_literal{val=skip},_,_,_,
                                      #b_literal{val=Size1},
                                      #b_literal{val=Unit1}]},
-                        #b_set{op=succeeded}=Succeeded],
+                        #b_set{op={succeeded,guard}}=Succeeded],
                     last=#b_br{fail=Fail}=Br}}|Bs] when is_integer(Size1) ->
             SkipBits = Size0 * Unit0 + Size1 * Unit1,
             Skip = Skip0#b_set{dst=SkipDst,
@@ -1662,7 +1646,7 @@ bsm_update_bits(_, Bits) -> Bits.
 bsm_shortcut([{L,#b_blk{is=Is,last=Last0}=Blk}|Bs], PosMap) ->
     case {Is,Last0} of
         {[#b_set{op=bs_match,dst=New,args=[_,Old|_]},
-          #b_set{op=succeeded,dst=Bool,args=[New]}],
+          #b_set{op={succeeded,guard},dst=Bool,args=[New]}],
          #b_br{bool=Bool,fail=Fail}} ->
             case PosMap of
                 #{Old:=Bits,Fail:={TailBits,NextFail}} when Bits > TailBits ->
@@ -1860,7 +1844,7 @@ opt_bs_put_split_int_1(Int, L, R) ->
 %%%   .
 %%%   .
 %%%   Size = bif:tuple_size Var
-%%%   BoolVar1 = succeeded Size
+%%%   BoolVar1 = succeeded:guard Size
 %%%   br BoolVar1, label 4, label 3
 %%%
 %%% 4:
@@ -1972,7 +1956,7 @@ opt_tup_size_2(PreIs, TupleSizeIs, PreL, EqL, Tuple, Fail, Count0, Acc) ->
       {PreL,PreBlk}|Acc],Count}.
 
 opt_tup_size_is([#b_set{op={bif,tuple_size},dst=Size,args=[Tuple]}=I,
-                 #b_set{op=succeeded,dst=Bool,args=[Size]}],
+                 #b_set{op={succeeded,_},dst=Bool,args=[Size]}],
                 Bool, Size, Acc) ->
     {reverse(Acc),[I],Tuple};
 opt_tup_size_is([I|Is], Bool, Size, Acc) ->
@@ -2067,9 +2051,11 @@ verify_merge_is([#b_set{op=Op}|_]) ->
 verify_merge_is(_) ->
     ok.
 
-is_merge_allowed(_, #b_blk{}, #b_blk{is=[#b_set{op=exception_trampoline}|_]}) ->
+is_merge_allowed(?EXCEPTION_BLOCK, #b_blk{}, #b_blk{}) ->
     false;
-is_merge_allowed(_, #b_blk{is=[#b_set{op=exception_trampoline}|_]}, #b_blk{}) ->
+is_merge_allowed(_L, #b_blk{is=[#b_set{op=landingpad} | _]}, #b_blk{}) ->
+    false;
+is_merge_allowed(_L, #b_blk{}, #b_blk{is=[#b_set{op=landingpad} | _]}) ->
     false;
 is_merge_allowed(L, #b_blk{}=Blk1, #b_blk{is=[#b_set{}=I|_]}=Blk2) ->
     not beam_ssa:is_loop_header(I) andalso
@@ -2188,7 +2174,6 @@ unsuitable_1([{L,#b_blk{is=[#b_set{op=Op}=I|_]}}|Bs]) ->
     Unsuitable = case Op of
                      bs_extract -> true;
                      bs_put -> true;
-                     exception_trampoline -> true;
                      {float,_} -> true;
                      landingpad -> true;
                      _ -> beam_ssa:is_loop_header(I)
@@ -2327,7 +2312,7 @@ insert_def_is([#b_set{op=Op}=I|Is]=Is0, V, Def) ->
                   _ -> here
               end,
     Action = case Is of
-                 [#b_set{op=succeeded}|_] -> here;
+                 [#b_set{op={succeeded,_}}|_] -> here;
                  _ -> Action0
              end,
     case Action of
@@ -2621,8 +2606,6 @@ non_guards(Linear) ->
 
 non_guards_1([{L,#b_blk{is=Is}}|Bs]) ->
     case Is of
-        [#b_set{op=exception_trampoline}|_] ->
-            [L | non_guards_1(Bs)];
         [#b_set{op=landingpad}|_] ->
             [L | non_guards_1(Bs)];
         _ ->
diff --git a/lib/compiler/src/beam_ssa_pre_codegen.erl b/lib/compiler/src/beam_ssa_pre_codegen.erl
index b94223c3cf..fdff727ee7 100644
--- a/lib/compiler/src/beam_ssa_pre_codegen.erl
+++ b/lib/compiler/src/beam_ssa_pre_codegen.erl
@@ -71,7 +71,7 @@
 -include("beam_ssa.hrl").
 
 -import(lists, [all/2,any/2,append/1,duplicate/2,
-                foldl/3,last/1,map/2,member/2,partition/2,
+                foldl/3,last/1,member/2,partition/2,
                 reverse/1,reverse/2,sort/1,splitwith/2,zip/2]).
 
 -spec module(beam_ssa:b_module(), [compile:option()]) ->
@@ -119,7 +119,6 @@ passes(Opts) ->
     Ps = [?PASS(assert_no_critical_edges),
 
           %% Preliminaries.
-          ?PASS(exception_trampolines),
           ?PASS(fix_bs),
           ?PASS(sanitize),
           ?PASS(match_fail_instructions),
@@ -489,7 +488,8 @@ bs_restores_is([#b_set{op=Op,dst=Dst,args=Args}|Is],
     FPos = SPos,
 
     bs_restores_is(Is, CtxChain, SPos, FPos, Rs);
-bs_restores_is([#b_set{op=succeeded,args=[Arg]}], CtxChain, SPos, FPos0, Rs) ->
+bs_restores_is([#b_set{op={succeeded,guard},args=[Arg]}],
+               CtxChain, SPos, FPos0, Rs) ->
     %% If we're branching on a match operation, the positions will be different
     %% depending on whether it succeeds.
     Ctx = bs_subst_ctx(Arg, CtxChain),
@@ -595,7 +595,7 @@ bs_insert_is([#b_set{dst=Dst}=I0|Is], Saves, Restores, XFrm, Acc0) ->
            end,
     Acc = [I | Pre] ++ Acc0,
     case Is of
-        [#b_set{op=succeeded,args=[Dst]}] ->
+        [#b_set{op={succeeded,_},args=[Dst]}] ->
             %% Defer the save sequence to the success block.
             {reverse(Acc, Is), Post};
         _ ->
@@ -622,7 +622,7 @@ bs_instrs([{L,#b_blk{is=Is0}=Blk}|Bs], CtxChain, Acc0) ->
 bs_instrs([], _, Acc) ->
     reverse(Acc).
 
-bs_instrs_is([#b_set{op=succeeded}=I|Is], CtxChain, Acc) ->
+bs_instrs_is([#b_set{op={succeeded,_}}=I|Is], CtxChain, Acc) ->
     %% This instruction refers to a specific operation, so we must not
     %% substitute the context argument.
     bs_instrs_is(Is, CtxChain, [I | Acc]);
@@ -717,54 +717,6 @@ legacy_bs_is([I|Is], Last, IsYreg, Count, Copies, Acc) ->
 legacy_bs_is([], _Last, _IsYreg, Count, Copies, Acc) ->
     {reverse(Acc),Count,Copies}.
 
-%% exception_trampolines(St0) -> St.
-%%
-%% Removes the "exception trampolines" that were added to prevent exceptions
-%% from being optimized away.
-
-exception_trampolines(#st{ssa=Blocks0}=St) ->
-    RPO = reverse(beam_ssa:rpo(Blocks0)),
-    Blocks = et_1(RPO, #{}, #{}, Blocks0),
-    St#st{ssa=Blocks}.
-
-et_1([L | Ls], Trampolines, Exceptions, Blocks) ->
-    #{ L := #b_blk{is=Is,last=Last0}=Block0 } = Blocks,
-    case {Is, Last0} of
-        {[#b_set{op=exception_trampoline,args=[Arg]}], #b_br{succ=Succ}} ->
-            et_1(Ls,
-                 Trampolines#{ L => Succ },
-                 Exceptions#{ L => Arg },
-                 maps:remove(L, Blocks));
-        {_, #b_br{succ=Same,fail=Same}} when Same =:= ?EXCEPTION_BLOCK ->
-            %% The exception block is just a marker saying that we should raise
-            %% an exception (= {f,0}) instead of jumping to a particular fail
-            %% block. Since it's not a reachable block we can't allow
-            %% unconditional jumps to it except through a trampoline.
-            error({illegal_jump_to_exception_block, L});
-        {_, #b_br{succ=Same,fail=Same}}
-          when map_get(Same, Trampolines) =:= ?EXCEPTION_BLOCK ->
-            %% This block always fails at runtime (and we are not in a
-            %% try/catch); rewrite the terminator to a return.
-            Last = #b_ret{arg=map_get(Same, Exceptions)},
-            Block = Block0#b_blk{last=Last},
-            et_1(Ls, Trampolines, Exceptions, Blocks#{ L := Block });
-        {_, #b_br{succ=Succ0,fail=Fail0}} ->
-            Succ = maps:get(Succ0, Trampolines, Succ0),
-            Fail = maps:get(Fail0, Trampolines, Fail0),
-            if
-                Succ =/= Succ0; Fail =/= Fail0 ->
-                    Last = Last0#b_br{succ=Succ,fail=Fail},
-                    Block = Block0#b_blk{last=Last},
-                    et_1(Ls, Trampolines, Exceptions, Blocks#{ L := Block });
-                Succ =:= Succ0, Fail =:= Fail0 ->
-                    et_1(Ls, Trampolines, Exceptions, Blocks)
-            end;
-        {_, _} ->
-             et_1(Ls, Trampolines, Exceptions, Blocks)
-    end;
-et_1([], _Trampolines, _Exceptions, Blocks) ->
-    Blocks.
-
 %% sanitize(St0) -> St.
 %%  Remove constructs that can cause problems later:
 %%
@@ -780,12 +732,12 @@ sanitize(#st{ssa=Blocks0,cnt=Count0}=St) ->
     St#st{ssa=Blocks,cnt=Count}.
 
 sanitize([L|Ls], Count0, Blocks0, Values0) ->
-    #b_blk{is=Is0} = Blk0 = map_get(L, Blocks0),
-    case sanitize_is(Is0, Count0, Values0, false, []) of
+    #b_blk{is=Is0,last=Last0} = Blk0 = map_get(L, Blocks0),
+    case sanitize_is(Is0, Last0, Count0, Values0, false, []) of
         no_change ->
             sanitize(Ls, Count0, Blocks0, Values0);
-        {Is,Count,Values} ->
-            Blk = Blk0#b_blk{is=Is},
+        {Is,Last,Count,Values} ->
+            Blk = Blk0#b_blk{is=Is,last=Last},
             Blocks = Blocks0#{L:=Blk},
             sanitize(Ls, Count, Blocks, Values)
     end;
@@ -806,49 +758,83 @@ sanitize([], Count, Blocks0, Values) ->
      end,Count}.
 
 sanitize_is([#b_set{op=get_map_element,args=Args0}=I0|Is],
-            Count0, Values, Changed, Acc) ->
+            Last, Count0, Values, Changed, Acc) ->
     case sanitize_args(Args0, Values) of
         [#b_literal{}=Map,Key] ->
             %% Bind the literal map to a variable.
             {MapVar,Count} = new_var('@ssa_map', Count0),
             I = I0#b_set{args=[MapVar,Key]},
             Copy = #b_set{op=copy,dst=MapVar,args=[Map]},
-            sanitize_is(Is, Count, Values, true, [I,Copy|Acc]);
+            sanitize_is(Is, Last, Count, Values, true, [I,Copy|Acc]);
         [_,_]=Args0 ->
-            sanitize_is(Is, Count0, Values, Changed, [I0|Acc]);
+            sanitize_is(Is, Last, Count0, Values, Changed, [I0|Acc]);
         [_,_]=Args ->
             I = I0#b_set{args=Args},
-            sanitize_is(Is, Count0, Values, Changed, [I|Acc])
+            sanitize_is(Is, Last, Count0, Values, true, [I|Acc])
+    end;
+sanitize_is([#b_set{op={succeeded,Kind},dst=Dst,args=[Arg0]}=I0],
+            #b_br{bool=Dst}=Last, Count, Values, _Changed, Acc) ->
+    %% We no longer need to distinguish between guard and body checks, so we'll
+    %% rewrite this as a plain 'succeeded'.
+    true = Kind =:= guard orelse Kind =:= body, %Assertion.
+    case sanitize_arg(Arg0, Values) of
+        #b_var{}=Arg ->
+            I = I0#b_set{op=succeeded,args=[Arg]},
+            {reverse(Acc, [I]), Last, Count, Values};
+        #b_literal{} ->
+            Value = #b_literal{val=true},
+            {reverse(Acc), Last, Count, Values#{ Dst => Value }}
+    end;
+sanitize_is([#b_set{op={succeeded,Kind},args=[Arg0]} | Is],
+            Last, Count, Values, _Changed, Acc) ->
+    %% We're no longer branching on this instruction and can safely remove it.
+    [] = Is, #b_br{succ=Same,fail=Same} = Last, %Assertion.
+    if
+        Same =:= ?EXCEPTION_BLOCK ->
+            %% The checked instruction always fails at runtime and we're not
+            %% in a try/catch; rewrite the terminator to a return.
+            body = Kind,                        %Assertion.
+            Arg = sanitize_arg(Arg0, Values),
+            sanitize_is(Is, #b_ret{arg=Arg}, Count, Values, true, Acc);
+        Same =/= ?EXCEPTION_BLOCK ->
+            %% We either always succeed, or always fail to somewhere other than
+            %% the exception block.
+            true = Kind =:= guard orelse Kind =:= body, %Assertion.
+            sanitize_is(Is, Last, Count, Values, true, Acc)
     end;
 sanitize_is([#b_set{op=Op,dst=Dst,args=Args0}=I0|Is0],
-            Count, Values, Changed0, Acc) ->
+            Last, Count, Values, Changed0, Acc) ->
     Args = sanitize_args(Args0, Values),
     case sanitize_instr(Op, Args, I0) of
         {value,Value0} ->
             Value = #b_literal{val=Value0},
-            sanitize_is(Is0, Count, Values#{Dst=>Value}, true, Acc);
+            sanitize_is(Is0, Last, Count, Values#{Dst=>Value}, true, Acc);
         {ok,I} ->
-            sanitize_is(Is0, Count, Values, true, [I|Acc]);
+            sanitize_is(Is0, Last, Count, Values, true, [I|Acc]);
         ok ->
             I = I0#b_set{args=Args},
             Changed = Changed0 orelse Args =/= Args0,
-            sanitize_is(Is0, Count, Values, Changed, [I|Acc])
+            sanitize_is(Is0, Last, Count, Values, Changed, [I|Acc])
     end;
-sanitize_is([], Count, Values, Changed, Acc) ->
+sanitize_is([], Last, Count, Values, Changed, Acc) ->
     case Changed of
         true ->
-            {reverse(Acc),Count,Values};
+            {reverse(Acc), Last, Count, Values};
         false ->
             no_change
     end.
 
 sanitize_args(Args, Values) ->
-    map(fun(Var) ->
-                case Values of
-                    #{Var:=New} -> New;
-                    #{} -> Var
-                end
-        end, Args).
+    [sanitize_arg(Arg, Values) || Arg <- Args].
+
+sanitize_arg(#b_var{}=Var, Values) ->
+    case Values of
+        #{Var:=New} -> New;
+        #{} -> Var
+    end;
+sanitize_arg(Arg, _Values) ->
+    Arg.
+
 
 sanitize_instr({bif,Bif}, [#b_literal{val=Lit}], _I) ->
     case erl_bifs:is_pure(erlang, Bif, 1) of
@@ -908,9 +894,8 @@ sanitize_instr(bs_init, [#b_literal{},_,#b_literal{val=Sz}|_], I0) ->
         is_integer(Sz), Sz >= 0 -> ok;
         true -> {ok,sanitize_badarg(I0)}
     end;
-sanitize_instr(succeeded, [#b_literal{}], _I) ->
-    {value,true};
-sanitize_instr(_, _, _) -> ok.
+sanitize_instr(_, _, _) ->
+    ok.
 
 sanitize_badarg(I) ->
     Func = #b_remote{mod=#b_literal{val=erlang},
@@ -1805,11 +1790,17 @@ find_yregs_terminator(Terminator, #dk{k=Killed}, Yregs0) ->
 %%%
 %%% 0:
 %%%   Res = call local literal foo/1, literal 42
+%%%   @ssa_bool:1 = succeeded:body Res
+%%%   br @ssa_bool:1, label 2, label 1
+%%% 2:
 %%%   _1 = bif:node Pid
-%%%   @ssa_bool = succeeded _1
-%%%   br @ssa_bool, label 3, label 1
+%%%   @ssa_bool:2 = succeeded:body _1
+%%%   br @ssa_bool:2, label 3, label 1
 %%% 3:
-%%%   @ssa_ignored = call local literal bar/0
+%%%   _2 = call local literal bar/0
+%%%   @ssa_bool:3 = succeeded:body _2
+%%%   br @ssa_bool:3, label 4, label 1
+%%% 4:
 %%%   ret Res
 %%%
 %%% It can be seen that the variables Pid and Res must be saved in Y
@@ -1820,12 +1811,17 @@ find_yregs_terminator(Terminator, #dk{k=Killed}, Yregs0) ->
 %%% 0:
 %%%   Pid:4 = copy Pid
 %%%   Res = call local literal foo/1, literal 42
+%%%   @ssa_bool:1 = succeeded:body Res
+%%%   br @ssa_bool:1, label 2, label 1
+%%% 2:
 %%%   _1 = bif:node Pid:4
-%%%   @ssa_bool = succeeded _1
-%%%   br @ssa_bool, label 3, label 1
-%%%
+%%%   @ssa_bool:2 = succeeded:body _1
+%%%   br @ssa_bool:2, label 3, label 1
 %%% 3:
-%%%   @ssa_ignored = call local literal bar/0
+%%%   _2 = call local literal bar/0
+%%%   @ssa_bool:3 = succeeded:body _2
+%%%   br @ssa_bool:3, label 4, label 1
+%%% 4:
 %%%   ret Res
 %%%
 %%% The Res and Pid:4 variables must be assigned to different Y registers
@@ -1835,13 +1831,18 @@ find_yregs_terminator(Terminator, #dk{k=Killed}, Yregs0) ->
 %%% 0:
 %%%   Pid:4 = copy Pid
 %%%   Res:6 = call local literal foo/1, literal 42
+%%%   @ssa_bool:1 = succeeded:body Res:6
+%%%   br @ssa_bool:1, label 2, label 1
+%%% 2:
 %%%   _1 = bif:node Pid:4
-%%%   @ssa_bool = succeeded _1
-%%%   br @ssa_bool, label 3, label 1
-%%%
+%%%   @ssa_bool:2 = succeeded:body _1
+%%%   br @ssa_bool:2, label 3, label 1
 %%% 3:
 %%%   Res = copy Res:6
-%%%   @ssa_ignored = call local literal bar/0
+%%%   _2 = call local literal bar/0
+%%%   @ssa_bool:3 = succeeded:body _2
+%%%   br @ssa_bool:3, label 4, label 1
+%%% 4:
 %%%   ret Res
 %%%
 %%% The new variable Res:6 is used to capture the return value from the call.
diff --git a/lib/compiler/src/beam_ssa_type.erl b/lib/compiler/src/beam_ssa_type.erl
index e0baadccb2..d85d5f9eff 100644
--- a/lib/compiler/src/beam_ssa_type.erl
+++ b/lib/compiler/src/beam_ssa_type.erl
@@ -644,20 +644,31 @@ simplify(#b_set{op=phi,dst=Dst,args=Args0}=I0, Ts0, Ds0, Ls, Sub) ->
             Ds = Ds0#{Dst=>I},
             {I, Ts, Ds}
     end;
-simplify(#b_set{op=succeeded,dst=Dst}=I0, Ts0, Ds0, _Ls, Sub) ->
-    case will_succeed(I0, Ts0, Ds0, Sub) of
-        yes ->
+simplify(#b_set{op={succeeded,Kind},args=[Arg],dst=Dst}=I0,
+         Ts0, Ds0, _Ls, Sub) ->
+    Type = case will_succeed(I0, Ts0, Ds0, Sub) of
+               yes -> beam_types:make_atom(true);
+               no -> beam_types:make_atom(false);
+               maybe -> beam_types:make_boolean()
+           end,
+    case Type of
+        #t_atom{elements=[true]} ->
+            %% The checked operation always succeeds, so it's safe to remove
+            %% this instruction regardless of whether we're in a guard or not.
             Lit = #b_literal{val=true},
             Sub#{ Dst => Lit };
-        no ->
+        #t_atom{elements=[false]} when Kind =:= guard ->
+            %% Failing operations are only safe to remove in guards.
             Lit = #b_literal{val=false},
             Sub#{ Dst => Lit };
-        maybe ->
+        _ ->
+            true = is_map_key(Arg, Ds0),        %Assertion.
+
             %% Note that we never simplify args; this instruction is specific
             %% to the operation being checked, and simplifying could break that
             %% connection.
             I = beam_ssa:normalize(I0),
-            Ts = Ts0#{ Dst => beam_types:make_boolean() },
+            Ts = Ts0#{ Dst => Type },
             Ds = Ds0#{ Dst => I },
             {I, Ts, Ds}
     end;
@@ -1804,7 +1815,7 @@ infer_types_switch(V, Lit, Ts0, IsTempVar, Ds) ->
 ts_remove_var(_V, none) -> none;
 ts_remove_var(V, Ts) -> maps:remove(V, Ts).
 
-infer_type(succeeded, [#b_var{}=Src], Ts, Ds) ->
+infer_type({succeeded,_}, [#b_var{}=Src], Ts, Ds) ->
     #b_set{op=Op,args=Args} = maps:get(Src, Ds),
     infer_success_type(Op, Args, Ts, Ds);