Merge branch 'john/compiler/fix-type-oscillation/OTP-16745/ERL-1289' into maint-23

* john/compiler/fix-type-oscillation/OTP-16745/ERL-1289:
  beam_ssa_type: Fix endless type oscillation

2 files changed, 44 insertions, 8 deletions

diff --git a/lib/compiler/src/beam_ssa_type.erl b/lib/compiler/src/beam_ssa_type.erl
index 9d0c83f7b7..d40e7f3fc0 100644
--- a/lib/compiler/src/beam_ssa_type.erl
+++ b/lib/compiler/src/beam_ssa_type.erl
@@ -53,7 +53,7 @@
 %% narrow a type down, it could push us over the edge and collapse all entries,
 %% possibly widening the return type and breaking optimizations that were based
 %% on the earlier (narrower) types.
--define(RETURN_LIMIT, 100).
+-define(RETURN_LIMIT, 30).
 
 %% Constants common to all subpasses.
 -record(metadata,
@@ -1486,14 +1486,18 @@ ust_limited_1([{SuccArgs, SuccRet}], CallArgs, CallRet) ->
     NewType = beam_types:join(SuccRet, CallRet),
     [{NewTypes, NewType}].
 
-%% Adds a new success type, collapsing it with entries that have the same
-%% return type to keep the list short.
+%% Adds a new success type. Note that we no longer try to keep the list short
+%% by combining entries with the same return type, as that can make effective
+%% return types less specific as analysis goes on, which may cause endless
+%% loops or render previous optimizations unsafe.
+%%
+%% See beam_type_SUITE:success_type_oscillation/1 for more details.
 ust_unlimited(SuccTypes, _CallArgs, none) ->
     %% 'none' is implied since functions can always fail.
     SuccTypes;
-ust_unlimited([{SuccArgs, Same} | SuccTypes], CallArgs, Same) ->
-    NewArgs = parallel_join(SuccArgs, CallArgs),
-    [{NewArgs, Same} | SuccTypes];
+ust_unlimited([{SameArgs, SameType} | _]=SuccTypes, SameArgs, SameType) ->
+    %% Already covered, return as-is.
+    SuccTypes;
 ust_unlimited([SuccType | SuccTypes], CallArgs, CallRet) ->
     [SuccType | ust_unlimited(SuccTypes, CallArgs, CallRet)];
 ust_unlimited([], CallArgs, CallRet) ->
diff --git a/lib/compiler/test/beam_type_SUITE.erl b/lib/compiler/test/beam_type_SUITE.erl
index f674ee119e..5189afde58 100644
--- a/lib/compiler/test/beam_type_SUITE.erl
+++ b/lib/compiler/test/beam_type_SUITE.erl
@@ -25,7 +25,7 @@
 	 cons/1,tuple/1,record_float/1,binary_float/1,float_compare/1,
 	 arity_checks/1,elixir_binaries/1,find_best/1,
          test_size/1,cover_lists_functions/1,list_append/1,bad_binary_unit/1,
-         none_argument/1]).
+         none_argument/1,success_type_oscillation/1]).
 
 suite() -> [{ct_hooks,[ts_install_cth]}].
 
@@ -51,7 +51,8 @@ groups() ->
        cover_lists_functions,
        list_append,
        bad_binary_unit,
-       none_argument
+       none_argument,
+       success_type_oscillation
       ]}].
 
 init_per_suite(Config) ->
@@ -564,5 +565,36 @@ uncompress(CompressedBinary) ->
     %% did not handle properly.
     zlib:uncompress(CompressedBinary).
 
+%% ERL-1289: The compiler could enter an endless loop when a return/argument
+%% type pairing was joined with another.
+%%
+%% While this always resulted in correct success types, the joined argument
+%% types could now cover more cases than they did before, making the effective
+%% return type less specific. When a function affected by this was analyzed
+%% again its success typing could become more specific again and start the
+%% process anew.
+success_type_oscillation(_Config) ->
+    Base = {a, []},
+
+    Base = sto_1(id(case_1_1)),
+    Base = sto_1(id(case_2_1)),
+    {b, [Base]} = sto_1(id(case_2_2)),
+
+    ok.
+
+sto_1(case_1_1) -> {a, []};
+sto_1(case_1_2) -> {a, []};
+sto_1(case_2_1) -> sto_1(case_1_1);
+sto_1(case_2_2) -> {b, [sto_1(case_1_1)]};
+sto_1(case_2_3) -> {b, [sto_1(case_1_1)]};
+sto_1(case_2_4) -> {b, [sto_1(case_1_2)]};
+sto_1(case_3_1) -> {b, [sto_1(case_2_1)]};
+sto_1(case_3_2) -> {b, [sto_1(case_2_2)]};
+sto_1(case_3_3) -> {b, [sto_1(case_2_3)]};
+sto_1(case_3_4) -> {b, [sto_1(case_2_4)]};
+sto_1(case_4_1) -> {b, [sto_1(case_3_1)]};
+sto_1(case_4_2) -> {b, [sto_1(case_3_2)]};
+sto_1(step_4_3) -> {b, [sto_1(case_3_3)]}.
+
 id(I) ->
     I.