Merge branch 'bjorn/erts/max_heap_size/25/OTP-18463' into maint

* bjorn/erts/max_heap_size/25/OTP-18463:
  Exit process immediately when the max heap size is exceeded

6 files changed, 136 insertions, 1 deletions

diff --git a/erts/emulator/beam/emu/bs_instrs.tab b/erts/emulator/beam/emu/bs_instrs.tab
index 846e4b6f2c..b5fa518312 100644
--- a/erts/emulator/beam/emu/bs_instrs.tab
+++ b/erts/emulator/beam/emu/bs_instrs.tab
@@ -132,6 +132,7 @@ TEST_BIN_VHEAP(VNh, Nh, Live) {
         ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
         PROCESS_MAIN_CHK_LOCKS(c_p);
         SWAPIN;
+        $MAYBE_EXIT_AFTER_GC();
     }
     HEAP_SPACE_VERIFIED(need);
 }
diff --git a/erts/emulator/beam/emu/instrs.tab b/erts/emulator/beam/emu/instrs.tab
index 2c39d885ef..0ca0d71a2d 100644
--- a/erts/emulator/beam/emu/instrs.tab
+++ b/erts/emulator/beam/emu/instrs.tab
@@ -1124,6 +1124,7 @@ catch_end(Y) {
                 ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
                 PROCESS_MAIN_CHK_LOCKS(c_p);
                 SWAPIN;
+                $MAYBE_EXIT_AFTER_GC();
             }
             r(0) = TUPLE2(HTOP, am_EXIT, x(1));
             HTOP += 3;
diff --git a/erts/emulator/beam/emu/macros.tab b/erts/emulator/beam/emu/macros.tab
index cef41e89f7..b65e01553a 100644
--- a/erts/emulator/beam/emu/macros.tab
+++ b/erts/emulator/beam/emu/macros.tab
@@ -74,6 +74,12 @@ GC_SWAPOUT() {
     c_p->i = I;
 }
 
+MAYBE_EXIT_AFTER_GC() {
+    if (ERTS_PSFLG_EXITING & erts_atomic32_read_nob(&c_p->state)) {
+        goto context_switch3;
+    }
+}
+
 GC_TEST(Ns, Nh, Live) {
     Uint need = $Nh + $Ns;
     if (ERTS_UNLIKELY((E - HTOP) < (need + S_RESERVED))) {
@@ -83,6 +89,7 @@ GC_TEST(Ns, Nh, Live) {
        ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
        PROCESS_MAIN_CHK_LOCKS(c_p);
        SWAPIN;
+       $MAYBE_EXIT_AFTER_GC();
     }
     HEAP_SPACE_VERIFIED($Nh);
 }
@@ -98,6 +105,7 @@ GC_TEST_PRESERVE(NeedHeap, Live, PreserveTerm) {
        PROCESS_MAIN_CHK_LOCKS(c_p);
        $PreserveTerm = reg[$Live];
        SWAPIN;
+       $MAYBE_EXIT_AFTER_GC();
     }
     HEAP_SPACE_VERIFIED($NeedHeap);
 }
diff --git a/erts/emulator/beam/jit/arm/beam_asm_global.cpp b/erts/emulator/beam/jit/arm/beam_asm_global.cpp
index 6185c70fcb..b4655b6c19 100644
--- a/erts/emulator/beam/jit/arm/beam_asm_global.cpp
+++ b/erts/emulator/beam/jit/arm/beam_asm_global.cpp
@@ -120,6 +120,10 @@ void BeamGlobalAssembler::emit_garbage_collect() {
     emit_leave_runtime<Update::eStack | Update::eHeap | Update::eXRegs>();
     emit_leave_runtime_frame();
 
+    a.ldr(TMP1.w(), arm::Mem(c_p, offsetof(Process, state.value)));
+    a.tst(TMP1, imm(ERTS_PSFLG_EXITING));
+    a.b_ne(labels[do_schedule]);
+
     a.ret(a64::x30);
 }
 
diff --git a/erts/emulator/beam/jit/x86/beam_asm_global.cpp b/erts/emulator/beam/jit/x86/beam_asm_global.cpp
index 59eaf057f3..3e588d3d45 100644
--- a/erts/emulator/beam/jit/x86/beam_asm_global.cpp
+++ b/erts/emulator/beam/jit/x86/beam_asm_global.cpp
@@ -95,6 +95,8 @@ BeamGlobalAssembler::BeamGlobalAssembler(JitAllocator *allocator)
 /* ARG3 = (HTOP + S_RESERVED + bytes needed) !!
  * ARG4 = Live registers */
 void BeamGlobalAssembler::emit_garbage_collect() {
+    Label exiting = a.newLabel();
+
     emit_enter_frame();
 
     /* Convert ARG3 to words needed and move it to the correct argument slot.
@@ -126,9 +128,21 @@ void BeamGlobalAssembler::emit_garbage_collect() {
     a.sub(FCALLS, RET);
 
     emit_leave_runtime<Update::eStack | Update::eHeap>();
-    emit_leave_frame();
 
+#ifdef WIN32
+    a.mov(ARG1d, x86::dword_ptr(c_p, offsetof(Process, state.value)));
+#else
+    a.mov(ARG1d, x86::dword_ptr(c_p, offsetof(Process, state.counter)));
+#endif
+    a.test(ARG1d, imm(ERTS_PSFLG_EXITING));
+    a.short_().jne(exiting);
+
+    emit_leave_frame();
     a.ret();
+
+    a.bind(exiting);
+    emit_unwind_frame();
+    a.jmp(labels[do_schedule]);
 }
 
 /* Handles trapping to exports from C code, setting registers up in the same
diff --git a/erts/emulator/test/process_SUITE.erl b/erts/emulator/test/process_SUITE.erl
index 9ab2366c58..89d37d4f3e 100644
--- a/erts/emulator/test/process_SUITE.erl
+++ b/erts/emulator/test/process_SUITE.erl
@@ -60,6 +60,7 @@
          process_flag_fullsweep_after/1, process_flag_heap_size/1,
          command_line_max_heap_size/1,
 	 spawn_opt_heap_size/1, spawn_opt_max_heap_size/1,
+         more_spawn_opt_max_heap_size/1,
 	 processes_large_tab/1, processes_default_tab/1, processes_small_tab/1,
 	 processes_this_tab/1, processes_apply_trap/1,
 	 processes_last_call_trap/1, processes_gc_trap/1,
@@ -118,6 +119,7 @@ all() ->
      process_flag_fullsweep_after, process_flag_heap_size,
      command_line_max_heap_size,
      spawn_opt_heap_size, spawn_opt_max_heap_size,
+     more_spawn_opt_max_heap_size,
      spawn_huge_arglist,
      otp_6237,
      {group, spawn_request},
@@ -2713,6 +2715,111 @@ flush() ->
             ok
     end.
 
+%% Make sure that when maximum allowed heap size is exceeded, the
+%% process will actually terminate.
+%%
+%% Despite the timetrap and limit of number of iterations, bugs
+%% provoked by the test case can cause the runtime system to hang in
+%% this test case.
+more_spawn_opt_max_heap_size(_Config) ->
+    ct:timetrap({minutes,1}),
+    Funs = [fun build_and_bif/0,
+            fun build_bin_and_bif/0,
+            fun build_and_recv_timeout/0,
+            fun build_and_recv_msg/0,
+            fun bif_and_recv_timeout/0,
+            fun bif_and_recv_msg/0
+           ],
+    _ = [begin
+             {Pid,Ref} = spawn_opt(F, [{max_heap_size,
+                                        #{size => 233, kill => true,
+                                          error_logger => false}},
+                                       monitor]),
+             io:format("~p ~p\n", [Pid,F]),
+             receive
+                 {'DOWN',Ref,process,Pid,Reason} ->
+                     killed = Reason
+             end
+         end || F <- Funs],
+    ok.
+
+%% This number should be greater than the default heap size.
+-define(MANY_ITERATIONS, 10_000).
+
+build_and_bif() ->
+    build_and_bif(?MANY_ITERATIONS, []).
+
+build_and_bif(0, Acc0) ->
+    Acc0;
+build_and_bif(N, Acc0) ->
+    Acc = [0|Acc0],
+    _ = erlang:crc32(Acc),
+    build_and_bif(N-1, Acc).
+
+build_bin_and_bif() ->
+    build_bin_and_bif(?MANY_ITERATIONS, <<>>).
+
+build_bin_and_bif(0, Acc0) ->
+    Acc0;
+build_bin_and_bif(N, Acc0) ->
+    Acc = <<0, Acc0/binary>>,
+    _ = erlang:crc32(Acc),
+    build_bin_and_bif(N-1, Acc).
+
+build_and_recv_timeout() ->
+    build_and_recv_timeout(?MANY_ITERATIONS, []).
+
+build_and_recv_timeout(0, Acc0) ->
+    Acc0;
+build_and_recv_timeout(N, Acc0) ->
+    Acc = [0|Acc0],
+    receive
+    after 1 ->
+            ok
+    end,
+    build_and_recv_timeout(N-1, Acc).
+
+build_and_recv_msg() ->
+    build_and_recv_msg(?MANY_ITERATIONS, []).
+
+build_and_recv_msg(0, Acc0) ->
+    Acc0;
+build_and_recv_msg(N, Acc0) ->
+    Acc = [0|Acc0],
+    receive
+        _ ->
+            ok
+    after 0 ->
+            ok
+    end,
+    build_and_recv_msg(N-1, Acc).
+
+bif_and_recv_timeout() ->
+    Bin = <<0:?MANY_ITERATIONS/unit:8>>,
+    bif_and_recv_timeout(Bin).
+
+bif_and_recv_timeout(Bin) ->
+    List = binary_to_list(Bin),
+    receive
+    after 1 ->
+            ok
+    end,
+    List.
+
+bif_and_recv_msg() ->
+    Bin = <<0:?MANY_ITERATIONS/unit:8>>,
+    bif_and_recv_msg(Bin).
+
+bif_and_recv_msg(Bin) ->
+    List = binary_to_list(Bin),
+    receive
+        _ ->
+            ok
+    after 0 ->
+            ok
+    end,
+    List.
+
 %% error_logger report handler proxy
 init(Pid) ->
     {ok, Pid}.