YJIT: fix CI issue reported by Koichi caused by small stack patch (#7442)

Includes small reproduction produced by Kokubun.

http://ci.rvm.jp/results/trunk-yjit@ruby-sp2-docker

2 files changed, 35 insertions, 0 deletions

diff --git a/bootstraptest/test_yjit.rb b/bootstraptest/test_yjit.rb
index b064bcdde7..13da1e35fb 100644
--- a/bootstraptest/test_yjit.rb
+++ b/bootstraptest/test_yjit.rb
@@ -3605,3 +3605,30 @@ assert_equal 'Hello World', %q{
   end
   bar
 }
+
+# Regression: this creates a temp stack with > 127 elements
+assert_normal_exit %q{
+  def foo(a)
+    [
+      a, a, a, a, a, a, a, a, a, a,
+      a, a, a, a, a, a, a, a, a, a,
+      a, a, a, a, a, a, a, a, a, a,
+      a, a, a, a, a, a, a, a, a, a,
+      a, a, a, a, a, a, a, a, a, a,
+      a, a, a, a, a, a, a, a, a, a,
+      a, a, a, a, a, a, a, a, a, a,
+      a, a, a, a, a, a, a, a, a, a,
+      a, a, a, a, a, a, a, a, a, a,
+      a, a, a, a, a, a, a, a, a, a,
+      a, a, a, a, a, a, a, a, a, a,
+      a, a, a, a, a, a, a, a, a, a,
+      a, a, a, a, a, a, a, a,
+    ]
+  end
+
+  def entry
+    foo(1)
+  end
+
+  entry
+}
diff --git a/yjit/src/codegen.rs b/yjit/src/codegen.rs
index 6409761625..c76d827eaa 100644
--- a/yjit/src/codegen.rs
+++ b/yjit/src/codegen.rs
@@ -5215,6 +5215,14 @@ fn gen_send_iseq(
         return CantCompile;
     }
 
+    // Reject ISEQs with very large temp stacks,
+    // this will allow us to use u8/i8 values to track stack_size and sp_offset
+    let stack_max = unsafe { rb_get_iseq_body_stack_max(iseq) };
+    if stack_max >= i8::MAX as u32 {
+        incr_counter!(iseq_stack_too_large);
+        return CantCompile;
+    }
+
     // No support for callees with these parameters yet as they require allocation
     // or complex handling.
     if unsafe { get_iseq_flags_has_post(iseq) } {