YJIT: Fix large ISeq rejection (#7576)

We crashed in some edge cases due to the recent change to not compile
encoded iseqs that are larger than `u16::MAX`.

- Match the C signature of rb_yjit_constant_ic_update() and clamp down
  to `IseqIdx` size
- Return failure instead of panicking with `unwrap()` in codegen when
  the iseq is too large

Co-authored-by: Maxime Chevalier-Boisvert <maxime.chevalierboisvert@shopify.com>
Co-authored-by: Noah Gibbs <noah.gibbs@shopify.com>

1 files changed, 7 insertions, 2 deletions

diff --git a/yjit/src/codegen.rs b/yjit/src/codegen.rs
index 824415346e..fb760cfd6c 100644
--- a/yjit/src/codegen.rs
+++ b/yjit/src/codegen.rs
@@ -781,8 +781,13 @@ pub fn gen_single_block(
     // Instruction sequence to compile
     let iseq = blockid.iseq;
     let iseq_size = unsafe { get_iseq_encoded_size(iseq) };
-    let iseq_size: u16 = iseq_size.try_into().unwrap();
-    let mut insn_idx: u16 = blockid.idx;
+    let iseq_size: IseqIdx = if let Ok(size) = iseq_size.try_into() {
+        size
+    } else {
+        // ISeq too large to compile
+        return Err(());
+    };
+    let mut insn_idx: IseqIdx = blockid.idx;
 
     // Initialize a JIT state object
     let mut jit = JITState::new(blockid, ctx.clone(), cb.get_write_ptr(), ec);