Replace error with PANIC for callbacks from JIT-compiled code.

2 files changed, 23 insertions, 3 deletions

diff --git a/doc/ext_ffi_semantics.html b/doc/ext_ffi_semantics.html
index f8da1e60..ab02f2b8 100644
--- a/doc/ext_ffi_semantics.html
+++ b/doc/ext_ffi_semantics.html
@@ -934,6 +934,22 @@ advisable in general. Do this only if you know the C function, that
 called the callback, copes with the forced stack unwinding and doesn't
 leak resources.
 </p>
+<p>
+One thing that's not allowed, is to let an FFI call into a C&nbsp;function
+get JIT-compiled, which in turn calls a callback, calling into Lua again.
+Usually this attempt is caught by the interpreter first and the
+C&nbsp;function is blacklisted for compilation.
+</p>
+<p>
+However, this heuristic may fail under specific circumstances: e.g. a
+message polling function might not run Lua callbacks right away and the call
+gets JIT-compiled. If it later happens to call back into Lua, you'll get a
+VM PANIC with the message <tt>"bad callback"</tt>. Then you'll need to
+manually turn off JIT-compilation with
+<a href="ext_jit.html#jit_onoff_func"><tt>jit.off()</tt></a> for the
+surrounding Lua function that invokes such a message polling function (or
+similar).
+</p>
 
 <h3 id="callback_resources">Callback resource handling</h3>
 <p>
diff --git a/src/lj_ccallback.c b/src/lj_ccallback.c
index 430643ee..e1d03fcf 100644
--- a/src/lj_ccallback.c
+++ b/src/lj_ccallback.c
@@ -527,10 +527,14 @@ static void callback_conv_result(CTState *cts, lua_State *L, TValue *o)
 lua_State * LJ_FASTCALL lj_ccallback_enter(CTState *cts, void *cf)
 {
   lua_State *L = cts->L;
+  global_State *g = cts->g;
   lua_assert(L != NULL);
-  if (gcref(cts->g->jit_L))
-    lj_err_caller(gco2th(gcref(cts->g->jit_L)), LJ_ERR_FFI_BADCBACK);
-  lj_trace_abort(cts->g);  /* Never record across callback. */
+  if (gcref(g->jit_L)) {
+    setstrV(L, L->top++, lj_err_str(L, LJ_ERR_FFI_BADCBACK));
+    if (g->panic) g->panic(L);
+    exit(EXIT_FAILURE);
+  }
+  lj_trace_abort(g);  /* Never record across callback. */
   /* Setup C frame. */
   cframe_prev(cf) = L->cframe;
   setcframe_L(cf, L);