thread_pthread.c (ubf_select): avoid deadlock on contention

vm->gvl.lock can be held by another thread, we must not wait
on it when called by the MJIT worker thread when it migrates
work to another thread.  ubf_select is designed to do retrying
anyways, so it has no obligation to wake up a timer thread.

cf.  http://ci.rvm.jp/results/trunk-mjit-wait@silicon-docker/1437880

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@65465 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

1 files changed, 10 insertions, 4 deletions

diff --git a/thread_pthread.c b/thread_pthread.c
index a8bb4c7178..97d6eafa31 100644
--- a/thread_pthread.c
+++ b/thread_pthread.c
@@ -1342,11 +1342,17 @@ ubf_select(void *ptr)
      * in unblock_function_clear.
      */
     if (cur != vm->gvl.timer && cur != sigwait_th) {
-        rb_native_mutex_lock(&vm->gvl.lock);
-        if (!vm->gvl.timer) {
-            rb_thread_wakeup_timer_thread(-1);
+        /*
+         * Double-checked locking above was to prevent nested locking
+         * by the SAME thread.  We use trylock here to prevent deadlocks
+         * between DIFFERENT threads
+         */
+        if (native_mutex_trylock(&vm->gvl.lock) == 0) {
+            if (!vm->gvl.timer) {
+                rb_thread_wakeup_timer_thread(-1);
+            }
+            rb_native_mutex_unlock(&vm->gvl.lock);
         }
-        rb_native_mutex_unlock(&vm->gvl.lock);
     }
 
     ubf_wakeup_thread(th);