wt needs to use its own implementation of rwlocks with

reader preference, at least where system rwlocks are fair.

include/my_global.h:
  wt uses mutex-based rwlock implementation unless on linux
include/waiting_threads.h:
  mutex-based rwlock implementation with reader preference
mysys/thr_rwlock.c:
  revert the change. make my_rw_locks fair
mysys/waiting_threads.c:
  mutex-based rwlock implementation with reader preference.
  convert complex multi-line macros to static functions

2 files changed, 27 insertions, 3 deletions

diff --git a/include/my_global.h b/include/my_global.h
index f2d5a0e5013..d3f99c9147c 100644
--- a/include/my_global.h
+++ b/include/my_global.h
@@ -1524,6 +1524,15 @@ inline void  operator delete[](void*, void*) { /* Do nothing */ }
 */
 #ifdef TARGET_OS_LINUX
 #define NEED_EXPLICIT_SYNC_DIR 1
+#else
+/*
+  On linux default rwlock scheduling policy is good enough for
+  waiting_threads.c, on other systems use our special implementation
+  (which is slower).
+
+  QQ perhaps this should be tested in configure ? how ?
+*/
+#define WT_RWLOCKS_USE_MUTEXES 1
 #endif
 
 #if !defined(__cplusplus) && !defined(bool)
diff --git a/include/waiting_threads.h b/include/waiting_threads.h
index 322b5972ec0..d0d5ffbd191 100644
--- a/include/waiting_threads.h
+++ b/include/waiting_threads.h
@@ -67,7 +67,6 @@ extern uint32    wt_success_stats;
        e.g. accessing a resource by thd->waiting_for is safe,
        a resource cannot be freed as there's a thread waiting for it
 */
-
 typedef struct st_wt_resource {
   WT_RESOURCE_ID  id;
   uint            waiter_count;
@@ -76,11 +75,27 @@ typedef struct st_wt_resource {
   pthread_mutex_t  *mutex;
 #endif
   /*
-    before the 'lock' all elements are mutable, after - immutable
-    in the sense that lf_hash_insert() won't memcpy() over them.
+    before the 'lock' all elements are mutable, after (and including) -
+    immutable in the sense that lf_hash_insert() won't memcpy() over them.
     See wt_init().
   */
+#ifdef WT_RWLOCKS_USE_MUTEXES
+  /*
+    we need a special rwlock-like 'lock' to allow readers bypass
+    waiting writers, otherwise readers can deadlock.
+    writer starvation is technically possible, but unlikely, because
+    the contention is expected to be low.
+  */
+  struct {
+    pthread_cond_t   cond;
+    pthread_mutex_t  mutex;
+    uint readers: 16;
+    uint pending_writers: 15;
+    uint write_locked: 1;
+  } lock;
+#else
   rw_lock_t lock;
+#endif
   pthread_cond_t   cond;
   DYNAMIC_ARRAY    owners;
 } WT_RESOURCE;