FUTEX_*_REQUEUE_PI support for non-x86 code

Add FUTEX_*_REQUEUE_PI support for the default C code and also add
implementations for s-390 and ppc.

8 files changed, 252 insertions, 42 deletions

diff --git a/nptl/ChangeLog b/nptl/ChangeLog
index bcc1660b4d..8fedcf4dd8 100644
--- a/nptl/ChangeLog
+++ b/nptl/ChangeLog
@@ -1,3 +1,33 @@
+2013-02-18  Siddhesh Poyarekar  <siddhesh@redhat.com>
+
+	[BZ #14920]
+	* pthreadP.h (USE_REQUEUE_PI): New macro to check if mutex is
+	PI-aware.
+	* pthread_cond_broadcast.c (__pthread_cond_broadcast): Use
+	PI-aware futex operations if available and mutex is PI-aware.
+	* pthread_cond_signal.c (__pthread_cond_signal): Likewise.
+	* nptl/pthread_cond_timedwait.c (__pthread_cond_timedwait):
+	Likewise.
+	* pthread_cond_wait.c (__condvar_cleanup): Adjust lock if
+	cancellation occurred just after futex returned successfully
+	from a PI operation with the mutex held.
+	(__pthread_cond_wait): Use PI-aware futex operations if
+	available and mutex is PI-aware.
+	* sysdeps/unix/sysv/linux/powerpc/lowlevellock.h
+	(FUTEX_WAIT_REQUEUE_PI): Define.
+	(FUTEX_CMP_REQUEUE_PI): Likewise.
+	(lll_futex_wait_requeue_pi): Likewise.
+	(lll_futex_timed_wait_requeue_pi): Likewise.
+	(lll_futex_cmp_requeue_pi): Likewise.
+	* nptl/sysdeps/unix/sysv/linux/s390/lowlevellock.h
+	(FUTEX_WAIT_REQUEUE_PI): Define.
+	(FUTEX_CMP_REQUEUE_PI): Likewise.
+	(lll_futex_wait_requeue_pi): Likewise.
+	(lll_futex_timed_wait_requeue_pi): Likewise.
+	(lll_futex_cmp_requeue_pi): Likewise.
+	* sysdeps/unix/sysv/linux/kernel-features.h: Define
+	__ASSUME_REQUEUE_PI for Linux version higher than 2.6.31.
+
 2013-02-04  Andreas Schwab  <schwab@suse.de>
 
 	[BZ #14142]
diff --git a/nptl/pthreadP.h b/nptl/pthreadP.h
index 993a79e250..d08b219b5d 100644
--- a/nptl/pthreadP.h
+++ b/nptl/pthreadP.h
@@ -577,4 +577,16 @@ extern void __wait_lookup_done (void) attribute_hidden;
 # define PTHREAD_STATIC_FN_REQUIRE(name) __asm (".globl " #name);
 #endif
 
+/* Test if the mutex is suitable for the FUTEX_WAIT_REQUEUE_PI operation.  */
+#if (defined lll_futex_wait_requeue_pi \
+     && defined __ASSUME_REQUEUE_PI)
+# define USE_REQUEUE_PI(mut) \
+   ((mut) && (mut) != (void *) ~0l \
+    && (((mut)->__data.__kind \
+	 & (PTHREAD_MUTEX_PRIO_INHERIT_NP | PTHREAD_MUTEX_ROBUST_NORMAL_NP)) \
+	== PTHREAD_MUTEX_PRIO_INHERIT_NP))
+#else
+# define USE_REQUEUE_PI(mut) 0
+#endif
+
 #endif	/* pthreadP.h */
diff --git a/nptl/pthread_cond_broadcast.c b/nptl/pthread_cond_broadcast.c
index 968ee03da7..0702ec0ec2 100644
--- a/nptl/pthread_cond_broadcast.c
+++ b/nptl/pthread_cond_broadcast.c
@@ -53,34 +53,37 @@ __pthread_cond_broadcast (cond)
       /* We are done.  */
       lll_unlock (cond->__data.__lock, pshared);
 
-      /* Do not use requeue for pshared condvars.  */
-      if (cond->__data.__mutex == (void *) ~0l)
-	goto wake_all;
-
       /* Wake everybody.  */
       pthread_mutex_t *mut = (pthread_mutex_t *) cond->__data.__mutex;
 
-      /* XXX: Kernel so far doesn't support requeue to PI futex.  */
-      /* XXX: Kernel so far can only requeue to the same type of futex,
-	 in this case private (we don't requeue for pshared condvars).  */
-      if (__builtin_expect (mut->__data.__kind
-			    & (PTHREAD_MUTEX_PRIO_INHERIT_NP
-			       | PTHREAD_MUTEX_PSHARED_BIT), 0))
+      /* Do not use requeue for pshared condvars.  */
+      if (mut == (void *) ~0l
+	  || PTHREAD_MUTEX_PSHARED (mut) & PTHREAD_MUTEX_PSHARED_BIT)
 	goto wake_all;
 
-      /* lll_futex_requeue returns 0 for success and non-zero
-	 for errors.  */
-      if (__builtin_expect (lll_futex_requeue (&cond->__data.__futex, 1,
-					       INT_MAX, &mut->__data.__lock,
-					       futex_val, LLL_PRIVATE), 0))
+#if (defined lll_futex_cmp_requeue_pi \
+     && defined __ASSUME_REQUEUE_PI)
+      int pi_flag = PTHREAD_MUTEX_PRIO_INHERIT_NP | PTHREAD_MUTEX_ROBUST_NP;
+      pi_flag &= mut->__data.__kind;
+
+      if (pi_flag == PTHREAD_MUTEX_PRIO_INHERIT_NP)
 	{
-	  /* The requeue functionality is not available.  */
-	wake_all:
-	  lll_futex_wake (&cond->__data.__futex, INT_MAX, pshared);
+	  if (lll_futex_cmp_requeue_pi (&cond->__data.__futex, 1, INT_MAX,
+					&mut->__data.__lock, futex_val,
+					LLL_PRIVATE) == 0)
+	    return 0;
 	}
-
-      /* That's all.  */
-      return 0;
+      else
+#endif
+	/* lll_futex_requeue returns 0 for success and non-zero
+	   for errors.  */
+	if (!__builtin_expect (lll_futex_requeue (&cond->__data.__futex, 1,
+						  INT_MAX, &mut->__data.__lock,
+						  futex_val, LLL_PRIVATE), 0))
+	  return 0;
+
+wake_all:
+      lll_futex_wake (&cond->__data.__futex, INT_MAX, pshared);
     }
 
   /* We are done.  */
diff --git a/nptl/pthread_cond_signal.c b/nptl/pthread_cond_signal.c
index 908a2acfaf..102d0b380c 100644
--- a/nptl/pthread_cond_signal.c
+++ b/nptl/pthread_cond_signal.c
@@ -47,12 +47,35 @@ __pthread_cond_signal (cond)
       ++cond->__data.__wakeup_seq;
       ++cond->__data.__futex;
 
-      /* Wake one.  */
-      if (! __builtin_expect (lll_futex_wake_unlock (&cond->__data.__futex, 1,
-						     1, &cond->__data.__lock,
-						     pshared), 0))
-	return 0;
+#if (defined lll_futex_cmp_requeue_pi \
+     && defined __ASSUME_REQUEUE_PI)
+      int pi_flag = PTHREAD_MUTEX_PRIO_INHERIT_NP | PTHREAD_MUTEX_ROBUST_NP;
+      pthread_mutex_t *mut = cond->__data.__mutex;
 
+      /* Do not use requeue for pshared condvars.  */
+      if (mut != (void *) ~0l)
+	pi_flag &= mut->__data.__kind;
+
+      if (__builtin_expect (pi_flag == PTHREAD_MUTEX_PRIO_INHERIT_NP, 0)
+	/* This can only really fail with a ENOSYS, since nobody can modify
+	   futex while we have the cond_lock.  */
+	  && lll_futex_cmp_requeue_pi (&cond->__data.__futex, 1, 0,
+				       &mut->__data.__lock,
+				       cond->__data.__futex, pshared) == 0)
+	{
+	  lll_unlock (cond->__data.__lock, pshared);
+	  return 0;
+	}
+      else
+#endif
+	/* Wake one.  */
+	if (! __builtin_expect (lll_futex_wake_unlock (&cond->__data.__futex,
+						       1, 1,
+						       &cond->__data.__lock,
+						       pshared), 0))
+	  return 0;
+
+      /* Fallback if neither of them work.  */
       lll_futex_wake (&cond->__data.__futex, 1, pshared);
     }
 
diff --git a/nptl/pthread_cond_timedwait.c b/nptl/pthread_cond_timedwait.c
index 0f52bd885c..0a2d092e6c 100644
--- a/nptl/pthread_cond_timedwait.c
+++ b/nptl/pthread_cond_timedwait.c
@@ -64,6 +64,11 @@ __pthread_cond_timedwait (cond, mutex, abstime)
   int pshared = (cond->__data.__mutex == (void *) ~0l)
 		? LLL_SHARED : LLL_PRIVATE;
 
+#if (defined lll_futex_timed_wait_requeue_pi \
+     && defined __ASSUME_REQUEUE_PI)
+  int pi_flag = 0;
+#endif
+
   /* Make sure we are alone.  */
   lll_lock (cond->__data.__lock, pshared);
 
@@ -155,17 +160,46 @@ __pthread_cond_timedwait (cond, mutex, abstime)
       /* Enable asynchronous cancellation.  Required by the standard.  */
       cbuffer.oldtype = __pthread_enable_asynccancel ();
 
+/* REQUEUE_PI was implemented after FUTEX_CLOCK_REALTIME, so it is sufficient
+   to check just the former.  */
+#if (defined lll_futex_timed_wait_requeue_pi \
+     && defined __ASSUME_REQUEUE_PI)
+      /* If pi_flag remained 1 then it means that we had the lock and the mutex
+	 but a spurious waker raced ahead of us.  Give back the mutex before
+	 going into wait again.  */
+      if (pi_flag)
+	{
+	  __pthread_mutex_cond_lock_adjust (mutex);
+	  __pthread_mutex_unlock_usercnt (mutex, 0);
+	}
+      pi_flag = USE_REQUEUE_PI (mutex);
+
+      if (pi_flag)
+	{
+	  unsigned int clockbit = (cond->__data.__nwaiters & 1
+				   ? 0 : FUTEX_CLOCK_REALTIME);
+	  err = lll_futex_timed_wait_requeue_pi (&cond->__data.__futex,
+						 futex_val, abstime, clockbit,
+						 &mutex->__data.__lock,
+						 pshared);
+	  pi_flag = (err == 0);
+	}
+      else
+#endif
+
+	{
 #if (!defined __ASSUME_FUTEX_CLOCK_REALTIME \
      || !defined lll_futex_timed_wait_bitset)
-      /* Wait until woken by signal or broadcast.  */
-      err = lll_futex_timed_wait (&cond->__data.__futex,
-				  futex_val, &rt, pshared);
+	  /* Wait until woken by signal or broadcast.  */
+	  err = lll_futex_timed_wait (&cond->__data.__futex,
+				      futex_val, &rt, pshared);
 #else
-      unsigned int clockbit = (cond->__data.__nwaiters & 1
-			       ? 0 : FUTEX_CLOCK_REALTIME);
-      err = lll_futex_timed_wait_bitset (&cond->__data.__futex, futex_val,
-					 abstime, clockbit, pshared);
+	  unsigned int clockbit = (cond->__data.__nwaiters & 1
+				   ? 0 : FUTEX_CLOCK_REALTIME);
+	  err = lll_futex_timed_wait_bitset (&cond->__data.__futex, futex_val,
+					     abstime, clockbit, pshared);
 #endif
+	}
 
       /* Disable asynchronous cancellation.  */
       __pthread_disable_asynccancel (cbuffer.oldtype);
@@ -217,7 +251,16 @@ __pthread_cond_timedwait (cond, mutex, abstime)
   __pthread_cleanup_pop (&buffer, 0);
 
   /* Get the mutex before returning.  */
-  err = __pthread_mutex_cond_lock (mutex);
+#if (defined lll_futex_timed_wait_requeue_pi \
+     && defined __ASSUME_REQUEUE_PI)
+  if (pi_flag)
+    {
+      __pthread_mutex_cond_lock_adjust (mutex);
+      err = 0;
+    }
+  else
+#endif
+    err = __pthread_mutex_cond_lock (mutex);
 
   return err ?: result;
 }
diff --git a/nptl/pthread_cond_wait.c b/nptl/pthread_cond_wait.c
index 0ae320cb83..01d42d7834 100644
--- a/nptl/pthread_cond_wait.c
+++ b/nptl/pthread_cond_wait.c
@@ -26,7 +26,6 @@
 #include <shlib-compat.h>
 #include <stap-probe.h>
 
-
 struct _condvar_cleanup_buffer
 {
   int oldtype;
@@ -85,8 +84,15 @@ __condvar_cleanup (void *arg)
     lll_futex_wake (&cbuffer->cond->__data.__futex, INT_MAX, pshared);
 
   /* Get the mutex before returning unless asynchronous cancellation
-     is in effect.  */
-  __pthread_mutex_cond_lock (cbuffer->mutex);
+     is in effect.  We don't try to get the mutex if we already own it.  */
+  if (!(USE_REQUEUE_PI (cbuffer->mutex))
+      || ((cbuffer->mutex->__data.__lock & FUTEX_TID_MASK)
+	  != THREAD_GETMEM (THREAD_SELF, tid)))
+  {
+    __pthread_mutex_cond_lock (cbuffer->mutex);
+  }
+  else
+    __pthread_mutex_cond_lock_adjust (cbuffer->mutex);
 }
 
 
@@ -101,6 +107,11 @@ __pthread_cond_wait (cond, mutex)
   int pshared = (cond->__data.__mutex == (void *) ~0l)
 		? LLL_SHARED : LLL_PRIVATE;
 
+#if (defined lll_futex_wait_requeue_pi \
+     && defined __ASSUME_REQUEUE_PI)
+  int pi_flag = 0;
+#endif
+
   LIBC_PROBE (cond_wait, 2, cond, mutex);
 
   /* Make sure we are alone.  */
@@ -144,15 +155,36 @@ __pthread_cond_wait (cond, mutex)
   do
     {
       unsigned int futex_val = cond->__data.__futex;
-
       /* Prepare to wait.  Release the condvar futex.  */
       lll_unlock (cond->__data.__lock, pshared);
 
       /* Enable asynchronous cancellation.  Required by the standard.  */
       cbuffer.oldtype = __pthread_enable_asynccancel ();
 
-      /* Wait until woken by signal or broadcast.  */
-      lll_futex_wait (&cond->__data.__futex, futex_val, pshared);
+#if (defined lll_futex_wait_requeue_pi \
+     && defined __ASSUME_REQUEUE_PI)
+      /* If pi_flag remained 1 then it means that we had the lock and the mutex
+	 but a spurious waker raced ahead of us.  Give back the mutex before
+	 going into wait again.  */
+      if (pi_flag)
+	{
+	  __pthread_mutex_cond_lock_adjust (mutex);
+	  __pthread_mutex_unlock_usercnt (mutex, 0);
+	}
+      pi_flag = USE_REQUEUE_PI (mutex);
+
+      if (pi_flag)
+	{
+	  err = lll_futex_wait_requeue_pi (&cond->__data.__futex,
+					   futex_val, &mutex->__data.__lock,
+					   pshared);
+
+	  pi_flag = (err == 0);
+	}
+      else
+#endif
+	  /* Wait until woken by signal or broadcast.  */
+	lll_futex_wait (&cond->__data.__futex, futex_val, pshared);
 
       /* Disable asynchronous cancellation.  */
       __pthread_disable_asynccancel (cbuffer.oldtype);
@@ -189,8 +221,17 @@ __pthread_cond_wait (cond, mutex)
   /* The cancellation handling is back to normal, remove the handler.  */
   __pthread_cleanup_pop (&buffer, 0);
 
-  /* Get the mutex before returning.  */
-  return __pthread_mutex_cond_lock (mutex);
+  /* Get the mutex before returning.  Not needed for PI.  */
+#if (defined lll_futex_wait_requeue_pi \
+     && defined __ASSUME_REQUEUE_PI)
+  if (pi_flag)
+    {
+      __pthread_mutex_cond_lock_adjust (mutex);
+      return 0;
+    }
+  else
+#endif
+    return __pthread_mutex_cond_lock (mutex);
 }
 
 versioned_symbol (libpthread, __pthread_cond_wait, pthread_cond_wait,
diff --git a/nptl/sysdeps/unix/sysv/linux/powerpc/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/powerpc/lowlevellock.h
index b4b1fd4afd..f33f703346 100644
--- a/nptl/sysdeps/unix/sysv/linux/powerpc/lowlevellock.h
+++ b/nptl/sysdeps/unix/sysv/linux/powerpc/lowlevellock.h
@@ -39,6 +39,8 @@
 #define FUTEX_TRYLOCK_PI	8
 #define FUTEX_WAIT_BITSET	9
 #define FUTEX_WAKE_BITSET	10
+#define FUTEX_WAIT_REQUEUE_PI   11
+#define FUTEX_CMP_REQUEUE_PI    12
 #define FUTEX_PRIVATE_FLAG	128
 #define FUTEX_CLOCK_REALTIME	256
 
@@ -149,6 +151,34 @@
     INTERNAL_SYSCALL_ERROR_P (__ret, __err);				      \
   })
 
+/* Priority Inheritance support.  */
+#define lll_futex_wait_requeue_pi(futexp, val, mutex, private) \
+  lll_futex_timed_wait_requeue_pi (futexp, val, NULL, 0, mutex, private)
+
+#define lll_futex_timed_wait_requeue_pi(futexp, val, timespec, clockbit,      \
+					mutex, private)			      \
+  ({									      \
+    INTERNAL_SYSCALL_DECL (__err);					      \
+    long int __ret;							      \
+    int __op = FUTEX_WAIT_REQUEUE_PI | clockbit;			      \
+									      \
+    __ret = INTERNAL_SYSCALL (futex, __err, 5, (futexp),		      \
+			      __lll_private_flag (__op, private),	      \
+			      (val), (timespec), mutex); 		      \
+    INTERNAL_SYSCALL_ERROR_P (__ret, __err) ? -__ret : __ret;		      \
+  })
+
+#define lll_futex_cmp_requeue_pi(futexp, nr_wake, nr_move, mutex, val, priv)  \
+  ({									      \
+    INTERNAL_SYSCALL_DECL (__err);					      \
+    long int __ret;							      \
+									      \
+    __ret = INTERNAL_SYSCALL (futex, __err, 6, (futexp),		      \
+			      __lll_private_flag (FUTEX_CMP_REQUEUE_PI, priv),\
+			      (nr_wake), (nr_move), (mutex), (val));	      \
+    INTERNAL_SYSCALL_ERROR_P (__ret, __err);				      \
+  })
+
 
 #ifdef UP
 # define __lll_acq_instr	""
diff --git a/nptl/sysdeps/unix/sysv/linux/s390/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/s390/lowlevellock.h
index a0163d6371..3dab05e057 100644
--- a/nptl/sysdeps/unix/sysv/linux/s390/lowlevellock.h
+++ b/nptl/sysdeps/unix/sysv/linux/s390/lowlevellock.h
@@ -37,6 +37,8 @@
 #define FUTEX_TRYLOCK_PI	8
 #define FUTEX_WAIT_BITSET	9
 #define FUTEX_WAKE_BITSET	10
+#define FUTEX_WAIT_REQUEUE_PI   11
+#define FUTEX_CMP_REQUEUE_PI    12
 #define FUTEX_PRIVATE_FLAG	128
 #define FUTEX_CLOCK_REALTIME	256
 
@@ -141,6 +143,32 @@
     INTERNAL_SYSCALL_ERROR_P (__ret, __err);				      \
   })
 
+/* Priority Inheritance support.  */
+#define lll_futex_wait_requeue_pi(futexp, val, mutex, private) \
+  lll_futex_timed_wait_requeue_pi (futexp, val, NULL, 0, mutex, private)
+
+#define lll_futex_timed_wait_requeue_pi(futexp, val, timespec, clockbit,      \
+					mutex, private)			      \
+  ({									      \
+    INTERNAL_SYSCALL_DECL (__err);					      \
+    int __op = FUTEX_WAIT_REQUEUE_PI | clockbit;			      \
+									      \
+    INTERNAL_SYSCALL (futex, __err, 5, (futexp),			      \
+		      __lll_private_flag (__op, private),		      \
+		      (val), (timespec), mutex); 			      \
+  })
+
+#define lll_futex_cmp_requeue_pi(futexp, nr_wake, nr_move, mutex, val, priv)  \
+  ({									      \
+    INTERNAL_SYSCALL_DECL (__err);					      \
+    long int __ret;							      \
+									      \
+    __ret = INTERNAL_SYSCALL (futex, __err, 6, (futexp),		      \
+			      __lll_private_flag (FUTEX_CMP_REQUEUE_PI, priv),\
+			      (nr_wake), (nr_move), (mutex), (val));	      \
+    INTERNAL_SYSCALL_ERROR_P (__ret, __err);				      \
+  })
+
 #define lll_compare_and_swap(futex, oldval, newval, operation) \
   do {									      \
     __typeof (futex) __futex = (futex);					      \