[BZ #4586]

2007-06-06  Jakub Jelinek  <jakub@redhat.com>
	BZ #4586
	* sysdeps/i386/ldbl2mpn.c (__mpn_extract_long_double): Treat
	pseudo-zeros as zero.
	* sysdeps/x86_64/ldbl2mpn.c: New file.
	* sysdeps/ia64/ldbl2mpn.c: New file.

40 files changed, 300 insertions, 122 deletions

diff --git a/nptl/allocatestack.c b/nptl/allocatestack.c
index c31ca808bf..76d75fef2c 100644
--- a/nptl/allocatestack.c
+++ b/nptl/allocatestack.c
@@ -949,7 +949,7 @@ __nptl_setxid (struct xid_command *cmdp)
   int cur = cmdp->cntr;
   while (cur != 0)
     {
-      lll_futex_wait (&cmdp->cntr, cur);
+      lll_private_futex_wait (&cmdp->cntr, cur);
       cur = cmdp->cntr;
     }
 
@@ -1035,7 +1035,7 @@ __wait_lookup_done (void)
 	continue;
 
       do
-	lll_futex_wait (gscope_flagp, THREAD_GSCOPE_FLAG_WAIT);
+	lll_private_futex_wait (gscope_flagp, THREAD_GSCOPE_FLAG_WAIT);
       while (*gscope_flagp == THREAD_GSCOPE_FLAG_WAIT);
     }
 
@@ -1057,7 +1057,7 @@ __wait_lookup_done (void)
 	continue;
 
       do
-	lll_futex_wait (gscope_flagp, THREAD_GSCOPE_FLAG_WAIT);
+	lll_private_futex_wait (gscope_flagp, THREAD_GSCOPE_FLAG_WAIT);
       while (*gscope_flagp == THREAD_GSCOPE_FLAG_WAIT);
     }
 
diff --git a/nptl/init.c b/nptl/init.c
index 82adeae6a6..be8359ea21 100644
--- a/nptl/init.c
+++ b/nptl/init.c
@@ -216,7 +216,7 @@ sighandler_setxid (int sig, siginfo_t *si, void *ctx)
 			__xidcmd->id[1], __xidcmd->id[2]);
 
   if (atomic_decrement_val (&__xidcmd->cntr) == 0)
-    lll_futex_wake (&__xidcmd->cntr, 1);
+    lll_private_futex_wake (&__xidcmd->cntr, 1);
 
   /* Reset the SETXID flag.  */
   struct pthread *self = THREAD_SELF;
@@ -225,7 +225,7 @@ sighandler_setxid (int sig, siginfo_t *si, void *ctx)
 
   /* And release the futex.  */
   self->setxid_futex = 1;
-  lll_futex_wake (&self->setxid_futex, 1);
+  lll_private_futex_wake (&self->setxid_futex, 1);
 }
 
 
diff --git a/nptl/lowlevellock.h b/nptl/lowlevellock.h
index 338da39990..0600e1794d 100644
--- a/nptl/lowlevellock.h
+++ b/nptl/lowlevellock.h
@@ -1,5 +1,5 @@
 /* Low level locking macros used in NPTL implementation.  Stub version.
-   Copyright (C) 2002 Free Software Foundation, Inc.
+   Copyright (C) 2002, 2007 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -21,16 +21,6 @@
 #include <atomic.h>
 
 
-/* Implement generic mutex.  Basic futex syscall support is required:
-
-     lll_futex_wait(futex, value) - call sys_futex with FUTEX_WAIT
-				    and third parameter VALUE
-
-     lll_futex_wake(futex, value) - call sys_futex with FUTEX_WAKE
-				    and third parameter VALUE
-*/
-
-
 /* Mutex lock counter:
    bit 31 clear means unlocked;
    bit 31 set means locked.
@@ -66,7 +56,9 @@ __generic_mutex_lock (int *mutex)
       if (v >= 0)
 	continue;
 
-      lll_futex_wait (mutex, v);
+      lll_futex_wait (mutex, v,
+		      // XYZ check mutex flag
+		      LLL_SHARED);
     }
 }
 
@@ -82,7 +74,9 @@ __generic_mutex_unlock (int *mutex)
 
   /* There are other threads waiting for this mutex, wake one of them
      up.  */
-  lll_futex_wake (mutex, 1);
+  lll_futex_wake (mutex, 1,
+		  // XYZ check mutex flag
+		  LLL_SHARED);
 }
 
 
diff --git a/nptl/pthread_barrier_wait.c b/nptl/pthread_barrier_wait.c
index c6b563f242..e96a3e5473 100644
--- a/nptl/pthread_barrier_wait.c
+++ b/nptl/pthread_barrier_wait.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2003, 2004 Free Software Foundation, Inc.
+/* Copyright (C) 2003, 2004, 2007 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Martin Schwidefsky <schwidefsky@de.ibm.com>, 2003.
 
@@ -45,7 +45,9 @@ pthread_barrier_wait (barrier)
       ++ibarrier->curr_event;
 
       /* Wake up everybody.  */
-      lll_futex_wake (&ibarrier->curr_event, INT_MAX);
+      lll_futex_wake (&ibarrier->curr_event, INT_MAX,
+		      // XYZ check mutex flag
+		      LLL_SHARED);
 
       /* This is the thread which finished the serialization.  */
       result = PTHREAD_BARRIER_SERIAL_THREAD;
@@ -61,7 +63,9 @@ pthread_barrier_wait (barrier)
 
       /* Wait for the event counter of the barrier to change.  */
       do
-	lll_futex_wait (&ibarrier->curr_event, event);
+	lll_futex_wait (&ibarrier->curr_event, event,
+			// XYZ check mutex flag
+			LLL_SHARED);
       while (event == ibarrier->curr_event);
     }
 
diff --git a/nptl/pthread_cond_broadcast.c b/nptl/pthread_cond_broadcast.c
index 2b8b5460f4..aec33f3bd8 100644
--- a/nptl/pthread_cond_broadcast.c
+++ b/nptl/pthread_cond_broadcast.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2003, 2004, 2006 Free Software Foundation, Inc.
+/* Copyright (C) 2003, 2004, 2006, 2007 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Martin Schwidefsky <schwidefsky@de.ibm.com>, 2003.
 
@@ -69,7 +69,9 @@ __pthread_cond_broadcast (cond)
 	{
 	  /* The requeue functionality is not available.  */
 	wake_all:
-	  lll_futex_wake (&cond->__data.__futex, INT_MAX);
+	  lll_futex_wake (&cond->__data.__futex, INT_MAX,
+			  // XYZ check mutex flag
+			  LLL_SHARED);
 	}
 
       /* That's all.  */
diff --git a/nptl/pthread_cond_destroy.c b/nptl/pthread_cond_destroy.c
index 0b8d4119d1..8574b6118f 100644
--- a/nptl/pthread_cond_destroy.c
+++ b/nptl/pthread_cond_destroy.c
@@ -59,14 +59,18 @@ __pthread_cond_destroy (cond)
 	  && cond->__data.__mutex != (void *) ~0l)
 	{
 	  pthread_mutex_t *mut = (pthread_mutex_t *) cond->__data.__mutex;
-	  lll_futex_wake (&mut->__data.__lock, INT_MAX);
+	  lll_futex_wake (&mut->__data.__lock, INT_MAX,
+			  // XYZ check mutex flag
+			  LLL_SHARED);
 	}
 
       do
 	{
 	  lll_mutex_unlock (cond->__data.__lock);
 
-	  lll_futex_wait (&cond->__data.__nwaiters, nwaiters);
+	  lll_futex_wait (&cond->__data.__nwaiters, nwaiters,
+			  // XYZ check mutex flag
+			  LLL_SHARED);
 
 	  lll_mutex_lock (cond->__data.__lock);
 
diff --git a/nptl/pthread_cond_signal.c b/nptl/pthread_cond_signal.c
index 5a9bbcad91..a4faf41854 100644
--- a/nptl/pthread_cond_signal.c
+++ b/nptl/pthread_cond_signal.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2003, 2004 Free Software Foundation, Inc.
+/* Copyright (C) 2003, 2004, 2007 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Martin Schwidefsky <schwidefsky@de.ibm.com>, 2003.
 
@@ -44,11 +44,15 @@ __pthread_cond_signal (cond)
 
       /* Wake one.  */
       if (! __builtin_expect (lll_futex_wake_unlock (&cond->__data.__futex, 1,
-						     1, &cond->__data.__lock),
+						     1, &cond->__data.__lock,
+						     // XYZ check mutex flag
+						     LLL_SHARED),
 						     0))
 	return 0;
 
-      lll_futex_wake (&cond->__data.__futex, 1);
+      lll_futex_wake (&cond->__data.__futex, 1,
+		      // XYZ check mutex flag
+		      LLL_SHARED);
     }
 
   /* We are done.  */
diff --git a/nptl/pthread_cond_timedwait.c b/nptl/pthread_cond_timedwait.c
index 261b4484ba..d1c29d2377 100644
--- a/nptl/pthread_cond_timedwait.c
+++ b/nptl/pthread_cond_timedwait.c
@@ -153,7 +153,9 @@ __pthread_cond_timedwait (cond, mutex, abstime)
 
       /* Wait until woken by signal or broadcast.  */
       err = lll_futex_timed_wait (&cond->__data.__futex,
-				  futex_val, &rt);
+				  futex_val, &rt,
+				  // XYZ check mutex flag
+				  LLL_SHARED);
 
       /* Disable asynchronous cancellation.  */
       __pthread_disable_asynccancel (cbuffer.oldtype);
@@ -196,7 +198,9 @@ __pthread_cond_timedwait (cond, mutex, abstime)
      and it can be successfully destroyed.  */
   if (cond->__data.__total_seq == -1ULL
       && cond->__data.__nwaiters < (1 << COND_NWAITERS_SHIFT))
-    lll_futex_wake (&cond->__data.__nwaiters, 1);
+    lll_futex_wake (&cond->__data.__nwaiters, 1,
+		    // XYZ check mutex flag
+		    LLL_SHARED);
 
   /* We are done with the condvar.  */
   lll_mutex_unlock (cond->__data.__lock);
diff --git a/nptl/pthread_cond_wait.c b/nptl/pthread_cond_wait.c
index 6e00a28f6a..e524aa6c94 100644
--- a/nptl/pthread_cond_wait.c
+++ b/nptl/pthread_cond_wait.c
@@ -71,7 +71,9 @@ __condvar_cleanup (void *arg)
   if (cbuffer->cond->__data.__total_seq == -1ULL
       && cbuffer->cond->__data.__nwaiters < (1 << COND_NWAITERS_SHIFT))
     {
-      lll_futex_wake (&cbuffer->cond->__data.__nwaiters, 1);
+      lll_futex_wake (&cbuffer->cond->__data.__nwaiters, 1,
+		      // XYZ check mutex flag
+		      LLL_SHARED);
       destroying = 1;
     }
 
@@ -80,7 +82,9 @@ __condvar_cleanup (void *arg)
 
   /* Wake everybody to make sure no condvar signal gets lost.  */
   if (! destroying)
-    lll_futex_wake (&cbuffer->cond->__data.__futex, INT_MAX);
+    lll_futex_wake (&cbuffer->cond->__data.__futex, INT_MAX,
+		    // XYZ check mutex flag
+		    LLL_SHARED);
 
   /* Get the mutex before returning unless asynchronous cancellation
      is in effect.  */
@@ -146,7 +150,9 @@ __pthread_cond_wait (cond, mutex)
       cbuffer.oldtype = __pthread_enable_asynccancel ();
 
       /* Wait until woken by signal or broadcast.  */
-      lll_futex_wait (&cond->__data.__futex, futex_val);
+      lll_futex_wait (&cond->__data.__futex, futex_val,
+		      // XYZ check mutex flag
+		      LLL_SHARED);
 
       /* Disable asynchronous cancellation.  */
       __pthread_disable_asynccancel (cbuffer.oldtype);
@@ -175,7 +181,9 @@ __pthread_cond_wait (cond, mutex)
      and it can be successfully destroyed.  */
   if (cond->__data.__total_seq == -1ULL
       && cond->__data.__nwaiters < (1 << COND_NWAITERS_SHIFT))
-    lll_futex_wake (&cond->__data.__nwaiters, 1);
+    lll_futex_wake (&cond->__data.__nwaiters, 1,
+		    // XYZ check mutex flag
+		    LLL_SHARED);
 
   /* We are done with the condvar.  */
   lll_mutex_unlock (cond->__data.__lock);
diff --git a/nptl/pthread_create.c b/nptl/pthread_create.c
index 79729ced03..b1f852dc95 100644
--- a/nptl/pthread_create.c
+++ b/nptl/pthread_create.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003, 2004, 2005, 2006 Free Software Foundation, Inc.
+/* Copyright (C) 2002,2003,2004,2005,2006,2007 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -385,7 +385,7 @@ start_thread (void *arg)
       /* Some other thread might call any of the setXid functions and expect
 	 us to reply.  In this case wait until we did that.  */
       do
-	lll_futex_wait (&pd->setxid_futex, 0);
+	lll_private_futex_wait (&pd->setxid_futex, 0);
       while (pd->cancelhandling & SETXID_BITMASK);
 
       /* Reset the value so that the stack can be reused.  */
diff --git a/nptl/pthread_mutex_lock.c b/nptl/pthread_mutex_lock.c
index 1c3ee4fe25..a17013bb76 100644
--- a/nptl/pthread_mutex_lock.c
+++ b/nptl/pthread_mutex_lock.c
@@ -408,7 +408,9 @@ __pthread_mutex_lock (mutex)
 		  break;
 
 		if (oldval != ceilval)
-		  lll_futex_wait (&mutex->__data.__lock, ceilval | 2);
+		  lll_futex_wait (&mutex->__data.__lock, ceilval | 2,
+				  // XYZ check mutex flag
+				  LLL_SHARED);
 	      }
 	    while (atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 							ceilval | 2, ceilval)
diff --git a/nptl/pthread_mutex_setprioceiling.c b/nptl/pthread_mutex_setprioceiling.c
index cd13d1c14c..301fb63d21 100644
--- a/nptl/pthread_mutex_setprioceiling.c
+++ b/nptl/pthread_mutex_setprioceiling.c
@@ -1,5 +1,5 @@
 /* Set current priority ceiling of pthread_mutex_t.
-   Copyright (C) 2006 Free Software Foundation, Inc.
+   Copyright (C) 2006, 2007 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Jakub Jelinek <jakub@redhat.com>, 2006.
 
@@ -80,7 +80,9 @@ pthread_mutex_setprioceiling (mutex, prioceiling, old_ceiling)
 	      break;
 
 	    if (oldval != ceilval)
-	      lll_futex_wait (&mutex->__data.__lock, ceilval | 2);
+	      lll_futex_wait (&mutex->__data.__lock, ceilval | 2,
+			      // XYZ check mutex flag
+			      LLL_SHARED);
 	  }
 	while (atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
 						    ceilval | 2, ceilval)
@@ -110,7 +112,9 @@ pthread_mutex_setprioceiling (mutex, prioceiling, old_ceiling)
 			 | (prioceiling << PTHREAD_MUTEX_PRIO_CEILING_SHIFT);
   atomic_full_barrier ();
 
-  lll_futex_wake (&mutex->__data.__lock, INT_MAX);
+  lll_futex_wake (&mutex->__data.__lock, INT_MAX,
+		  // XYZ check mutex flag
+		  LLL_SHARED);
 
   return 0;
 }
diff --git a/nptl/pthread_mutex_timedlock.c b/nptl/pthread_mutex_timedlock.c
index 8fd681c6ef..74504e44a1 100644
--- a/nptl/pthread_mutex_timedlock.c
+++ b/nptl/pthread_mutex_timedlock.c
@@ -441,7 +441,9 @@ pthread_mutex_timedlock (mutex, abstime)
 		      }
 
 		    lll_futex_timed_wait (&mutex->__data.__lock,
-					  ceilval | 2, &rt);
+					  ceilval | 2, &rt,
+					  // XYZ check mutex flag
+					  LLL_SHARED);
 		  }
 	      }
 	    while (atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
diff --git a/nptl/pthread_mutex_unlock.c b/nptl/pthread_mutex_unlock.c
index 33919d60af..1e6b276580 100644
--- a/nptl/pthread_mutex_unlock.c
+++ b/nptl/pthread_mutex_unlock.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003, 2005, 2006 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2005, 2006, 2007 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -240,7 +240,9 @@ __pthread_mutex_unlock_usercnt (mutex, decr)
 						   newval, oldval));
 
       if ((oldval & ~PTHREAD_MUTEX_PRIO_CEILING_MASK) > 1)
-	lll_futex_wake (&mutex->__data.__lock, 1);
+	lll_futex_wake (&mutex->__data.__lock, 1,
+			// XYZ check mutex flag
+			LLL_SHARED);
 
       int oldprio = newval >> PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
       return __pthread_tpp_change_priority (oldprio, -1);
diff --git a/nptl/pthread_rwlock_timedrdlock.c b/nptl/pthread_rwlock_timedrdlock.c
index caff5894fe..654d628b2f 100644
--- a/nptl/pthread_rwlock_timedrdlock.c
+++ b/nptl/pthread_rwlock_timedrdlock.c
@@ -114,7 +114,9 @@ pthread_rwlock_timedrdlock (rwlock, abstime)
 
       /* Wait for the writer to finish.  */
       err = lll_futex_timed_wait (&rwlock->__data.__readers_wakeup,
-				  waitval, &rt);
+				  waitval, &rt,
+				  // XYZ check mutex flag
+				  LLL_SHARED);
 
       /* Get the lock.  */
       lll_mutex_lock (rwlock->__data.__lock);
diff --git a/nptl/pthread_rwlock_timedwrlock.c b/nptl/pthread_rwlock_timedwrlock.c
index 97c0598f96..354beb0846 100644
--- a/nptl/pthread_rwlock_timedwrlock.c
+++ b/nptl/pthread_rwlock_timedwrlock.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2003, 2004 Free Software Foundation, Inc.
+/* Copyright (C) 2003, 2004, 2007 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Martin Schwidefsky <schwidefsky@de.ibm.com>, 2003.
 
@@ -104,7 +104,9 @@ pthread_rwlock_timedwrlock (rwlock, abstime)
 
       /* Wait for the writer or reader(s) to finish.  */
       err = lll_futex_timed_wait (&rwlock->__data.__writer_wakeup,
-				  waitval, &rt);
+				  waitval, &rt,
+				  // XYZ check mutex flag
+				  LLL_SHARED);
 
       /* Get the lock.  */
       lll_mutex_lock (rwlock->__data.__lock);
diff --git a/nptl/pthread_rwlock_unlock.c b/nptl/pthread_rwlock_unlock.c
index 9cae8b6c22..87a77a94ab 100644
--- a/nptl/pthread_rwlock_unlock.c
+++ b/nptl/pthread_rwlock_unlock.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2003 Free Software Foundation, Inc.
+/* Copyright (C) 2003, 2007 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Martin Schwidefsky <schwidefsky@de.ibm.com>, 2003.
 
@@ -38,14 +38,18 @@ __pthread_rwlock_unlock (pthread_rwlock_t *rwlock)
 	{
 	  ++rwlock->__data.__writer_wakeup;
 	  lll_mutex_unlock (rwlock->__data.__lock);
-	  lll_futex_wake (&rwlock->__data.__writer_wakeup, 1);
+	  lll_futex_wake (&rwlock->__data.__writer_wakeup, 1,
+			  // XYZ check mutex flag
+			  LLL_SHARED);
 	  return 0;
 	}
       else if (rwlock->__data.__nr_readers_queued)
 	{
 	  ++rwlock->__data.__readers_wakeup;
 	  lll_mutex_unlock (rwlock->__data.__lock);
-	  lll_futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX);
+	  lll_futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX,
+			  // XYZ check mutex flag
+			  LLL_SHARED);
 	  return 0;
 	}
     }
diff --git a/nptl/sysdeps/alpha/tls.h b/nptl/sysdeps/alpha/tls.h
index 64ddcd5c01..388a399c73 100644
--- a/nptl/sysdeps/alpha/tls.h
+++ b/nptl/sysdeps/alpha/tls.h
@@ -131,7 +131,7 @@ typedef struct
 	= atomic_exchange_rel (&THREAD_SELF->header.gscope_flag,	     \
 			       THREAD_GSCOPE_FLAG_UNUSED);		     \
       if (__res == THREAD_GSCOPE_FLAG_WAIT)				     \
-	lll_futex_wake (&THREAD_SELF->header.gscope_flag, 1);		     \
+	lll_private_futex_wake (&THREAD_SELF->header.gscope_flag, 1);	     \
     }									     \
   while (0)
 #define THREAD_GSCOPE_SET_FLAG() \
diff --git a/nptl/sysdeps/i386/tls.h b/nptl/sysdeps/i386/tls.h
index 7d3c5f97ad..1ffd4cf494 100644
--- a/nptl/sysdeps/i386/tls.h
+++ b/nptl/sysdeps/i386/tls.h
@@ -449,7 +449,7 @@ union user_desc_init
 		    : "i" (offsetof (struct pthread, header.gscope_flag)),    \
 		      "0" (THREAD_GSCOPE_FLAG_UNUSED));			      \
       if (__res == THREAD_GSCOPE_FLAG_WAIT)				      \
-	lll_futex_wake (&THREAD_SELF->header.gscope_flag, 1);		      \
+	lll_private_futex_wake (&THREAD_SELF->header.gscope_flag, 1);	      \
     }									      \
   while (0)
 #define THREAD_GSCOPE_SET_FLAG() \
diff --git a/nptl/sysdeps/ia64/tls.h b/nptl/sysdeps/ia64/tls.h
index a144c28b09..a5b7b76e2a 100644
--- a/nptl/sysdeps/ia64/tls.h
+++ b/nptl/sysdeps/ia64/tls.h
@@ -173,7 +173,7 @@ register struct pthread *__thread_self __asm__("r13");
 	= atomic_exchange_rel (&THREAD_SELF->header.gscope_flag,	     \
 			       THREAD_GSCOPE_FLAG_UNUSED);		     \
       if (__res == THREAD_GSCOPE_FLAG_WAIT)				     \
-	lll_futex_wake (&THREAD_SELF->header.gscope_flag, 1);		     \
+	lll_private_futex_wake (&THREAD_SELF->header.gscope_flag, 1);	     \
     }									     \
   while (0)
 #define THREAD_GSCOPE_SET_FLAG() \
diff --git a/nptl/sysdeps/powerpc/tls.h b/nptl/sysdeps/powerpc/tls.h
index bd9c99e06e..06c60e8ba2 100644
--- a/nptl/sysdeps/powerpc/tls.h
+++ b/nptl/sysdeps/powerpc/tls.h
@@ -190,7 +190,7 @@ register void *__thread_register __asm__ ("r13");
 	= atomic_exchange_rel (&THREAD_SELF->header.gscope_flag,	     \
 			       THREAD_GSCOPE_FLAG_UNUSED);		     \
       if (__res == THREAD_GSCOPE_FLAG_WAIT)				     \
-	lll_futex_wake (&THREAD_SELF->header.gscope_flag, 1);		     \
+	lll_private_futex_wake (&THREAD_SELF->header.gscope_flag, 1);	     \
     }									     \
   while (0)
 #define THREAD_GSCOPE_SET_FLAG() \
diff --git a/nptl/sysdeps/pthread/aio_misc.h b/nptl/sysdeps/pthread/aio_misc.h
index c5a11f4550..25ab74e272 100644
--- a/nptl/sysdeps/pthread/aio_misc.h
+++ b/nptl/sysdeps/pthread/aio_misc.h
@@ -1,4 +1,4 @@
-/* Copyright (C) 2006 Free Software Foundation, Inc.
+/* Copyright (C) 2006, 2007 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
 
    The GNU C Library is free software; you can redistribute it and/or
@@ -30,7 +30,7 @@
 #define AIO_MISC_NOTIFY(waitlist) \
   do {									      \
     if (*waitlist->counterp > 0 && --*waitlist->counterp == 0)		      \
-      lll_futex_wake (waitlist->counterp, 1);				      \
+      lll_private_futex_wake (waitlist->counterp, 1);			      \
   } while (0)
 
 #define AIO_MISC_WAIT(result, futex, timeout, cancel)			      \
@@ -49,7 +49,8 @@
 	int status;							      \
 	do								      \
 	  {								      \
-	    status = lll_futex_timed_wait (futexaddr, oldval, timeout);	      \
+	    status = lll_private_futex_timed_wait (futexaddr, oldval,	      \
+						   timeout);		      \
 	    if (status != -EWOULDBLOCK)					      \
 	      break;							      \
 									      \
diff --git a/nptl/sysdeps/pthread/gai_misc.h b/nptl/sysdeps/pthread/gai_misc.h
index 9f6a73dad1..5f6990040a 100644
--- a/nptl/sysdeps/pthread/gai_misc.h
+++ b/nptl/sysdeps/pthread/gai_misc.h
@@ -1,4 +1,4 @@
-/* Copyright (C) 2006 Free Software Foundation, Inc.
+/* Copyright (C) 2006, 2007 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
 
    The GNU C Library is free software; you can redistribute it and/or
@@ -31,7 +31,7 @@
 #define GAI_MISC_NOTIFY(waitlist) \
   do {									      \
     if (*waitlist->counterp > 0 && --*waitlist->counterp == 0)		      \
-      lll_futex_wake (waitlist->counterp, 1);				      \
+      lll_private_futex_wake (waitlist->counterp, 1);			      \
   } while (0)
 
 #define GAI_MISC_WAIT(result, futex, timeout, cancel) \
@@ -50,7 +50,8 @@
 	int status;							      \
 	do								      \
 	  {								      \
-	    status = lll_futex_timed_wait (futexaddr, oldval, timeout);	      \
+	    status = lll_private_futex_timed_wait (futexaddr, oldval,	      \
+						   timeout);		      \
 	    if (status != -EWOULDBLOCK)					      \
 	      break;							      \
 									      \
diff --git a/nptl/sysdeps/s390/tls.h b/nptl/sysdeps/s390/tls.h
index de8a81bd09..f10db8f227 100644
--- a/nptl/sysdeps/s390/tls.h
+++ b/nptl/sysdeps/s390/tls.h
@@ -183,7 +183,7 @@ typedef struct
 	= atomic_exchange_rel (&THREAD_SELF->header.gscope_flag,	     \
 			       THREAD_GSCOPE_FLAG_UNUSED);		     \
       if (__res == THREAD_GSCOPE_FLAG_WAIT)				     \
-	lll_futex_wake (&THREAD_SELF->header.gscope_flag, 1);		     \
+	lll_private_futex_wake (&THREAD_SELF->header.gscope_flag, 1);	     \
     }									     \
   while (0)
 #define THREAD_GSCOPE_SET_FLAG() \
diff --git a/nptl/sysdeps/sh/tls.h b/nptl/sysdeps/sh/tls.h
index 6d6eff665f..07bab4f576 100644
--- a/nptl/sysdeps/sh/tls.h
+++ b/nptl/sysdeps/sh/tls.h
@@ -160,7 +160,7 @@ typedef struct
 	= atomic_exchange_rel (&THREAD_SELF->header.gscope_flag,	     \
 			       THREAD_GSCOPE_FLAG_UNUSED);		     \
       if (__res == THREAD_GSCOPE_FLAG_WAIT)				     \
-	lll_futex_wake (&THREAD_SELF->header.gscope_flag, 1);		     \
+	lll_private_futex_wake (&THREAD_SELF->header.gscope_flag, 1);	     \
     }									     \
   while (0)
 #define THREAD_GSCOPE_SET_FLAG() \
diff --git a/nptl/sysdeps/sparc/tls.h b/nptl/sysdeps/sparc/tls.h
index 60e52fd4ea..788c974c1c 100644
--- a/nptl/sysdeps/sparc/tls.h
+++ b/nptl/sysdeps/sparc/tls.h
@@ -151,7 +151,7 @@ register struct pthread *__thread_self __asm__("%g7");
 	= atomic_exchange_rel (&THREAD_SELF->header.gscope_flag,	     \
 			       THREAD_GSCOPE_FLAG_UNUSED);		     \
       if (__res == THREAD_GSCOPE_FLAG_WAIT)				     \
-	lll_futex_wake (&THREAD_SELF->header.gscope_flag, 1);		     \
+	lll_private_futex_wake (&THREAD_SELF->header.gscope_flag, 1);	     \
     }									     \
   while (0)
 #define THREAD_GSCOPE_SET_FLAG() \
diff --git a/nptl/sysdeps/unix/sysv/linux/fork.c b/nptl/sysdeps/unix/sysv/linux/fork.c
index 98bb237c06..c6dadb5683 100644
--- a/nptl/sysdeps/unix/sysv/linux/fork.c
+++ b/nptl/sysdeps/unix/sysv/linux/fork.c
@@ -203,7 +203,7 @@ __libc_fork (void)
 
 	  if (atomic_decrement_and_test (&allp->handler->refcntr)
 	      && allp->handler->need_signal)
-	    lll_futex_wake (allp->handler->refcntr, 1);
+	    lll_private_futex_wake (allp->handler->refcntr, 1);
 
 	  allp = allp->next;
 	}
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h
index aa963f77e7..474b39ae18 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h
+++ b/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h
@@ -23,6 +23,8 @@
 #include <time.h>
 #include <sys/param.h>
 #include <bits/pthreadtypes.h>
+#include <kernel-features.h>
+#include <tcb-offsets.h>
 
 #ifndef LOCK_INSTR
 # ifdef UP
@@ -41,6 +43,13 @@
 #define FUTEX_PRIVATE_FLAG	128
 
 
+/* Values for 'private' parameter of locking macros.  Yes, the
+   definition seems to be backwards.  But it is not.  The bit will be
+   reversed before passing to the system call.  */
+#define LLL_PRIVATE	0
+#define LLL_SHARED	FUTEX_PRIVATE_FLAG
+
+
 /* Initializer for compatibility lock.  */
 #define LLL_MUTEX_LOCK_INITIALIZER		(0)
 #define LLL_MUTEX_LOCK_INITIALIZER_LOCKED	(1)
@@ -145,7 +154,11 @@ LLL_STUB_UNWIND_INFO_START					\
 LLL_STUB_UNWIND_INFO_END
 
 
-#define lll_futex_wait(futex, val) \
+#define lll_futex_wait(futex, val, private) \
+  lll_futex_timed_wait (futex, val, NULL, private)
+
+
+#define lll_futex_timed_wait(futex, val, timeout, private) \
   ({									      \
     int __status;							      \
     register __typeof (val) _val asm ("edx") = (val);			      \
@@ -153,7 +166,7 @@ LLL_STUB_UNWIND_INFO_END
 		      LLL_ENTER_KERNEL					      \
 		      LLL_EBX_LOAD					      \
 		      : "=a" (__status)					      \
-		      : "0" (SYS_futex), LLL_EBX_REG (futex), "S" (0),	      \
+		      : "0" (SYS_futex), LLL_EBX_REG (futex), "S" (timeout),  \
 			"c" (FUTEX_WAIT), "d" (_val),			      \
 			"i" (offsetof (tcbhead_t, sysinfo))		      \
 		      : "memory");					      \
@@ -161,7 +174,27 @@ LLL_STUB_UNWIND_INFO_END
   })
 
 
-#define lll_futex_timed_wait(futex, val, timeout)			      \
+#define lll_futex_wake(futex, nr, private) \
+  do {									      \
+    int __ignore;							      \
+    register __typeof (nr) _nr asm ("edx") = (nr);			      \
+    __asm __volatile (LLL_EBX_LOAD					      \
+		      LLL_ENTER_KERNEL					      \
+		      LLL_EBX_LOAD					      \
+		      : "=a" (__ignore)					      \
+		      : "0" (SYS_futex), LLL_EBX_REG (futex),		      \
+			"c" (FUTEX_WAKE), "d" (_nr),			      \
+			"i" (0) /* phony, to align next arg's number */,      \
+			"i" (offsetof (tcbhead_t, sysinfo)));		      \
+  } while (0)
+
+
+#define lll_private_futex_wait(futex, val) \
+  lll_private_futex_timed_wait (futex, val, NULL)
+
+
+#ifdef __ASSUME_PRIVATE_FUTEX
+# define lll_private_futex_timed_wait(futex, val, timeout) \
   ({									      \
     int __status;							      \
     register __typeof (val) _val asm ("edx") = (val);			      \
@@ -170,14 +203,14 @@ LLL_STUB_UNWIND_INFO_END
 		      LLL_EBX_LOAD					      \
 		      : "=a" (__status)					      \
 		      : "0" (SYS_futex), LLL_EBX_REG (futex), "S" (timeout),  \
-			"c" (FUTEX_WAIT), "d" (_val),			      \
+			"c" (FUTEX_WAIT | FUTEX_PRIVATE_FLAG)), "d" (_val),   \
 			"i" (offsetof (tcbhead_t, sysinfo))		      \
 		      : "memory");					      \
     __status;								      \
   })
 
 
-#define lll_futex_wake(futex, nr) \
+# define lll_private_futex_wake(futex, nr) \
   do {									      \
     int __ignore;							      \
     register __typeof (nr) _nr asm ("edx") = (nr);			      \
@@ -186,10 +219,45 @@ LLL_STUB_UNWIND_INFO_END
 		      LLL_EBX_LOAD					      \
 		      : "=a" (__ignore)					      \
 		      : "0" (SYS_futex), LLL_EBX_REG (futex),		      \
-			"c" (FUTEX_WAKE), "d" (_nr),			      \
+			"c" (FUTEX_WAKE | FUTEX_PRIVATE_FLAG), "d" (_nr),     \
 			"i" (0) /* phony, to align next arg's number */,      \
 			"i" (offsetof (tcbhead_t, sysinfo)));		      \
   } while (0)
+#else
+# define lll_private_futex_timed_wait(futex, val, timeout) \
+  ({									      \
+    int __status;							      \
+    int __ignore;							      \
+    register __typeof (val) _val asm ("edx") = (val);			      \
+    __asm __volatile ("movl %%gs:%P7, %%ecx\n\t"			      \
+		      LLL_EBX_LOAD					      \
+		      LLL_ENTER_KERNEL					      \
+		      LLL_EBX_LOAD					      \
+		      : "=a" (__status), "=c" (__ignore)		      \
+		      : LLL_EBX_REG (futex), "0" (SYS_futex), "S" (timeout),  \
+			"d" (_val), "i" (offsetof (tcbhead_t, sysinfo)),      \
+			"i" (PRIVATE_FUTEX)				      \
+		      : "memory");					      \
+    __status;								      \
+  })
+
+
+# define lll_private_futex_wake(futex, nr) \
+  do {									      \
+    int __ignore;							      \
+    int __ignore2;							      \
+    register __typeof (nr) _nr asm ("edx") = (nr);			      \
+    __asm __volatile ("orl %%gs:%P7, %%ecx\n\t"				      \
+		      LLL_EBX_LOAD					      \
+		      LLL_ENTER_KERNEL					      \
+		      LLL_EBX_LOAD					      \
+		      : "=a" (__ignore), "=c" (__ignore2)		      \
+		      : LLL_EBX_REG (futex), "0" (SYS_futex),		      \
+			"1" (FUTEX_WAKE), "d" (_nr),			      \
+			"i" (offsetof (tcbhead_t, sysinfo)),		      \
+			"i" (PRIVATE_FUTEX));				      \
+  } while (0)
+#endif
 
 
 /* Does not preserve %eax and %ecx.  */
@@ -415,21 +483,6 @@ extern int __lll_mutex_unlock_wake (int *__futex)
 				"i" (offsetof (tcbhead_t, sysinfo))); })
 
 
-#define lll_futex_wake(futex, nr) \
-  do {									      \
-    int __ignore;							      \
-    register __typeof (nr) _nr asm ("edx") = (nr);			      \
-    __asm __volatile (LLL_EBX_LOAD					      \
-		      LLL_ENTER_KERNEL					      \
-		      LLL_EBX_LOAD					      \
-		      : "=a" (__ignore)					      \
-		      : "0" (SYS_futex), LLL_EBX_REG (futex),		      \
-			"c" (FUTEX_WAKE), "d" (_nr),			      \
-			"i" (0) /* phony, to align next arg's number */,      \
-			"i" (offsetof (tcbhead_t, sysinfo)));		      \
-  } while (0)
-
-
 #define lll_mutex_islocked(futex) \
   (futex != 0)
 
diff --git a/nptl/sysdeps/unix/sysv/linux/lowlevellock.c b/nptl/sysdeps/unix/sysv/linux/lowlevellock.c
index 38d78884d4..ab7f605f0c 100644
--- a/nptl/sysdeps/unix/sysv/linux/lowlevellock.c
+++ b/nptl/sysdeps/unix/sysv/linux/lowlevellock.c
@@ -31,7 +31,9 @@ __lll_lock_wait (int *futex)
     {
       int oldval = atomic_compare_and_exchange_val_acq (futex, 2, 1);
       if (oldval != 0)
-	lll_futex_wait (futex, 2);
+	lll_futex_wait (futex, 2,
+			// XYZ check mutex flag
+			LLL_SHARED);
     }
   while (atomic_compare_and_exchange_bool_acq (futex, 2, 0) != 0);
 }
@@ -68,7 +70,9 @@ __lll_timedlock_wait (int *futex, const struct timespec *abstime)
       /* Wait.  */
       int oldval = atomic_compare_and_exchange_val_acq (futex, 2, 1);
       if (oldval != 0)
-	lll_futex_timed_wait (futex, 2, &rt);
+	lll_futex_timed_wait (futex, 2, &rt,
+			      // XYZ check mutex flag
+			      LLL_SHARED);
     }
   while (atomic_compare_and_exchange_bool_acq (futex, 2, 0) != 0);
 
@@ -108,8 +112,9 @@ __lll_timedwait_tid (int *tidp, const struct timespec *abstime)
       if (rt.tv_sec < 0)
 	return ETIMEDOUT;
 
-      /* Wait until thread terminates.  */
-      if (lll_futex_timed_wait (tidp, tid, &rt) == -ETIMEDOUT)
+      /* Wait until thread terminates.  The kernel so far does not use
+	 the private futex operations for this.  */
+      if (lll_futex_timed_wait (tidp, tid, &rt, LLL_SHARED) == -ETIMEDOUT)
 	return ETIMEDOUT;
     }
 
diff --git a/nptl/sysdeps/unix/sysv/linux/lowlevelrobustlock.c b/nptl/sysdeps/unix/sysv/linux/lowlevelrobustlock.c
index 30ef991bd0..54cee0859b 100644
--- a/nptl/sysdeps/unix/sysv/linux/lowlevelrobustlock.c
+++ b/nptl/sysdeps/unix/sysv/linux/lowlevelrobustlock.c
@@ -44,7 +44,9 @@ __lll_robust_lock_wait (int *futex)
 	  && atomic_compare_and_exchange_bool_acq (futex, newval, oldval))
 	continue;
 
-      lll_futex_wait (futex, newval);
+      lll_futex_wait (futex, newval,
+		      // XYZ check mutex flag
+		      LLL_SHARED);
 
     try:
       ;
@@ -100,7 +102,9 @@ __lll_robust_timedlock_wait (int *futex, const struct timespec *abstime)
 	  && atomic_compare_and_exchange_bool_acq (futex, newval, oldval))
 	continue;
 
-      lll_futex_timed_wait (futex, newval, &rt);
+      lll_futex_timed_wait (futex, newval, &rt,
+			    // XYZ check mutex flag
+			    LLL_SHARED);
 
     try:
       ;
diff --git a/nptl/sysdeps/unix/sysv/linux/rtld-lowlevel.h b/nptl/sysdeps/unix/sysv/linux/rtld-lowlevel.h
index 6b3d3682da..e805b63001 100644
--- a/nptl/sysdeps/unix/sysv/linux/rtld-lowlevel.h
+++ b/nptl/sysdeps/unix/sysv/linux/rtld-lowlevel.h
@@ -1,5 +1,5 @@
 /* Defintions for lowlevel handling in ld.so.
-   Copyright (C) 2006 Free Software Foundation, Inc.
+   Copyright (C) 2006, 2007 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
 
    The GNU C Library is free software; you can redistribute it and/or
@@ -76,7 +76,7 @@ typedef int __rtld_mrlock_t;
 	    atomic_or (&(lock), __RTLD_MRLOCK_RWAIT);			      \
 	    oldval |= __RTLD_MRLOCK_RWAIT;				      \
 	  }								      \
-	lll_futex_wait (lock, oldval);					      \
+	lll_private_futex_wait (lock, oldval);				      \
       }									      \
   out:;									      \
   } while (0)
@@ -90,7 +90,7 @@ typedef int __rtld_mrlock_t;
 			  == (__RTLD_MRLOCK_INC | __RTLD_MRLOCK_WWAIT), 0))   \
       /* We have to wake all threads since there might be some queued	      \
 	 readers already.  */						      \
-      lll_futex_wake (&(lock), 0x7fffffff);				      \
+      lll_private_futex_wake (&(lock), 0x7fffffff);			      \
   } while (0)
 
 
@@ -119,7 +119,7 @@ typedef int __rtld_mrlock_t;
 	  }								      \
 	atomic_or (&(lock), __RTLD_MRLOCK_WWAIT);			      \
 	oldval |= __RTLD_MRLOCK_WWAIT;					      \
-	lll_futex_wait (lock, oldval);					      \
+	lll_private_futex_wait (lock, oldval);				      \
       }									      \
   out:;									      \
   } while (0)
@@ -129,7 +129,7 @@ typedef int __rtld_mrlock_t;
   do {				 \
     int oldval = atomic_exchange_and_add (&(lock), -__RTLD_MRLOCK_WRITER);    \
     if (__builtin_expect ((oldval & __RTLD_MRLOCK_RWAIT) != 0, 0))	      \
-      lll_futex_wake (&(lock), 0x7fffffff);				      \
+      lll_private_futex_wake (&(lock), 0x7fffffff);			      \
   } while (0)
 
 
@@ -142,12 +142,12 @@ typedef int __rtld_mrlock_t;
 	int val = word;							      \
 	if (val == 0)							      \
 	  break;							      \
-	lll_futex_wait (&(word), val);					      \
+	lll_private_futex_wait (&(word), val);				      \
       }									      \
   } while (0)
 
 
 #define __rtld_notify(word) \
-  lll_futex_wake (&(word), 1)
+  lll_private_futex_wake (&(word), 1)
 
 #endif
diff --git a/nptl/sysdeps/unix/sysv/linux/sem_post.c b/nptl/sysdeps/unix/sysv/linux/sem_post.c
index f8bc695292..7f90325585 100644
--- a/nptl/sysdeps/unix/sysv/linux/sem_post.c
+++ b/nptl/sysdeps/unix/sysv/linux/sem_post.c
@@ -35,7 +35,9 @@ __new_sem_post (sem_t *sem)
   atomic_full_barrier ();
   if (isem->nwaiters > 0)
     {
-      int err = lll_futex_wake (&isem->value, 1);
+      int err = lll_futex_wake (&isem->value, 1,
+				// XYZ check mutex flag
+				LLL_SHARED);
       if (__builtin_expect (err, 0) < 0)
 	{
 	  __set_errno (-err);
@@ -55,7 +57,8 @@ __old_sem_post (sem_t *sem)
   int *futex = (int *) sem;
 
   int nr = atomic_increment_val (futex);
-  int err = lll_futex_wake (futex, 1);
+  /* We always have to assume it is a shared semaphore.  */
+  int err = lll_futex_wake (futex, 1, LLL_SHARED);
   if (__builtin_expect (err, 0) < 0)
     {
       __set_errno (-err);
diff --git a/nptl/sysdeps/unix/sysv/linux/sem_timedwait.c b/nptl/sysdeps/unix/sysv/linux/sem_timedwait.c
index 4c8acd0938..8f92d78abe 100644
--- a/nptl/sysdeps/unix/sysv/linux/sem_timedwait.c
+++ b/nptl/sysdeps/unix/sysv/linux/sem_timedwait.c
@@ -84,7 +84,9 @@ sem_timedwait (sem_t *sem, const struct timespec *abstime)
       /* Enable asynchronous cancellation.  Required by the standard.  */
       int oldtype = __pthread_enable_asynccancel ();
 
-      err = lll_futex_timed_wait (&isem->value, 0, &rt);
+      err = lll_futex_timed_wait (&isem->value, 0, &rt,
+				  // XYZ check mutex flag
+				  LLL_SHARED);
 
       /* Disable asynchronous cancellation.  */
       __pthread_disable_asynccancel (oldtype);
diff --git a/nptl/sysdeps/unix/sysv/linux/sem_wait.c b/nptl/sysdeps/unix/sysv/linux/sem_wait.c
index 05114c9704..12f3f16c2d 100644
--- a/nptl/sysdeps/unix/sysv/linux/sem_wait.c
+++ b/nptl/sysdeps/unix/sysv/linux/sem_wait.c
@@ -56,7 +56,9 @@ __new_sem_wait (sem_t *sem)
       /* Enable asynchronous cancellation.  Required by the standard.  */
       int oldtype = __pthread_enable_asynccancel ();
 
-      err = lll_futex_wait (&isem->value, 0);
+      err = lll_futex_wait (&isem->value, 0,
+			    // XYZ check mutex flag
+			    LLL_SHARED);
 
       /* Disable asynchronous cancellation.  */
       __pthread_disable_asynccancel (oldtype);
@@ -100,7 +102,8 @@ __old_sem_wait (sem_t *sem)
       /* Enable asynchronous cancellation.  Required by the standard.  */
       int oldtype = __pthread_enable_asynccancel ();
 
-      err = lll_futex_wait (futex, 0);
+      /* Always assume the semaphore is shared.  */
+      err = lll_futex_wait (futex, 0, LLL_SHARED);
 
       /* Disable asynchronous cancellation.  */
       __pthread_disable_asynccancel (oldtype);
diff --git a/nptl/sysdeps/unix/sysv/linux/sparc/pthread_once.c b/nptl/sysdeps/unix/sysv/linux/sparc/pthread_once.c
index 3b07cc127d..83fedef8e8 100644
--- a/nptl/sysdeps/unix/sysv/linux/sparc/pthread_once.c
+++ b/nptl/sysdeps/unix/sysv/linux/sparc/pthread_once.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2003, 2004 Free Software Foundation, Inc.
+/* Copyright (C) 2003, 2004, 2007 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Jakub Jelinek <jakub@redhat.com>, 2003.
 
@@ -30,7 +30,7 @@ clear_once_control (void *arg)
   pthread_once_t *once_control = (pthread_once_t *) arg;
 
   *once_control = 0;
-  lll_futex_wake (once_control, INT_MAX);
+  lll_private_futex_wake (once_control, INT_MAX);
 }
 
 
@@ -84,7 +84,7 @@ __pthread_once (once_control, init_routine)
       atomic_increment (once_control);
 
       /* Wake up all other threads.  */
-      lll_futex_wake (once_control, INT_MAX);
+      lll_private_futex_wake (once_control, INT_MAX);
       break;
     }
 
diff --git a/nptl/sysdeps/unix/sysv/linux/sparc/sparc32/pthread_barrier_wait.c b/nptl/sysdeps/unix/sysv/linux/sparc/sparc32/pthread_barrier_wait.c
index 4dfd11dcbe..868e0d2819 100644
--- a/nptl/sysdeps/unix/sysv/linux/sparc/sparc32/pthread_barrier_wait.c
+++ b/nptl/sysdeps/unix/sysv/linux/sparc/sparc32/pthread_barrier_wait.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2003, 2004, 2006 Free Software Foundation, Inc.
+/* Copyright (C) 2003, 2004, 2006, 2007 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Martin Schwidefsky <schwidefsky@de.ibm.com>, 2003.
 
@@ -52,7 +52,9 @@ pthread_barrier_wait (barrier)
       ++ibarrier->b.curr_event;
 
       /* Wake up everybody.  */
-      lll_futex_wake (&ibarrier->b.curr_event, INT_MAX);
+      lll_futex_wake (&ibarrier->b.curr_event, INT_MAX,
+		      // XYZ check mutex flag
+		      LLL_SHARED);
 
       /* This is the thread which finished the serialization.  */
       result = PTHREAD_BARRIER_SERIAL_THREAD;
@@ -68,7 +70,9 @@ pthread_barrier_wait (barrier)
 
       /* Wait for the event counter of the barrier to change.  */
       do
-	lll_futex_wait (&ibarrier->b.curr_event, event);
+	lll_futex_wait (&ibarrier->b.curr_event, event,
+			// XYZ check mutex flag
+			LLL_SHARED);
       while (event == ibarrier->b.curr_event);
     }
 
diff --git a/nptl/sysdeps/unix/sysv/linux/sparc/sparc32/sem_post.c b/nptl/sysdeps/unix/sysv/linux/sparc/sparc32/sem_post.c
index be1cc60b11..527aedfdc7 100644
--- a/nptl/sysdeps/unix/sysv/linux/sparc/sparc32/sem_post.c
+++ b/nptl/sysdeps/unix/sysv/linux/sparc/sparc32/sem_post.c
@@ -1,5 +1,5 @@
 /* sem_post -- post to a POSIX semaphore.  SPARC version.
-   Copyright (C) 2003, 2004, 2006 Free Software Foundation, Inc.
+   Copyright (C) 2003, 2004, 2006, 2007 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Jakub Jelinek <jakub@redhat.com>, 2003.
 
@@ -39,7 +39,9 @@ __new_sem_post (sem_t *sem)
       nr = ++*futex;
       __sparc32_atomic_do_unlock24 (futex + 1);
     }
-  int err = lll_futex_wake (futex, nr);
+  int err = lll_futex_wake (futex, nr,
+			    // XYZ check mutex flag
+			    LLL_SHARED);
   if (__builtin_expect (err, 0) < 0)
     {
       __set_errno (-err);
diff --git a/nptl/sysdeps/unix/sysv/linux/unregister-atfork.c b/nptl/sysdeps/unix/sysv/linux/unregister-atfork.c
index 964f5b7094..240ce597f9 100644
--- a/nptl/sysdeps/unix/sysv/linux/unregister-atfork.c
+++ b/nptl/sysdeps/unix/sysv/linux/unregister-atfork.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003, 2005 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2005, 2007 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -104,7 +104,7 @@ __unregister_atfork (dso_handle)
       atomic_decrement (&deleted->handler->refcntr);
       unsigned int val;
       while ((val = deleted->handler->refcntr) != 0)
-	lll_futex_wait (&deleted->handler->refcntr, val);
+	lll_private_futex_wait (&deleted->handler->refcntr, val);
 
       deleted = deleted->next;
     }
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h
index d3055cbb23..b86d95e93d 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h
@@ -23,6 +23,8 @@
 #include <time.h>
 #include <sys/param.h>
 #include <bits/pthreadtypes.h>
+#include <kernel-features.h>
+#include <tcb-offsets.h>
 
 #ifndef LOCK_INSTR
 # ifdef UP
@@ -42,6 +44,13 @@
 #define FUTEX_PRIVATE_FLAG	128
 
 
+/* Values for 'private' parameter of locking macros.  Yes, the
+   definition seems to be backwards.  But it is not.  The bit will be
+   reversed before passing to the system call.  */
+#define LLL_PRIVATE	0
+#define LLL_SHARED	FUTEX_PRIVATE_FLAG
+
+
 /* Initializer for compatibility lock.  */
 #define LLL_MUTEX_LOCK_INITIALIZER		(0)
 #define LLL_MUTEX_LOCK_INITIALIZER_LOCKED	(1)
@@ -149,44 +158,97 @@ LLL_STUB_UNWIND_INFO_START					\
 LLL_STUB_UNWIND_INFO_END
 
 
-#define lll_futex_wait(futex, val) \
+#define lll_futex_wait(futex, val, private) \
+  lll_futex_timed_wait(futex, val, NULL, private)
+
+
+#define lll_futex_timed_wait(futex, val, timeout, private) \
   ({									      \
+    register const struct timespec *__to __asm ("r10") = timeout;	      \
     int __status;							      \
     register __typeof (val) _val __asm ("edx") = (val);			      \
-    __asm __volatile ("xorq %%r10, %%r10\n\t"				      \
-		      "syscall"						      \
+    __asm __volatile ("syscall"						      \
 		      : "=a" (__status)					      \
 		      : "0" (SYS_futex), "D" (futex), "S" (FUTEX_WAIT),	      \
-			"d" (_val)					      \
-		      : "memory", "cc", "r10", "r11", "cx");		      \
+		        "d" (_val), "r" (__to)				      \
+		      : "memory", "cc", "r11", "cx");			      \
     __status;								      \
   })
 
 
-#define lll_futex_timed_wait(futex, val, timeout)			      \
+#define lll_futex_wake(futex, nr, private) \
+  do {									      \
+    int __ignore;							      \
+    register __typeof (nr) _nr __asm ("edx") = (nr);			      \
+    __asm __volatile ("syscall"						      \
+		      : "=a" (__ignore)					      \
+		      : "0" (SYS_futex), "D" (futex), "S" (FUTEX_WAKE),	      \
+			"d" (_nr)					      \
+		      : "memory", "cc", "r10", "r11", "cx");		      \
+  } while (0)
+
+
+#define lll_private_futex_wait(futex, val) \
+  lll_private_futex_timed_wait (futex, val, NULL)
+
+
+#ifdef __ASSUME_PRIVATE_FUTEX
+# define lll_private_futex_timed_wait(futex, val, timeout) \
   ({									      \
     register const struct timespec *__to __asm ("r10") = timeout;	      \
     int __status;							      \
     register __typeof (val) _val __asm ("edx") = (val);			      \
     __asm __volatile ("syscall"						      \
 		      : "=a" (__status)					      \
-		      : "0" (SYS_futex), "D" (futex), "S" (FUTEX_WAIT),	      \
+		      : "0" (SYS_futex), "D" (futex),			      \
+			"S" (FUTEX_WAIT | FUTEX_PRIVATE_FLAG),		      \
 		        "d" (_val), "r" (__to)				      \
 		      : "memory", "cc", "r11", "cx");			      \
     __status;								      \
   })
 
 
-#define lll_futex_wake(futex, nr) \
+# define lll_private_futex_wake(futex, nr) \
   do {									      \
     int __ignore;							      \
     register __typeof (nr) _nr __asm ("edx") = (nr);			      \
     __asm __volatile ("syscall"						      \
 		      : "=a" (__ignore)					      \
-		      : "0" (SYS_futex), "D" (futex), "S" (FUTEX_WAKE),	      \
+		      : "0" (SYS_futex), "D" (futex),			      \
+			"S" (FUTEX_WAKE | FUTEX_PRIVATE_FLAG),		      \
 			"d" (_nr)					      \
 		      : "memory", "cc", "r10", "r11", "cx");		      \
   } while (0)
+#else
+# define lll_private_futex_timed_wait(futex, val, timeout) \
+  ({									      \
+    register const struct timespec *__to __asm ("r10") = timeout;	      \
+    int __status;							      \
+    int __ignore;							      \
+    register __typeof (val) _val __asm ("edx") = (val);			      \
+    __asm __volatile ("movl %%fs:%P3, %%esi\n\t"			      \
+		      "syscall"						      \
+		      : "=a" (__status), "=S" (__ignore)		      \
+		      : "0" (SYS_futex), "i" (PRIVATE_FUTEX), "D" (futex),    \
+		        "d" (_val), "r" (__to)				      \
+		      : "memory", "cc", "r11", "cx");			      \
+    __status;								      \
+  })
+
+
+# define lll_private_futex_wake(futex, nr) \
+  do {									      \
+    int __ignore;							      \
+    int __ignore2;							      \
+    register __typeof (nr) _nr __asm ("edx") = (nr);			      \
+    __asm __volatile ("orl %%fs:%P3, %%esi\n\t"				      \
+		      "syscall"						      \
+		      : "=a" (__ignore), "=S" (__ignore2)		      \
+		      : "0" (SYS_futex), "i" (PRIVATE_FUTEX), "D" (futex),    \
+			"1" (FUTEX_WAKE), "d" (_nr)			      \
+		      : "memory", "cc", "r10", "r11", "cx");		      \
+  } while (0)
+#endif
 
 
 /* Does not preserve %eax and %ecx.  */
diff --git a/nptl/sysdeps/x86_64/tls.h b/nptl/sysdeps/x86_64/tls.h
index f3532529bb..029848a8fa 100644
--- a/nptl/sysdeps/x86_64/tls.h
+++ b/nptl/sysdeps/x86_64/tls.h
@@ -355,7 +355,7 @@ typedef struct
 		    : "i" (offsetof (struct pthread, header.gscope_flag)),    \
 		      "0" (THREAD_GSCOPE_FLAG_UNUSED));			      \
       if (__res == THREAD_GSCOPE_FLAG_WAIT)				      \
-	lll_futex_wake (&THREAD_SELF->header.gscope_flag, 1);		      \
+	lll_private_futex_wake (&THREAD_SELF->header.gscope_flag, 1);	      \
     }									      \
   while (0)
 #define THREAD_GSCOPE_SET_FLAG() \