diff options
Diffstat (limited to 'nptl')
| -rw-r--r-- | nptl/ChangeLog | 17 | ||||
| -rw-r--r-- | nptl/pthreadP.h | 2 | ||||
| -rw-r--r-- | nptl/pthread_mutex_lock.c | 21 | ||||
| -rw-r--r-- | nptl/pthread_mutex_unlock.c | 7 | ||||
| -rw-r--r-- | nptl/sysdeps/unix/sysv/linux/pthread-pi-defines.sym | 1 | ||||
| -rw-r--r-- | nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S | 6 | ||||
| -rw-r--r-- | nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S | 6 | ||||
| -rw-r--r-- | nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S | 24 | ||||
| -rw-r--r-- | nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S | 23 |
9 files changed, 83 insertions, 24 deletions
diff --git a/nptl/ChangeLog b/nptl/ChangeLog index 8dd93732b5..8f37da7936 100644 --- a/nptl/ChangeLog +++ b/nptl/ChangeLog @@ -1,3 +1,20 @@ +2009-07-28 Ulrich Drepper <drepper@redhat.com> + + * pthread_mutex_lock.c [NO_INCR] (__pthread_mutex_cond_lock_adjust): + New function. + * pthreadP.h: Declare __pthread_mutex_cond_lock_adjust. + * sysdeps/unix/sysv/linux/pthread-pi-defines.sym: Add ROBUST_BIT. + * sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S: Don't use + requeue_pi for robust mutexes. + * sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S: Likewise. + * sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S: Likewise. + Don't only skip __pthread_mutex_cond_lock. Call instead + __pthread_mutex_cond_lock_adjust. + * sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S: Likewise. + + * pthread_mutex_unlock.c (__pthread_mutex_unlock_full): Minor + optimization of PI mutex handling. + 2009-07-27 Ulrich Drepper <drepper@redhat.com> [BZ #10418] diff --git a/nptl/pthreadP.h b/nptl/pthreadP.h index ed9fc625ba..43ca44c829 100644 --- a/nptl/pthreadP.h +++ b/nptl/pthreadP.h @@ -418,6 +418,8 @@ extern int __pthread_mutex_lock_internal (pthread_mutex_t *__mutex) attribute_hidden; extern int __pthread_mutex_cond_lock (pthread_mutex_t *__mutex) attribute_hidden internal_function; +extern void __pthread_mutex_cond_lock_adjust (pthread_mutex_t *__mutex) + attribute_hidden internal_function; extern int __pthread_mutex_unlock (pthread_mutex_t *__mutex); extern int __pthread_mutex_unlock_internal (pthread_mutex_t *__mutex) attribute_hidden; diff --git a/nptl/pthread_mutex_lock.c b/nptl/pthread_mutex_lock.c index 406e588fdb..50dc18803d 100644 --- a/nptl/pthread_mutex_lock.c +++ b/nptl/pthread_mutex_lock.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2002-2007, 2008 Free Software Foundation, Inc. +/* Copyright (C) 2002-2007, 2008, 2009 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. @@ -473,3 +473,22 @@ __pthread_mutex_lock_full (pthread_mutex_t *mutex) strong_alias (__pthread_mutex_lock, pthread_mutex_lock) strong_alias (__pthread_mutex_lock, __pthread_mutex_lock_internal) #endif + + +#ifdef NO_INCR +void +__pthread_mutex_cond_lock_adjust (mutex) + pthread_mutex_t *mutex; +{ + assert ((mutex->__data.__kind & PTHREAD_MUTEX_PRIO_INHERIT_NP) != 0); + assert ((mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP) == 0); + assert ((mutex->__data.__kind & PTHREAD_MUTEX_PSHARED_BIT) == 0); + + /* Record the ownership. */ + pid_t id = THREAD_GETMEM (THREAD_SELF, tid); + mutex->__data.__owner = id; + + if (mutex->__data.__kind == PTHREAD_MUTEX_PI_RECURSIVE_NP) + ++mutex->__data.__count; +} +#endif diff --git a/nptl/pthread_mutex_unlock.c b/nptl/pthread_mutex_unlock.c index fbe8274a55..f9fe10b0f2 100644 --- a/nptl/pthread_mutex_unlock.c +++ b/nptl/pthread_mutex_unlock.c @@ -150,7 +150,7 @@ __pthread_mutex_unlock_full (pthread_mutex_t *mutex, int decr) if (--mutex->__data.__count != 0) /* We still hold the mutex. */ return 0; - goto continue_pi; + goto continue_pi_non_robust; case PTHREAD_MUTEX_PI_ROBUST_RECURSIVE_NP: /* Recursive mutex. */ @@ -173,7 +173,7 @@ __pthread_mutex_unlock_full (pthread_mutex_t *mutex, int decr) /* We still hold the mutex. */ return 0; - goto continue_pi; + goto continue_pi_robust; case PTHREAD_MUTEX_PI_ERRORCHECK_NP: case PTHREAD_MUTEX_PI_NORMAL_NP: @@ -195,9 +195,9 @@ __pthread_mutex_unlock_full (pthread_mutex_t *mutex, int decr) pi_notrecoverable: newowner = PTHREAD_MUTEX_NOTRECOVERABLE; - continue_pi: if ((mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP) != 0) { + continue_pi_robust: /* Remove mutex from the list. Note: robust PI futexes are signaled by setting bit 0. */ THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, @@ -206,6 +206,7 @@ __pthread_mutex_unlock_full (pthread_mutex_t *mutex, int decr) DEQUEUE_MUTEX (mutex); } + continue_pi_non_robust: mutex->__data.__owner = newowner; if (decr) /* One less user. */ diff --git a/nptl/sysdeps/unix/sysv/linux/pthread-pi-defines.sym b/nptl/sysdeps/unix/sysv/linux/pthread-pi-defines.sym index d985c6a79b..46fbd0de74 100644 --- a/nptl/sysdeps/unix/sysv/linux/pthread-pi-defines.sym +++ b/nptl/sysdeps/unix/sysv/linux/pthread-pi-defines.sym @@ -3,5 +3,6 @@ -- These PI macros are used by assembly code. MUTEX_KIND offsetof (pthread_mutex_t, __data.__kind) +ROBUST_BIT PTHREAD_MUTEX_ROBUST_NORMAL_NP PI_BIT PTHREAD_MUTEX_PRIO_INHERIT_NP PS_BIT PTHREAD_MUTEX_PSHARED_BIT diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S index 0f10ec910c..224a56088e 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S @@ -75,8 +75,10 @@ __pthread_cond_broadcast: jne 9f /* Requeue to a PI mutex if the PI bit is set. */ - testl $PI_BIT, MUTEX_KIND(%r8) - jne 81f + movl MUTEX_KIND(%r8), %eax + andl $(ROBUST_BIT|PI_BIT), %eax + cmpl $PI_BIT, %eax + je 81f /* Wake up all threads. */ #ifdef __ASSUME_PRIVATE_FUTEX diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S index f1050fea7c..4d001eec7f 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S @@ -64,8 +64,10 @@ __pthread_cond_signal: /* Get the address of the mutex used. */ movq dep_mutex(%r8), %rcx - testl $PI_BIT, MUTEX_KIND(%rcx) - jne 9f + movl MUTEX_KIND(%rcx), %eax + andl $(ROBUST_BIT|PI_BIT), %eax + cmpl $PI_BIT, %eax + je 9f #ifdef __ASSUME_PRIVATE_FUTEX movl $(FUTEX_WAKE_OP|FUTEX_PRIVATE_FLAG), %esi diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S index 7486825d5f..4913beb8af 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S @@ -165,9 +165,12 @@ __pthread_cond_timedwait: je 60f movq dep_mutex(%rdi), %r8 - /* Requeue to a PI mutex if the PI bit is set. */ - testl $PI_BIT, MUTEX_KIND(%r8) - je 61f + /* Requeue to a non-robust PI mutex if the PI bit is set and + the robust bit is not set. */ + movl MUTEX_KIND(%r8), %eax + andl $(ROBUST_BIT|PI_BIT), %eax + cmpl $PI_BIT, %eax + jne 61f movl $(FUTEX_WAIT_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %esi xorl %eax, %eax @@ -289,11 +292,10 @@ __pthread_cond_timedwait: /* If requeue_pi is used the kernel performs the locking of the mutex. */ -41: xorl %eax, %eax +41: movq 16(%rsp), %rdi testl %r15d, %r15d - jnz 63f + jnz 64f - movq 16(%rsp), %rdi callq __pthread_mutex_cond_lock 63: testq %rax, %rax @@ -316,12 +318,18 @@ __pthread_cond_timedwait: retq - /* Initial locking failed. */ -31: cfi_adjust_cfa_offset(4 * 8 + FRAME_SIZE) + cfi_adjust_cfa_offset(4 * 8 + FRAME_SIZE) cfi_rel_offset(%r12, FRAME_SIZE + 24) cfi_rel_offset(%r13, FRAME_SIZE + 16) cfi_rel_offset(%r14, FRAME_SIZE + 8) cfi_rel_offset(%r15, FRAME_SIZE) + +64: callq __pthread_mutex_cond_lock_adjust + movq %r14, %rax + jmp 48b + + /* Initial locking failed. */ +31: #if cond_lock != 0 addq $cond_lock, %rdi #endif diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S index 2fab38e277..a66523eab6 100644 --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S @@ -134,9 +134,12 @@ __pthread_cond_wait: je 60f movq dep_mutex-cond_futex(%rdi), %r8 - /* Requeue to a PI mutex if the PI bit is set. */ - testl $PI_BIT, MUTEX_KIND(%r8) - je 61f + /* Requeue to a non-robust PI mutex if the PI bit is set and + the robust bit is not set. */ + movl MUTEX_KIND(%r8), %eax + andl $(ROBUST_BIT|PI_BIT), %eax + cmpl $PI_BIT, %eax + jne 61f movl $(FUTEX_WAIT_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %esi movl $SYS_futex, %eax @@ -234,11 +237,10 @@ __pthread_cond_wait: /* If requeue_pi is used the kernel performs the locking of the mutex. */ -11: xorl %eax, %eax +11: movq 16(%rsp), %rdi testl %r13d, %r13d - jnz 14f + jnz 18f - movq 16(%rsp), %rdi callq __pthread_mutex_cond_lock 14: addq $FRAME_SIZE, %rsp @@ -254,11 +256,16 @@ __pthread_cond_wait: /* We return the result of the mutex_lock operation. */ retq - /* Initial locking failed. */ -1: cfi_adjust_cfa_offset(16 + FRAME_SIZE) cfi_rel_offset(%r12, FRAME_SIZE + 8) cfi_rel_offset(%r13, FRAME_SIZE) + +18: callq __pthread_mutex_cond_lock_adjust + xorl %eax, %eax + jmp 14b + + /* Initial locking failed. */ +1: #if cond_lock != 0 addq $cond_lock, %rdi #endif |