diff options
Diffstat (limited to 'libc/nptl')
30 files changed, 861 insertions, 30 deletions
diff --git a/libc/nptl/ChangeLog b/libc/nptl/ChangeLog index 7c0354858..064e431c1 100644 --- a/libc/nptl/ChangeLog +++ b/libc/nptl/ChangeLog @@ -1,3 +1,74 @@ +2013-07-03 H.J. Lu <hongjiu.lu@intel.com> + + * sysdeps/unix/sysv/linux/x86/init-arch.c: New file. + * sysdeps/unix/sysv/linux/x86/init-arch.h: Likewise. + +2013-07-02 Andi Kleen <ak@linux.intel.com> + + * sysdeps/unix/sysv/linux/x86/elision-conf.c (elision_init): + Check ENABLE_LOCK_ELISION. + +2013-07-02 Andi Kleen <ak@linux.intel.com> + + * pthread_mutexattr_settype.c (__pthread_mutexattr_settype): + Disable elision for PTHREAD_MUTEX_DEFAULT. + +2013-07-02 Andi Kleen <ak@linux.intel.com> + Hongjiu Lu <hongjiu.lu@intel.com> + + * pthread_mutex_lock.c + (__pthread_mutex_lock): Add lock elision support. + * pthread_mutex_timedlock.c (pthread_mutex_timedlock): Likewise. + * pthread_mutex_trylock.c (__pthread_mutex_trylock): Likewise. + * pthread_mutex_unlock.c (__pthread_mutex_unlock_usercnt): Likewise. + * sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c: Likewise. + * sysdeps/unix/sysv/linux/x86/bits/pthreadtypes.h: Likewise. + * sysdeps/unix/sysv/linux/x86/Makefile: New file. + * sysdeps/unix/sysv/linux/x86/force-elision.h: New file + * sysdeps/unix/sysv/linux/x86/pthread_mutex_cond_lock.c: Likewise. + * sysdeps/unix/sysv/linux/x86/pthread_mutex_lock.c: Likewise. + * sysdeps/unix/sysv/linux/x86/pthread_mutex_timedlock.c: Likewise. + * sysdeps/unix/sysv/linux/x86/pthread_mutex_trylock.c: Likewise. + * sysdeps/unix/sysv/linux/x86/pthread_mutex_unlock.c: Likewise. + +2013-07-02 Andi Kleen <ak@linux.intel.com> + + * tst-mutex5.c: Include config.h. + (do_test): Add checks for ENABLE_LOCK_ELISION. + * tst-mutex8.c: Include config.h + (tf): Add checks for ENABLE_LOCK_ELISION. + (check_type): Likewise. + +2013-07-02 Andi Kleen <ak@linux.intel.com> + + * pthreadP.h: Add elision types. + (PTHREAD_MUTEX_TYPE_ELISION): Add. + * sysdeps/pthread/pthread.h: Add elision initializers. + (PTHREAD_MUTEX_ELISION_NP, PTHREAD_MUTEX_NO_ELISION_NP, + PTHREAD_MUTEX_PSHARED_NP): Add new flags. + (__PTHREAD_SPINS): Add. + +2013-07-02 Andi Kleen <ak@linux.intel.com> + Hongjiu Lu <hongjiu.lu@intel.com> + + * sysdeps/unix/sysv/linux/i386/lowlevellock.h (__lll_timedwait_tid, + lll_timedlock_elision, __lll_lock_elision, __lll_unlock_elision, + __lll_trylock_elision, lll_lock_elision, lll_unlock_elision, + lll_trylock_elision): Add. + * sysdeps/unix/sysv/linux/x86/Makefile: Imply x86. + * sysdeps/unix/sysv/linux/x86/elision-conf.c: New file. + * sysdeps/unix/sysv/linux/x86/elision-conf.h: New file. + * sysdeps/unix/sysv/linux/x86/elision-lock.c: New file. + * sysdeps/unix/sysv/linux/x86/elision-timed.c: New file. + * sysdeps/unix/sysv/linux/x86/elision-trylock.c: New file. + * sysdeps/unix/sysv/linux/x86/elision-unlock.c: New file. + * sysdeps/unix/sysv/linux/x86_64/lowlevellock.h (__lll_timedwait_tid, + lll_timedlock_elision, __lll_lock_elision, __lll_unlock_elision, + __lll_trylock_elision, lll_lock_elision, lll_unlock_elision, + lll_trylock_elision): Add. + * nptl/sysdeps/unix/sysv/linux/x86/hle.h: New file. + * elision-conf.h: New file. + 2013-06-24 Vladimir Nikulichev <v.nikulichev@gmail.com> [BZ #12310] diff --git a/libc/nptl/elision-conf.h b/libc/nptl/elision-conf.h new file mode 100644 index 000000000..40a8c178f --- /dev/null +++ b/libc/nptl/elision-conf.h @@ -0,0 +1 @@ +/* empty */ diff --git a/libc/nptl/pthreadP.h b/libc/nptl/pthreadP.h index 7883fdfa1..789bbf6ae 100644 --- a/libc/nptl/pthreadP.h +++ b/libc/nptl/pthreadP.h @@ -61,6 +61,10 @@ enum { PTHREAD_MUTEX_KIND_MASK_NP = 3, + + PTHREAD_MUTEX_ELISION_NP = 256, + PTHREAD_MUTEX_NO_ELISION_NP = 512, + PTHREAD_MUTEX_ROBUST_NORMAL_NP = 16, PTHREAD_MUTEX_ROBUST_RECURSIVE_NP = PTHREAD_MUTEX_ROBUST_NORMAL_NP | PTHREAD_MUTEX_RECURSIVE_NP, @@ -93,12 +97,23 @@ enum PTHREAD_MUTEX_PP_ERRORCHECK_NP = PTHREAD_MUTEX_PRIO_PROTECT_NP | PTHREAD_MUTEX_ERRORCHECK_NP, PTHREAD_MUTEX_PP_ADAPTIVE_NP - = PTHREAD_MUTEX_PRIO_PROTECT_NP | PTHREAD_MUTEX_ADAPTIVE_NP + = PTHREAD_MUTEX_PRIO_PROTECT_NP | PTHREAD_MUTEX_ADAPTIVE_NP, + PTHREAD_MUTEX_ELISION_FLAGS_NP + = PTHREAD_MUTEX_ELISION_NP | PTHREAD_MUTEX_NO_ELISION_NP, + + PTHREAD_MUTEX_TIMED_ELISION_NP = + PTHREAD_MUTEX_TIMED_NP | PTHREAD_MUTEX_ELISION_NP, + PTHREAD_MUTEX_TIMED_NO_ELISION_NP = + PTHREAD_MUTEX_TIMED_NP | PTHREAD_MUTEX_NO_ELISION_NP, }; #define PTHREAD_MUTEX_PSHARED_BIT 128 #define PTHREAD_MUTEX_TYPE(m) \ ((m)->__data.__kind & 127) +/* Don't include NO_ELISION, as that type is always the same + as the underlying lock type. */ +#define PTHREAD_MUTEX_TYPE_ELISION(m) \ + ((m)->__data.__kind & (127|PTHREAD_MUTEX_ELISION_NP)) #if LLL_PRIVATE == 0 && LLL_SHARED == 128 # define PTHREAD_MUTEX_PSHARED(m) \ diff --git a/libc/nptl/pthread_mutex_lock.c b/libc/nptl/pthread_mutex_lock.c index fbedfd7d3..b37f39a0a 100644 --- a/libc/nptl/pthread_mutex_lock.c +++ b/libc/nptl/pthread_mutex_lock.c @@ -25,6 +25,14 @@ #include <lowlevellock.h> #include <stap-probe.h> +#ifndef lll_lock_elision +#define lll_lock_elision(lock, try_lock, private) ({ \ + lll_lock (lock, private); 0; }) +#endif + +#ifndef lll_trylock_elision +#define lll_trylock_elision(a,t) lll_trylock(a) +#endif #ifndef LLL_MUTEX_LOCK # define LLL_MUTEX_LOCK(mutex) \ @@ -34,39 +42,60 @@ # define LLL_ROBUST_MUTEX_LOCK(mutex, id) \ lll_robust_lock ((mutex)->__data.__lock, id, \ PTHREAD_ROBUST_MUTEX_PSHARED (mutex)) +# define LLL_MUTEX_LOCK_ELISION(mutex) \ + lll_lock_elision ((mutex)->__data.__lock, (mutex)->__data.__elision, \ + PTHREAD_MUTEX_PSHARED (mutex)) +# define LLL_MUTEX_TRYLOCK_ELISION(mutex) \ + lll_trylock_elision((mutex)->__data.__lock, (mutex)->__data.__elision, \ + PTHREAD_MUTEX_PSHARED (mutex)) #endif +#ifndef FORCE_ELISION +#define FORCE_ELISION(m, s) +#endif static int __pthread_mutex_lock_full (pthread_mutex_t *mutex) __attribute_noinline__; - int __pthread_mutex_lock (mutex) pthread_mutex_t *mutex; { assert (sizeof (mutex->__size) >= sizeof (mutex->__data)); - unsigned int type = PTHREAD_MUTEX_TYPE (mutex); + unsigned int type = PTHREAD_MUTEX_TYPE_ELISION (mutex); LIBC_PROBE (mutex_entry, 1, mutex); - if (__builtin_expect (type & ~PTHREAD_MUTEX_KIND_MASK_NP, 0)) + if (__builtin_expect (type & ~(PTHREAD_MUTEX_KIND_MASK_NP + | PTHREAD_MUTEX_ELISION_FLAGS_NP), 0)) return __pthread_mutex_lock_full (mutex); - pid_t id = THREAD_GETMEM (THREAD_SELF, tid); - - if (__builtin_expect (type, PTHREAD_MUTEX_TIMED_NP) - == PTHREAD_MUTEX_TIMED_NP) + if (__builtin_expect (type == PTHREAD_MUTEX_TIMED_NP, 1)) { + FORCE_ELISION (mutex, goto elision); simple: /* Normal mutex. */ LLL_MUTEX_LOCK (mutex); assert (mutex->__data.__owner == 0); } - else if (__builtin_expect (type == PTHREAD_MUTEX_RECURSIVE_NP, 1)) +#ifdef HAVE_ELISION + else if (__builtin_expect (type == PTHREAD_MUTEX_TIMED_ELISION_NP, 1)) + { + elision: __attribute__((unused)) + /* This case can never happen on a system without elision, + as the mutex type initialization functions will not + allow to set the elision flags. */ + /* Don't record owner or users for elision case. This is a + tail call. */ + return LLL_MUTEX_LOCK_ELISION (mutex); + } +#endif + else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex) + == PTHREAD_MUTEX_RECURSIVE_NP, 1)) { /* Recursive mutex. */ + pid_t id = THREAD_GETMEM (THREAD_SELF, tid); /* Check whether we already hold the mutex. */ if (mutex->__data.__owner == id) @@ -87,7 +116,8 @@ __pthread_mutex_lock (mutex) assert (mutex->__data.__owner == 0); mutex->__data.__count = 1; } - else if (__builtin_expect (type == PTHREAD_MUTEX_ADAPTIVE_NP, 1)) + else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex) + == PTHREAD_MUTEX_ADAPTIVE_NP, 1)) { if (! __is_smp) goto simple; @@ -117,13 +147,16 @@ __pthread_mutex_lock (mutex) } else { - assert (type == PTHREAD_MUTEX_ERRORCHECK_NP); + pid_t id = THREAD_GETMEM (THREAD_SELF, tid); + assert (PTHREAD_MUTEX_TYPE (mutex) == PTHREAD_MUTEX_ERRORCHECK_NP); /* Check whether we already hold the mutex. */ if (__builtin_expect (mutex->__data.__owner == id, 0)) return EDEADLK; goto simple; } + pid_t id = THREAD_GETMEM (THREAD_SELF, tid); + /* Record the ownership. */ mutex->__data.__owner = id; #ifndef NO_INCR diff --git a/libc/nptl/pthread_mutex_timedlock.c b/libc/nptl/pthread_mutex_timedlock.c index 3a36424af..689cefa2d 100644 --- a/libc/nptl/pthread_mutex_timedlock.c +++ b/libc/nptl/pthread_mutex_timedlock.c @@ -25,6 +25,17 @@ #include <stap-probe.h> +#ifndef lll_timedlock_elision +#define lll_timedlock_elision(a,dummy,b,c) lll_timedlock(a, b, c) +#endif + +#ifndef lll_trylock_elision +#define lll_trylock_elision(a,t) lll_trylock(a) +#endif + +#ifndef FORCE_ELISION +#define FORCE_ELISION(m, s) +#endif int pthread_mutex_timedlock (mutex, abstime) @@ -40,10 +51,11 @@ pthread_mutex_timedlock (mutex, abstime) /* We must not check ABSTIME here. If the thread does not block abstime must not be checked for a valid value. */ - switch (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex), + switch (__builtin_expect (PTHREAD_MUTEX_TYPE_ELISION (mutex), PTHREAD_MUTEX_TIMED_NP)) { /* Recursive mutex. */ + case PTHREAD_MUTEX_RECURSIVE_NP|PTHREAD_MUTEX_ELISION_NP: case PTHREAD_MUTEX_RECURSIVE_NP: /* Check whether we already hold the mutex. */ if (mutex->__data.__owner == id) @@ -78,12 +90,22 @@ pthread_mutex_timedlock (mutex, abstime) /* FALLTHROUGH */ case PTHREAD_MUTEX_TIMED_NP: + FORCE_ELISION (mutex, goto elision); simple: /* Normal mutex. */ result = lll_timedlock (mutex->__data.__lock, abstime, PTHREAD_MUTEX_PSHARED (mutex)); break; + case PTHREAD_MUTEX_TIMED_ELISION_NP: + elision: __attribute__((unused)) + /* Don't record ownership */ + return lll_timedlock_elision (mutex->__data.__lock, + mutex->__data.__spins, + abstime, + PTHREAD_MUTEX_PSHARED (mutex)); + + case PTHREAD_MUTEX_ADAPTIVE_NP: if (! __is_smp) goto simple; diff --git a/libc/nptl/pthread_mutex_trylock.c b/libc/nptl/pthread_mutex_trylock.c index 8f5279d2f..600c8483b 100644 --- a/libc/nptl/pthread_mutex_trylock.c +++ b/libc/nptl/pthread_mutex_trylock.c @@ -22,6 +22,16 @@ #include "pthreadP.h" #include <lowlevellock.h> +#ifndef lll_trylock_elision +#define lll_trylock_elision(a,t) lll_trylock(a) +#endif + +#ifndef DO_ELISION +#define DO_ELISION(m) 0 +#endif + +/* We don't force elision in trylock, because this can lead to inconsistent + lock state if the lock was actually busy. */ int __pthread_mutex_trylock (mutex) @@ -30,10 +40,11 @@ __pthread_mutex_trylock (mutex) int oldval; pid_t id = THREAD_GETMEM (THREAD_SELF, tid); - switch (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex), + switch (__builtin_expect (PTHREAD_MUTEX_TYPE_ELISION (mutex), PTHREAD_MUTEX_TIMED_NP)) { /* Recursive mutex. */ + case PTHREAD_MUTEX_RECURSIVE_NP|PTHREAD_MUTEX_ELISION_NP: case PTHREAD_MUTEX_RECURSIVE_NP: /* Check whether we already hold the mutex. */ if (mutex->__data.__owner == id) @@ -57,10 +68,20 @@ __pthread_mutex_trylock (mutex) } break; - case PTHREAD_MUTEX_ERRORCHECK_NP: + case PTHREAD_MUTEX_TIMED_ELISION_NP: + elision: + if (lll_trylock_elision (mutex->__data.__lock, + mutex->__data.__elision) != 0) + break; + /* Don't record the ownership. */ + return 0; + case PTHREAD_MUTEX_TIMED_NP: + if (DO_ELISION (mutex)) + goto elision; + /*FALL THROUGH*/ case PTHREAD_MUTEX_ADAPTIVE_NP: - /* Normal mutex. */ + case PTHREAD_MUTEX_ERRORCHECK_NP: if (lll_trylock (mutex->__data.__lock) != 0) break; @@ -378,4 +399,9 @@ __pthread_mutex_trylock (mutex) return EBUSY; } + +#ifndef __pthread_mutex_trylock +#ifndef pthread_mutex_trylock strong_alias (__pthread_mutex_trylock, pthread_mutex_trylock) +#endif +#endif diff --git a/libc/nptl/pthread_mutex_unlock.c b/libc/nptl/pthread_mutex_unlock.c index c0249f76e..691450362 100644 --- a/libc/nptl/pthread_mutex_unlock.c +++ b/libc/nptl/pthread_mutex_unlock.c @@ -23,6 +23,10 @@ #include <lowlevellock.h> #include <stap-probe.h> +#ifndef lll_unlock_elision +#define lll_unlock_elision(a,b) ({ lll_unlock (a,b); 0; }) +#endif + static int internal_function __pthread_mutex_unlock_full (pthread_mutex_t *mutex, int decr) @@ -34,8 +38,9 @@ __pthread_mutex_unlock_usercnt (mutex, decr) pthread_mutex_t *mutex; int decr; { - int type = PTHREAD_MUTEX_TYPE (mutex); - if (__builtin_expect (type & ~PTHREAD_MUTEX_KIND_MASK_NP, 0)) + int type = PTHREAD_MUTEX_TYPE_ELISION (mutex); + if (__builtin_expect (type & + ~(PTHREAD_MUTEX_KIND_MASK_NP|PTHREAD_MUTEX_ELISION_FLAGS_NP), 0)) return __pthread_mutex_unlock_full (mutex, decr); if (__builtin_expect (type, PTHREAD_MUTEX_TIMED_NP) @@ -55,7 +60,14 @@ __pthread_mutex_unlock_usercnt (mutex, decr) return 0; } - else if (__builtin_expect (type == PTHREAD_MUTEX_RECURSIVE_NP, 1)) + else if (__builtin_expect (type == PTHREAD_MUTEX_TIMED_ELISION_NP, 1)) + { + /* Don't reset the owner/users fields for elision. */ + return lll_unlock_elision (mutex->__data.__lock, + PTHREAD_MUTEX_PSHARED (mutex)); + } + else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex) + == PTHREAD_MUTEX_RECURSIVE_NP, 1)) { /* Recursive mutex. */ if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid)) @@ -66,7 +78,8 @@ __pthread_mutex_unlock_usercnt (mutex, decr) return 0; goto normal; } - else if (__builtin_expect (type == PTHREAD_MUTEX_ADAPTIVE_NP, 1)) + else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex) + == PTHREAD_MUTEX_ADAPTIVE_NP, 1)) goto normal; else { diff --git a/libc/nptl/pthread_mutexattr_settype.c b/libc/nptl/pthread_mutexattr_settype.c index 7b476e95b..a85cdcb38 100644 --- a/libc/nptl/pthread_mutexattr_settype.c +++ b/libc/nptl/pthread_mutexattr_settype.c @@ -30,6 +30,11 @@ __pthread_mutexattr_settype (attr, kind) if (kind < PTHREAD_MUTEX_NORMAL || kind > PTHREAD_MUTEX_ADAPTIVE_NP) return EINVAL; + /* Cannot distinguish between DEFAULT and NORMAL. So any settype + call disables elision for now. */ + if (kind == PTHREAD_MUTEX_DEFAULT) + kind |= PTHREAD_MUTEX_NO_ELISION_NP; + iattr = (struct pthread_mutexattr *) attr; iattr->mutexkind = (iattr->mutexkind & PTHREAD_MUTEXATTR_FLAG_BITS) | kind; diff --git a/libc/nptl/sysdeps/pthread/pthread.h b/libc/nptl/sysdeps/pthread/pthread.h index ded5ae59f..61d534687 100644 --- a/libc/nptl/sysdeps/pthread/pthread.h +++ b/libc/nptl/sysdeps/pthread/pthread.h @@ -83,27 +83,39 @@ enum /* Mutex initializers. */ +#if __PTHREAD_MUTEX_HAVE_ELISION == 1 /* 64bit layout. */ +#define __PTHREAD_SPINS 0, 0 +#elif __PTHREAD_MUTEX_HAVE_ELISION == 2 /* 32bit layout. */ +#define __PTHREAD_SPINS { 0, 0 } +#else +#define __PTHREAD_SPINS 0 +#endif + #ifdef __PTHREAD_MUTEX_HAVE_PREV # define PTHREAD_MUTEX_INITIALIZER \ - { { 0, 0, 0, 0, 0, 0, { 0, 0 } } } + { { 0, 0, 0, 0, 0, __PTHREAD_SPINS, { 0, 0 } } } # ifdef __USE_GNU # define PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP \ - { { 0, 0, 0, 0, PTHREAD_MUTEX_RECURSIVE_NP, 0, { 0, 0 } } } + { { 0, 0, 0, 0, PTHREAD_MUTEX_RECURSIVE_NP, __PTHREAD_SPINS, { 0, 0 } } } # define PTHREAD_ERRORCHECK_MUTEX_INITIALIZER_NP \ - { { 0, 0, 0, 0, PTHREAD_MUTEX_ERRORCHECK_NP, 0, { 0, 0 } } } + { { 0, 0, 0, 0, PTHREAD_MUTEX_ERRORCHECK_NP, __PTHREAD_SPINS, { 0, 0 } } } # define PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP \ - { { 0, 0, 0, 0, PTHREAD_MUTEX_ADAPTIVE_NP, 0, { 0, 0 } } } + { { 0, 0, 0, 0, PTHREAD_MUTEX_ADAPTIVE_NP, __PTHREAD_SPINS, { 0, 0 } } } +# define PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP \ + { { 0, 0, 0, 0, PTHREAD_MUTEX_ADAPTIVE_NP, __PTHREAD_SPINS, { 0, 0 } } } + # endif #else # define PTHREAD_MUTEX_INITIALIZER \ - { { 0, 0, 0, 0, 0, { 0 } } } + { { 0, 0, 0, 0, 0, { __PTHREAD_SPINS } } } # ifdef __USE_GNU # define PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP \ - { { 0, 0, 0, PTHREAD_MUTEX_RECURSIVE_NP, 0, { 0 } } } + { { 0, 0, 0, PTHREAD_MUTEX_RECURSIVE_NP, 0, { __PTHREAD_SPINS } } } # define PTHREAD_ERRORCHECK_MUTEX_INITIALIZER_NP \ - { { 0, 0, 0, PTHREAD_MUTEX_ERRORCHECK_NP, 0, { 0 } } } + { { 0, 0, 0, PTHREAD_MUTEX_ERRORCHECK_NP, 0, { __PTHREAD_SPINS } } } # define PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP \ - { { 0, 0, 0, PTHREAD_MUTEX_ADAPTIVE_NP, 0, { 0 } } } + { { 0, 0, 0, PTHREAD_MUTEX_ADAPTIVE_NP, 0, { __PTHREAD_SPINS } } } + # endif #endif diff --git a/libc/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h b/libc/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h index f665ac9e6..cc3ec5ba2 100644 --- a/libc/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h +++ b/libc/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h @@ -430,6 +430,12 @@ LLL_STUB_UNWIND_INFO_END : "memory"); \ result; }) +extern int __lll_timedlock_elision (int *futex, short *adapt_count, + const struct timespec *timeout, + int private) attribute_hidden; + +#define lll_timedlock_elision(futex, adapt_count, timeout, private) \ + __lll_timedlock_elision(&(futex), &(adapt_count), timeout, private) #define lll_robust_timedlock(futex, timeout, id, private) \ ({ int result, ignore1, ignore2, ignore3; \ @@ -583,6 +589,22 @@ extern int __lll_timedwait_tid (int *tid, const struct timespec *abstime) } \ __result; }) +extern int __lll_lock_elision (int *futex, short *adapt_count, int private) + attribute_hidden; + +extern int __lll_unlock_elision(int *lock, int private) + attribute_hidden; + +extern int __lll_trylock_elision(int *lock, short *adapt_count) + attribute_hidden; + +#define lll_lock_elision(futex, adapt_count, private) \ + __lll_lock_elision (&(futex), &(adapt_count), private) +#define lll_unlock_elision(futex, private) \ + __lll_unlock_elision (&(futex), private) +#define lll_trylock_elision(futex, adapt_count) \ + __lll_trylock_elision(&(futex), &(adapt_count)) + #endif /* !__ASSEMBLER__ */ #endif /* lowlevellock.h */ diff --git a/libc/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c b/libc/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c index b417da5e2..7b6fbc18a 100644 --- a/libc/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c +++ b/libc/nptl/sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c @@ -2,8 +2,15 @@ #define LLL_MUTEX_LOCK(mutex) \ lll_cond_lock ((mutex)->__data.__lock, PTHREAD_MUTEX_PSHARED (mutex)) + +/* Not actually elided so far. Needed? */ +#define LLL_MUTEX_LOCK_ELISION(mutex) \ + ({ lll_cond_lock ((mutex)->__data.__lock, PTHREAD_MUTEX_PSHARED (mutex)); 0; }) + #define LLL_MUTEX_TRYLOCK(mutex) \ lll_cond_trylock ((mutex)->__data.__lock) +#define LLL_MUTEX_TRYLOCK_ELISION(mutex) LLL_MUTEX_TRYLOCK(mutex) + #define LLL_ROBUST_MUTEX_LOCK(mutex, id) \ lll_robust_cond_lock ((mutex)->__data.__lock, id, \ PTHREAD_ROBUST_MUTEX_PSHARED (mutex)) diff --git a/libc/nptl/sysdeps/unix/sysv/linux/x86/Makefile b/libc/nptl/sysdeps/unix/sysv/linux/x86/Makefile new file mode 100644 index 000000000..61b7552bc --- /dev/null +++ b/libc/nptl/sysdeps/unix/sysv/linux/x86/Makefile @@ -0,0 +1,3 @@ +libpthread-sysdep_routines += init-arch +libpthread-sysdep_routines += elision-lock elision-unlock elision-timed \ + elision-trylock diff --git a/libc/nptl/sysdeps/unix/sysv/linux/x86/bits/pthreadtypes.h b/libc/nptl/sysdeps/unix/sysv/linux/x86/bits/pthreadtypes.h index ccd896ca0..1852e0784 100644 --- a/libc/nptl/sysdeps/unix/sysv/linux/x86/bits/pthreadtypes.h +++ b/libc/nptl/sysdeps/unix/sysv/linux/x86/bits/pthreadtypes.h @@ -101,14 +101,23 @@ typedef union binary compatibility. */ int __kind; #ifdef __x86_64__ - int __spins; + short __spins; + short __elision; __pthread_list_t __list; # define __PTHREAD_MUTEX_HAVE_PREV 1 +# define __PTHREAD_MUTEX_HAVE_ELISION 1 #else unsigned int __nusers; __extension__ union { - int __spins; + struct + { + short __espins; + short __elision; +# define __spins d.__espins +# define __elision d.__elision +# define __PTHREAD_MUTEX_HAVE_ELISION 2 + } d; __pthread_slist_t __list; }; #endif diff --git a/libc/nptl/sysdeps/unix/sysv/linux/x86/elision-conf.c b/libc/nptl/sysdeps/unix/sysv/linux/x86/elision-conf.c new file mode 100644 index 000000000..118cfa72f --- /dev/null +++ b/libc/nptl/sysdeps/unix/sysv/linux/x86/elision-conf.c @@ -0,0 +1,90 @@ +/* elision-conf.c: Lock elision tunable parameters. + Copyright (C) 2013 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#include "config.h" +#include <pthreadP.h> +#include <init-arch.h> +#include <elision-conf.h> +#include <unistd.h> + +/* Reasonable initial tuning values, may be revised in the future. + This is a conservative initial value. */ + +struct elision_config __elision_aconf = + { + /* How often to not attempt to use elision if a transaction aborted + because the lock is already acquired. Expressed in number of lock + acquisition attempts. */ + .skip_lock_busy = 3, + /* How often to not attempt to use elision if a transaction aborted due + to reasons other than other threads' memory accesses. Expressed in + number of lock acquisition attempts. */ + .skip_lock_internal_abort = 3, + /* How often we retry using elision if there is chance for the transaction + to finish execution (e.g., it wasn't aborted due to the lock being + already acquired. */ + .retry_try_xbegin = 3, + /* Same as SKIP_LOCK_INTERNAL_ABORT but for trylock. */ + .skip_trylock_internal_abort = 3, + }; + +/* Elided rwlock toggle, set when elision is available and is + enabled for rwlocks. */ + +int __rwlock_rtm_enabled attribute_hidden; + +/* Retries for elided rwlocks on read. Conservative initial value. */ + +int __rwlock_rtm_read_retries attribute_hidden = 3; + +/* Set when the CPU supports elision. When false elision is never attempted. */ + +int __elision_available attribute_hidden; + +/* Force elision for all new locks. This is used to decide whether existing + DEFAULT locks should be automatically upgraded to elision in + pthread_mutex_lock(). Disabled for suid programs. Only used when elision + is available. */ + +int __pthread_force_elision attribute_hidden; + +/* Initialize elison. */ + +static void +elision_init (int argc __attribute__ ((unused)), + char **argv __attribute__ ((unused)), + char **environ) +{ + __elision_available = HAS_RTM; +#ifdef ENABLE_LOCK_ELISION + __pthread_force_elision = __libc_enable_secure ? 0 : __elision_available; + __rwlock_rtm_enabled = __libc_enable_secure ? 0 : __elision_available; +#endif +} + +#ifdef SHARED +# define INIT_SECTION ".init_array" +#else +# define INIT_SECTION ".preinit_array" +#endif + +void (*const __pthread_init_array []) (int, char **, char **) + __attribute__ ((section (INIT_SECTION), aligned (sizeof (void *)))) = +{ + &elision_init +}; diff --git a/libc/nptl/sysdeps/unix/sysv/linux/x86/elision-conf.h b/libc/nptl/sysdeps/unix/sysv/linux/x86/elision-conf.h new file mode 100644 index 000000000..55b81db93 --- /dev/null +++ b/libc/nptl/sysdeps/unix/sysv/linux/x86/elision-conf.h @@ -0,0 +1,44 @@ +/* elision-conf.h: Lock elision tunable parameters. + Copyright (C) 2013 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ +#ifndef _ELISION_CONF_H +#define _ELISION_CONF_H 1 + +#include <pthread.h> +#include <cpuid.h> +#include <time.h> + +/* Should make sure there is no false sharing on this. */ + +struct elision_config +{ + int skip_lock_busy; + int skip_lock_internal_abort; + int retry_try_xbegin; + int skip_trylock_internal_abort; +}; + +extern struct elision_config __elision_aconf attribute_hidden; + +extern int __rwlock_rtm_enabled attribute_hidden; +extern int __elision_available attribute_hidden; +extern int __pthread_force_elision attribute_hidden; + +/* Tell the test suite to test elision for this architecture. */ +#define HAVE_ELISION 1 + +#endif diff --git a/libc/nptl/sysdeps/unix/sysv/linux/x86/elision-lock.c b/libc/nptl/sysdeps/unix/sysv/linux/x86/elision-lock.c new file mode 100644 index 000000000..de16f1b02 --- /dev/null +++ b/libc/nptl/sysdeps/unix/sysv/linux/x86/elision-lock.c @@ -0,0 +1,95 @@ +/* elision-lock.c: Elided pthread mutex lock. + Copyright (C) 2011-2013 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#include <pthread.h> +#include "pthreadP.h" +#include "lowlevellock.h" +#include "hle.h" +#include <elision-conf.h> + +#if !defined(LLL_LOCK) && !defined(EXTRAARG) +/* Make sure the configuration code is always linked in for static + libraries. */ +#include "elision-conf.c" +#endif + +#ifndef EXTRAARG +#define EXTRAARG +#endif +#ifndef LLL_LOCK +#define LLL_LOCK(a,b) lll_lock(a,b), 0 +#endif + +#define aconf __elision_aconf + +/* Adaptive lock using transactions. + By default the lock region is run as a transaction, and when it + aborts or the lock is busy the lock adapts itself. */ + +int +__lll_lock_elision (int *futex, short *adapt_count, EXTRAARG int private) +{ + if (*adapt_count <= 0) + { + unsigned status; + int try_xbegin; + + for (try_xbegin = aconf.retry_try_xbegin; + try_xbegin > 0; + try_xbegin--) + { + if ((status = _xbegin()) == _XBEGIN_STARTED) + { + if (*futex == 0) + return 0; + + /* Lock was busy. Fall back to normal locking. + Could also _xend here but xabort with 0xff code + is more visible in the profiler. */ + _xabort (_ABORT_LOCK_BUSY); + } + + if (!(status & _XABORT_RETRY)) + { + if ((status & _XABORT_EXPLICIT) + && _XABORT_CODE (status) == _ABORT_LOCK_BUSY) + { + /* Right now we skip here. Better would be to wait a bit + and retry. This likely needs some spinning. */ + if (*adapt_count != aconf.skip_lock_busy) + *adapt_count = aconf.skip_lock_busy; + } + /* Internal abort. There is no chance for retry. + Use the normal locking and next time use lock. + Be careful to avoid writing to the lock. */ + else if (*adapt_count != aconf.skip_lock_internal_abort) + *adapt_count = aconf.skip_lock_internal_abort; + break; + } + } + } + else + { + /* Use a normal lock until the threshold counter runs out. + Lost updates possible. */ + (*adapt_count)--; + } + + /* Use a normal lock here. */ + return LLL_LOCK ((*futex), private); +} diff --git a/libc/nptl/sysdeps/unix/sysv/linux/x86/elision-timed.c b/libc/nptl/sysdeps/unix/sysv/linux/x86/elision-timed.c new file mode 100644 index 000000000..8825d1287 --- /dev/null +++ b/libc/nptl/sysdeps/unix/sysv/linux/x86/elision-timed.c @@ -0,0 +1,26 @@ +/* elision-timed.c: Lock elision timed lock. + Copyright (C) 2013 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#include <time.h> +#include <elision-conf.h> +#include "lowlevellock.h" +#define __lll_lock_elision __lll_timedlock_elision +#define EXTRAARG const struct timespec *t, +#undef LLL_LOCK +#define LLL_LOCK(a, b) lll_timedlock(a, t, b) +#include "elision-lock.c" diff --git a/libc/nptl/sysdeps/unix/sysv/linux/x86/elision-trylock.c b/libc/nptl/sysdeps/unix/sysv/linux/x86/elision-trylock.c new file mode 100644 index 000000000..689a6fbe2 --- /dev/null +++ b/libc/nptl/sysdeps/unix/sysv/linux/x86/elision-trylock.c @@ -0,0 +1,71 @@ +/* elision-trylock.c: Lock eliding trylock for pthreads. + Copyright (C) 2013 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#include <pthread.h> +#include <pthreadP.h> +#include <lowlevellock.h> +#include "hle.h" +#include <elision-conf.h> + +#define aconf __elision_aconf + +/* Try to elide a futex trylock. FUTEX is the futex variable. ADAPT_COUNT is the + adaptation counter in the mutex. */ + +int +__lll_trylock_elision (int *futex, short *adapt_count) +{ + /* Implement POSIX semantics by forbiding nesting + trylock. Sorry. After the abort the code is re-executed + non transactional and if the lock was already locked + return an error. */ + _xabort (_ABORT_NESTED_TRYLOCK); + + /* Only try a transaction if it's worth it. */ + if (*adapt_count <= 0) + { + unsigned status; + + if ((status = _xbegin()) == _XBEGIN_STARTED) + { + if (*futex == 0) + return 0; + + /* Lock was busy. Fall back to normal locking. + Could also _xend here but xabort with 0xff code + is more visible in the profiler. */ + _xabort (_ABORT_LOCK_BUSY); + } + + if (!(status & _XABORT_RETRY)) + { + /* Internal abort. No chance for retry. For future + locks don't try speculation for some time. */ + if (*adapt_count != aconf.skip_trylock_internal_abort) + *adapt_count = aconf.skip_trylock_internal_abort; + } + /* Could do some retries here. */ + } + else + { + /* Lost updates are possible, but harmless. */ + (*adapt_count)--; + } + + return lll_trylock (*futex); +} diff --git a/libc/nptl/sysdeps/unix/sysv/linux/x86/elision-unlock.c b/libc/nptl/sysdeps/unix/sysv/linux/x86/elision-unlock.c new file mode 100644 index 000000000..bb13c6b30 --- /dev/null +++ b/libc/nptl/sysdeps/unix/sysv/linux/x86/elision-unlock.c @@ -0,0 +1,33 @@ +/* elision-unlock.c: Commit an elided pthread lock. + Copyright (C) 2013 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#include "pthreadP.h" +#include "lowlevellock.h" +#include "hle.h" + +int +__lll_unlock_elision(int *lock, int private) +{ + /* When the lock was free we're in a transaction. + When you crash here you unlocked a free lock. */ + if (*lock == 0) + _xend(); + else + lll_unlock ((*lock), private); + return 0; +} diff --git a/libc/nptl/sysdeps/unix/sysv/linux/x86/force-elision.h b/libc/nptl/sysdeps/unix/sysv/linux/x86/force-elision.h new file mode 100644 index 000000000..703e930be --- /dev/null +++ b/libc/nptl/sysdeps/unix/sysv/linux/x86/force-elision.h @@ -0,0 +1,31 @@ +/* force-elision.h: Automatic enabling of elision for mutexes + Copyright (C) 2013 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +/* Check for elision on this lock without upgrading. */ +#define DO_ELISION(m) \ + (__pthread_force_elision \ + && (m->__data.__kind & PTHREAD_MUTEX_NO_ELISION_NP) == 0) \ + +/* Automatically enable elision for existing user lock kinds. */ +#define FORCE_ELISION(m, s) \ + if (__pthread_force_elision \ + && (m->__data.__kind & PTHREAD_MUTEX_ELISION_FLAGS_NP) == 0) \ + { \ + mutex->__data.__kind |= PTHREAD_MUTEX_ELISION_NP; \ + s; \ + } diff --git a/libc/nptl/sysdeps/unix/sysv/linux/x86/hle.h b/libc/nptl/sysdeps/unix/sysv/linux/x86/hle.h new file mode 100644 index 000000000..a08f0fa71 --- /dev/null +++ b/libc/nptl/sysdeps/unix/sysv/linux/x86/hle.h @@ -0,0 +1,75 @@ +/* Shared RTM header. Emulate TSX intrinsics for compilers and assemblers + that do not support the intrinsics and instructions yet. */ +#ifndef _HLE_H +#define _HLE_H 1 + +#ifdef __ASSEMBLER__ + +.macro XBEGIN target + .byte 0xc7,0xf8 + .long \target-1f +1: +.endm + +.macro XEND + .byte 0x0f,0x01,0xd5 +.endm + +.macro XABORT code + .byte 0xc6,0xf8,\code +.endm + +.macro XTEST + .byte 0x0f,0x01,0xd6 +.endm + +#endif + +/* Official RTM intrinsics interface matching gcc/icc, but works + on older gcc compatible compilers and binutils. + We should somehow detect if the compiler supports it, because + it may be able to generate slightly better code. */ + +#define _XBEGIN_STARTED (~0u) +#define _XABORT_EXPLICIT (1 << 0) +#define _XABORT_RETRY (1 << 1) +#define _XABORT_CONFLICT (1 << 2) +#define _XABORT_CAPACITY (1 << 3) +#define _XABORT_DEBUG (1 << 4) +#define _XABORT_NESTED (1 << 5) +#define _XABORT_CODE(x) (((x) >> 24) & 0xff) + +#define _ABORT_LOCK_BUSY 0xff +#define _ABORT_LOCK_IS_LOCKED 0xfe +#define _ABORT_NESTED_TRYLOCK 0xfd + +#ifndef __ASSEMBLER__ + +#define __force_inline __attribute__((__always_inline__)) inline + +static __force_inline int _xbegin(void) +{ + int ret = _XBEGIN_STARTED; + asm volatile (".byte 0xc7,0xf8 ; .long 0" : "+a" (ret) :: "memory"); + return ret; +} + +static __force_inline void _xend(void) +{ + asm volatile (".byte 0x0f,0x01,0xd5" ::: "memory"); +} + +static __force_inline void _xabort(const unsigned int status) +{ + asm volatile (".byte 0xc6,0xf8,%P0" :: "i" (status) : "memory"); +} + +static __force_inline int _xtest(void) +{ + unsigned char out; + asm volatile (".byte 0x0f,0x01,0xd6 ; setnz %0" : "=r" (out) :: "memory"); + return out; +} + +#endif +#endif diff --git a/libc/nptl/sysdeps/unix/sysv/linux/x86/init-arch.c b/libc/nptl/sysdeps/unix/sysv/linux/x86/init-arch.c new file mode 100644 index 000000000..00a94d83d --- /dev/null +++ b/libc/nptl/sysdeps/unix/sysv/linux/x86/init-arch.c @@ -0,0 +1 @@ +#include <sysdeps/x86_64/multiarch/init-arch.c> diff --git a/libc/nptl/sysdeps/unix/sysv/linux/x86/init-arch.h b/libc/nptl/sysdeps/unix/sysv/linux/x86/init-arch.h new file mode 100644 index 000000000..cd2d0befe --- /dev/null +++ b/libc/nptl/sysdeps/unix/sysv/linux/x86/init-arch.h @@ -0,0 +1 @@ +#include <sysdeps/x86_64/multiarch/init-arch.h> diff --git a/libc/nptl/sysdeps/unix/sysv/linux/x86/pthread_mutex_cond_lock.c b/libc/nptl/sysdeps/unix/sysv/linux/x86/pthread_mutex_cond_lock.c new file mode 100644 index 000000000..962f10107 --- /dev/null +++ b/libc/nptl/sysdeps/unix/sysv/linux/x86/pthread_mutex_cond_lock.c @@ -0,0 +1,21 @@ +/* Copyright (C) 2013 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +/* The cond lock is not actually elided yet, but we still need to handle + already elided locks. */ +#include <elision-conf.h> +#include "sysdeps/unix/sysv/linux/pthread_mutex_cond_lock.c" diff --git a/libc/nptl/sysdeps/unix/sysv/linux/x86/pthread_mutex_lock.c b/libc/nptl/sysdeps/unix/sysv/linux/x86/pthread_mutex_lock.c new file mode 100644 index 000000000..e63946053 --- /dev/null +++ b/libc/nptl/sysdeps/unix/sysv/linux/x86/pthread_mutex_lock.c @@ -0,0 +1,21 @@ +/* Elided version of pthread_mutex_lock. + Copyright (C) 2011, 2012, 2013 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ +#include <elision-conf.h> +#include "force-elision.h" + +#include "nptl/pthread_mutex_lock.c" diff --git a/libc/nptl/sysdeps/unix/sysv/linux/x86/pthread_mutex_timedlock.c b/libc/nptl/sysdeps/unix/sysv/linux/x86/pthread_mutex_timedlock.c new file mode 100644 index 000000000..5115d0304 --- /dev/null +++ b/libc/nptl/sysdeps/unix/sysv/linux/x86/pthread_mutex_timedlock.c @@ -0,0 +1,20 @@ +/* Elided version of pthread_mutex_timedlock. + Copyright (C) 2011, 2012, 2013 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ +#include <elision-conf.h> +#include "force-elision.h" +#include "nptl/pthread_mutex_timedlock.c" diff --git a/libc/nptl/sysdeps/unix/sysv/linux/x86/pthread_mutex_trylock.c b/libc/nptl/sysdeps/unix/sysv/linux/x86/pthread_mutex_trylock.c new file mode 100644 index 000000000..9bb4a659d --- /dev/null +++ b/libc/nptl/sysdeps/unix/sysv/linux/x86/pthread_mutex_trylock.c @@ -0,0 +1,21 @@ +/* Elided version of pthread_mutex_trylock. + Copyright (C) 2011, 2012, 2013 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ +#include <elision-conf.h> +#include "force-elision.h" + +#include "nptl/pthread_mutex_trylock.c" diff --git a/libc/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h b/libc/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h index 7a176aec3..35fb01a0d 100644 --- a/libc/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h +++ b/libc/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h @@ -427,6 +427,13 @@ LLL_STUB_UNWIND_INFO_END : "memory", "cx", "cc", "r10", "r11"); \ result; }) +extern int __lll_timedlock_elision (int *futex, short *adapt_count, + const struct timespec *timeout, + int private) attribute_hidden; + +#define lll_timedlock_elision(futex, adapt_count, timeout, private) \ + __lll_timedlock_elision(&(futex), &(adapt_count), timeout, private) + #define lll_robust_timedlock(futex, timeout, id, private) \ ({ int result, ignore1, ignore2, ignore3; \ __asm __volatile (LOCK_INSTR "cmpxchgl %1, %4\n\t" \ @@ -597,6 +604,22 @@ extern int __lll_timedwait_tid (int *tid, const struct timespec *abstime) } \ __result; }) +extern int __lll_lock_elision (int *futex, short *adapt_count, int private) + attribute_hidden; + +extern int __lll_unlock_elision (int *lock, int private) + attribute_hidden; + +extern int __lll_trylock_elision (int *lock, short *adapt_count) + attribute_hidden; + +#define lll_lock_elision(futex, adapt_count, private) \ + __lll_lock_elision (&(futex), &(adapt_count), private) +#define lll_unlock_elision(futex, private) \ + __lll_unlock_elision (&(futex), private) +#define lll_trylock_elision(futex, adapt_count) \ + __lll_trylock_elision (&(futex), &(adapt_count)) + #endif /* !__ASSEMBLER__ */ #endif /* lowlevellock.h */ diff --git a/libc/nptl/tst-mutex5.c b/libc/nptl/tst-mutex5.c index f19cd8c31..c223f6782 100644 --- a/libc/nptl/tst-mutex5.c +++ b/libc/nptl/tst-mutex5.c @@ -22,6 +22,7 @@ #include <time.h> #include <unistd.h> #include <sys/time.h> +#include <config.h> #ifndef TYPE @@ -85,6 +86,8 @@ do_test (void) return 1; } + /* Elided locks do not time out. */ +#ifdef ENABLE_LOCK_ELISION if (pthread_mutex_trylock (&m) == 0) { puts ("mutex_trylock succeeded"); @@ -180,6 +183,7 @@ do_test (void) puts ("3rd timedlock didn't return right away"); return 1; } +#endif if (pthread_mutex_unlock (&m) != 0) { diff --git a/libc/nptl/tst-mutex8.c b/libc/nptl/tst-mutex8.c index 2089c50d4..d2307e47e 100644 --- a/libc/nptl/tst-mutex8.c +++ b/libc/nptl/tst-mutex8.c @@ -93,6 +93,8 @@ tf (void *arg) static int check_type (const char *mas, pthread_mutexattr_t *ma) { + int e __attribute__((unused)); + if (pthread_mutex_init (m, ma) != 0) { printf ("1st mutex_init failed for %s\n", mas); @@ -117,7 +119,10 @@ check_type (const char *mas, pthread_mutexattr_t *ma) return 1; } - int e = pthread_mutex_destroy (m); + /* Elided mutexes don't fail destroy. If elision is not explicitly disabled + we don't know, so can also not check this. */ +#ifndef ENABLE_LOCK_ELISION + e = pthread_mutex_destroy (m); if (e == 0) { printf ("mutex_destroy of self-locked mutex succeeded for %s\n", mas); @@ -129,6 +134,7 @@ check_type (const char *mas, pthread_mutexattr_t *ma) mas); return 1; } +#endif if (pthread_mutex_unlock (m) != 0) { @@ -142,6 +148,8 @@ check_type (const char *mas, pthread_mutexattr_t *ma) return 1; } + /* Elided mutexes don't fail destroy. */ +#ifndef ENABLE_LOCK_ELISION e = pthread_mutex_destroy (m); if (e == 0) { @@ -155,6 +163,7 @@ mutex_destroy of self-trylocked mutex did not return EBUSY %s\n", mas); return 1; } +#endif if (pthread_mutex_unlock (m) != 0) { @@ -189,6 +198,8 @@ mutex_destroy of self-trylocked mutex did not return EBUSY %s\n", return 1; } + /* Elided mutexes don't fail destroy. */ +#ifndef ENABLE_LOCK_ELISION e = pthread_mutex_destroy (m); if (e == 0) { @@ -201,6 +212,7 @@ mutex_destroy of self-trylocked mutex did not return EBUSY %s\n", mutex_destroy of condvar-used mutex did not return EBUSY for %s\n", mas); return 1; } +#endif done = true; if (pthread_cond_signal (&c) != 0) @@ -259,6 +271,8 @@ mutex_destroy of condvar-used mutex did not return EBUSY for %s\n", mas); return 1; } + /* Elided mutexes don't fail destroy. */ +#ifndef ENABLE_LOCK_ELISION e = pthread_mutex_destroy (m); if (e == 0) { @@ -273,6 +287,7 @@ mutex_destroy of condvar-used mutex did not return EBUSY for %s\n", mas); mas); return 1; } +#endif if (pthread_cancel (th) != 0) { |