diff options
Diffstat (limited to 'libgomp/config/linux')
| -rw-r--r-- | libgomp/config/linux/affinity.c | 14 | ||||
| -rw-r--r-- | libgomp/config/linux/alpha/futex.h | 46 | ||||
| -rw-r--r-- | libgomp/config/linux/bar.c | 91 | ||||
| -rw-r--r-- | libgomp/config/linux/bar.h | 76 | ||||
| -rw-r--r-- | libgomp/config/linux/ia64/futex.h | 44 | ||||
| -rw-r--r-- | libgomp/config/linux/lock.c | 115 | ||||
| -rw-r--r-- | libgomp/config/linux/mutex.c | 9 | ||||
| -rw-r--r-- | libgomp/config/linux/omp-lock.h | 8 | ||||
| -rw-r--r-- | libgomp/config/linux/powerpc/futex.h | 39 | ||||
| -rw-r--r-- | libgomp/config/linux/proc.c | 18 | ||||
| -rw-r--r-- | libgomp/config/linux/ptrlock.c | 70 | ||||
| -rw-r--r-- | libgomp/config/linux/ptrlock.h | 65 | ||||
| -rw-r--r-- | libgomp/config/linux/s390/futex.h | 35 | ||||
| -rw-r--r-- | libgomp/config/linux/sem.c | 7 | ||||
| -rw-r--r-- | libgomp/config/linux/sparc/futex.h | 47 | ||||
| -rw-r--r-- | libgomp/config/linux/wait.h | 68 | ||||
| -rw-r--r-- | libgomp/config/linux/x86/futex.h | 70 |
17 files changed, 693 insertions, 129 deletions
diff --git a/libgomp/config/linux/affinity.c b/libgomp/config/linux/affinity.c index 8fcce5f3a5b..7b6d6c008d7 100644 --- a/libgomp/config/linux/affinity.c +++ b/libgomp/config/linux/affinity.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2006, 2007 Free Software Foundation, Inc. +/* Copyright (C) 2006, 2007, 2008 Free Software Foundation, Inc. Contributed by Jakub Jelinek <jakub@redhat.com>. This file is part of the GNU OpenMP Library (libgomp). @@ -38,9 +38,6 @@ #ifdef HAVE_PTHREAD_AFFINITY_NP static unsigned int affinity_counter; -#ifndef HAVE_SYNC_BUILTINS -static gomp_mutex_t affinity_lock; -#endif void gomp_init_affinity (void) @@ -76,9 +73,6 @@ gomp_init_affinity (void) CPU_SET (gomp_cpu_affinity[0], &cpuset); pthread_setaffinity_np (pthread_self (), sizeof (cpuset), &cpuset); affinity_counter = 1; -#ifndef HAVE_SYNC_BUILTINS - gomp_mutex_init (&affinity_lock); -#endif } void @@ -87,13 +81,7 @@ gomp_init_thread_affinity (pthread_attr_t *attr) unsigned int cpu; cpu_set_t cpuset; -#ifdef HAVE_SYNC_BUILTINS cpu = __sync_fetch_and_add (&affinity_counter, 1); -#else - gomp_mutex_lock (&affinity_lock); - cpu = affinity_counter++; - gomp_mutex_unlock (&affinity_lock); -#endif cpu %= gomp_cpu_affinity_len; CPU_ZERO (&cpuset); CPU_SET (gomp_cpu_affinity[cpu], &cpuset); diff --git a/libgomp/config/linux/alpha/futex.h b/libgomp/config/linux/alpha/futex.h index 98681a8526e..4f0bda2424c 100644 --- a/libgomp/config/linux/alpha/futex.h +++ b/libgomp/config/linux/alpha/futex.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2005 Free Software Foundation, Inc. +/* Copyright (C) 2005, 2008 Free Software Foundation, Inc. Contributed by Richard Henderson <rth@redhat.com>. This file is part of the GNU OpenMP Library (libgomp). @@ -30,8 +30,6 @@ #ifndef SYS_futex #define SYS_futex 394 #endif -#define FUTEX_WAIT 0 -#define FUTEX_WAKE 1 static inline void @@ -45,7 +43,7 @@ futex_wait (int *addr, int val) sc_0 = SYS_futex; sc_16 = (long) addr; - sc_17 = FUTEX_WAIT; + sc_17 = gomp_futex_wait; sc_18 = val; sc_19 = 0; __asm volatile ("callsys" @@ -53,6 +51,20 @@ futex_wait (int *addr, int val) : "0"(sc_0), "r" (sc_16), "r"(sc_17), "r"(sc_18), "1"(sc_19) : "$1", "$2", "$3", "$4", "$5", "$6", "$7", "$8", "$22", "$23", "$24", "$25", "$27", "$28", "memory"); + if (__builtin_expect (sc_19, 0) && sc_0 == ENOSYS) + { + gomp_futex_wait &= ~FUTEX_PRIVATE_FLAG; + gomp_futex_wake &= ~FUTEX_PRIVATE_FLAG; + sc_0 = SYS_futex; + sc_17 &= ~FUTEX_PRIVATE_FLAG; + sc_19 = 0; + __asm volatile ("callsys" + : "=r" (sc_0), "=r"(sc_19) + : "0"(sc_0), "r" (sc_16), "r"(sc_17), "r"(sc_18), + "1"(sc_19) + : "$1", "$2", "$3", "$4", "$5", "$6", "$7", "$8", + "$22", "$23", "$24", "$25", "$27", "$28", "memory"); + } } static inline void @@ -66,11 +78,35 @@ futex_wake (int *addr, int count) sc_0 = SYS_futex; sc_16 = (long) addr; - sc_17 = FUTEX_WAKE; + sc_17 = gomp_futex_wake; sc_18 = count; __asm volatile ("callsys" : "=r" (sc_0), "=r"(sc_19) : "0"(sc_0), "r" (sc_16), "r"(sc_17), "r"(sc_18) : "$1", "$2", "$3", "$4", "$5", "$6", "$7", "$8", "$22", "$23", "$24", "$25", "$27", "$28", "memory"); + if (__builtin_expect (sc_19, 0) && sc_0 == ENOSYS) + { + gomp_futex_wait &= ~FUTEX_PRIVATE_FLAG; + gomp_futex_wake &= ~FUTEX_PRIVATE_FLAG; + sc_0 = SYS_futex; + sc_17 &= ~FUTEX_PRIVATE_FLAG; + __asm volatile ("callsys" + : "=r" (sc_0), "=r"(sc_19) + : "0"(sc_0), "r" (sc_16), "r"(sc_17), "r"(sc_18) + : "$1", "$2", "$3", "$4", "$5", "$6", "$7", "$8", + "$22", "$23", "$24", "$25", "$27", "$28", "memory"); + } +} + +static inline void +cpu_relax (void) +{ + __asm volatile ("" : : : "memory"); +} + +static inline void +atomic_write_barrier (void) +{ + __asm volatile ("wmb" : : : "memory"); } diff --git a/libgomp/config/linux/bar.c b/libgomp/config/linux/bar.c index 5c4f32e6f8b..7af36d2c421 100644 --- a/libgomp/config/linux/bar.c +++ b/libgomp/config/linux/bar.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2005 Free Software Foundation, Inc. +/* Copyright (C) 2005, 2008 Free Software Foundation, Inc. Contributed by Richard Henderson <rth@redhat.com>. This file is part of the GNU OpenMP Library (libgomp). @@ -29,36 +29,97 @@ mechanism for libgomp. This type is private to the library. This implementation uses atomic instructions and the futex syscall. */ -#include "libgomp.h" -#include "futex.h" #include <limits.h> +#include "wait.h" void -gomp_barrier_wait_end (gomp_barrier_t *bar, bool last) +gomp_barrier_wait_end (gomp_barrier_t *bar, gomp_barrier_state_t state) { - if (last) + if (__builtin_expect ((state & 1) != 0, 0)) { - bar->generation++; - futex_wake (&bar->generation, INT_MAX); + /* Next time we'll be awaiting TOTAL threads again. */ + bar->awaited = bar->total; + atomic_write_barrier (); + bar->generation += 4; + futex_wake ((int *) &bar->generation, INT_MAX); } else { - unsigned int generation = bar->generation; - - gomp_mutex_unlock (&bar->mutex); + unsigned int generation = state; do - futex_wait (&bar->generation, generation); + do_wait ((int *) &bar->generation, generation); while (bar->generation == generation); } +} + +void +gomp_barrier_wait (gomp_barrier_t *bar) +{ + gomp_barrier_wait_end (bar, gomp_barrier_wait_start (bar)); +} - if (__sync_add_and_fetch (&bar->arrived, -1) == 0) - gomp_mutex_unlock (&bar->mutex); +/* Like gomp_barrier_wait, except that if the encountering thread + is not the last one to hit the barrier, it returns immediately. + The intended usage is that a thread which intends to gomp_barrier_destroy + this barrier calls gomp_barrier_wait, while all other threads + call gomp_barrier_wait_last. When gomp_barrier_wait returns, + the barrier can be safely destroyed. */ + +void +gomp_barrier_wait_last (gomp_barrier_t *bar) +{ + gomp_barrier_state_t state = gomp_barrier_wait_start (bar); + if (state & 1) + gomp_barrier_wait_end (bar, state); +} + +void +gomp_team_barrier_wake (gomp_barrier_t *bar, int count) +{ + futex_wake ((int *) &bar->generation, count == 0 ? INT_MAX : count); +} + +void +gomp_team_barrier_wait_end (gomp_barrier_t *bar, gomp_barrier_state_t state) +{ + unsigned int generation; + + if (__builtin_expect ((state & 1) != 0, 0)) + { + /* Next time we'll be awaiting TOTAL threads again. */ + struct gomp_thread *thr = gomp_thread (); + struct gomp_team *team = thr->ts.team; + bar->awaited = bar->total; + atomic_write_barrier (); + if (__builtin_expect (team->task_count, 0)) + { + gomp_barrier_handle_tasks (state); + state &= ~1; + } + else + { + bar->generation = state + 3; + futex_wake ((int *) &bar->generation, INT_MAX); + return; + } + } + + generation = state; + do + { + do_wait ((int *) &bar->generation, generation); + if (__builtin_expect (bar->generation & 1, 0)) + gomp_barrier_handle_tasks (state); + if ((bar->generation & 2)) + generation |= 2; + } + while (bar->generation != state + 4); } void -gomp_barrier_wait (gomp_barrier_t *barrier) +gomp_team_barrier_wait (gomp_barrier_t *bar) { - gomp_barrier_wait_end (barrier, gomp_barrier_wait_start (barrier)); + gomp_team_barrier_wait_end (bar, gomp_barrier_wait_start (bar)); } diff --git a/libgomp/config/linux/bar.h b/libgomp/config/linux/bar.h index 57268585d8b..85150caf189 100644 --- a/libgomp/config/linux/bar.h +++ b/libgomp/config/linux/bar.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2005 Free Software Foundation, Inc. +/* Copyright (C) 2005, 2008 Free Software Foundation, Inc. Contributed by Richard Henderson <rth@redhat.com>. This file is part of the GNU OpenMP Library (libgomp). @@ -36,40 +36,86 @@ typedef struct { - gomp_mutex_t mutex; - unsigned total; - unsigned arrived; - int generation; + /* Make sure total/generation is in a mostly read cacheline, while + awaited in a separate cacheline. */ + unsigned total __attribute__((aligned (64))); + unsigned generation; + unsigned awaited __attribute__((aligned (64))); } gomp_barrier_t; +typedef unsigned int gomp_barrier_state_t; static inline void gomp_barrier_init (gomp_barrier_t *bar, unsigned count) { - gomp_mutex_init (&bar->mutex); bar->total = count; - bar->arrived = 0; + bar->awaited = count; bar->generation = 0; } static inline void gomp_barrier_reinit (gomp_barrier_t *bar, unsigned count) { - gomp_mutex_lock (&bar->mutex); + __sync_fetch_and_add (&bar->awaited, count - bar->total); bar->total = count; - gomp_mutex_unlock (&bar->mutex); } static inline void gomp_barrier_destroy (gomp_barrier_t *bar) { - /* Before destroying, make sure all threads have left the barrier. */ - gomp_mutex_lock (&bar->mutex); } extern void gomp_barrier_wait (gomp_barrier_t *); -extern void gomp_barrier_wait_end (gomp_barrier_t *, bool); +extern void gomp_barrier_wait_last (gomp_barrier_t *); +extern void gomp_barrier_wait_end (gomp_barrier_t *, gomp_barrier_state_t); +extern void gomp_team_barrier_wait (gomp_barrier_t *); +extern void gomp_team_barrier_wait_end (gomp_barrier_t *, + gomp_barrier_state_t); +extern void gomp_team_barrier_wake (gomp_barrier_t *, int); -static inline bool gomp_barrier_wait_start (gomp_barrier_t *bar) +static inline gomp_barrier_state_t +gomp_barrier_wait_start (gomp_barrier_t *bar) { - gomp_mutex_lock (&bar->mutex); - return ++bar->arrived == bar->total; + unsigned int ret = bar->generation & ~3; + /* Do we need any barrier here or is __sync_add_and_fetch acting + as the needed LoadLoad barrier already? */ + ret += __sync_add_and_fetch (&bar->awaited, -1) == 0; + return ret; +} + +static inline bool +gomp_barrier_last_thread (gomp_barrier_state_t state) +{ + return state & 1; +} + +/* All the inlines below must be called with team->task_lock + held. */ + +static inline void +gomp_team_barrier_set_task_pending (gomp_barrier_t *bar) +{ + bar->generation |= 1; +} + +static inline void +gomp_team_barrier_clear_task_pending (gomp_barrier_t *bar) +{ + bar->generation &= ~1; +} + +static inline void +gomp_team_barrier_set_waiting_for_tasks (gomp_barrier_t *bar) +{ + bar->generation |= 2; +} + +static inline bool +gomp_team_barrier_waiting_for_tasks (gomp_barrier_t *bar) +{ + return (bar->generation & 2) != 0; +} + +static inline void +gomp_team_barrier_done (gomp_barrier_t *bar, gomp_barrier_state_t state) +{ + bar->generation = (state & ~3) + 4; } #endif /* GOMP_BARRIER_H */ diff --git a/libgomp/config/linux/ia64/futex.h b/libgomp/config/linux/ia64/futex.h index 5e54982d6f7..35aa4f1fea0 100644 --- a/libgomp/config/linux/ia64/futex.h +++ b/libgomp/config/linux/ia64/futex.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2005 Free Software Foundation, Inc. +/* Copyright (C) 2005, 2008 Free Software Foundation, Inc. Contributed by Richard Henderson <rth@redhat.com>. This file is part of the GNU OpenMP Library (libgomp). @@ -29,23 +29,24 @@ #include <sys/syscall.h> -#define FUTEX_WAIT 0 -#define FUTEX_WAKE 1 -static inline void -sys_futex0(int *addr, int op, int val) +static inline long +sys_futex0(int *addr, long op, int val) { register long out0 asm ("out0") = (long) addr; register long out1 asm ("out1") = op; register long out2 asm ("out2") = val; register long out3 asm ("out3") = 0; + register long r8 asm ("r8"); + register long r10 asm ("r10"); register long r15 asm ("r15") = SYS_futex; __asm __volatile ("break 0x100000" - : "=r"(r15), "=r"(out0), "=r"(out1), "=r"(out2), "=r"(out3) + : "=r"(r15), "=r"(out0), "=r"(out1), "=r"(out2), "=r"(out3), + "=r"(r8), "=r"(r10) : "r"(r15), "r"(out0), "r"(out1), "r"(out2), "r"(out3) - : "memory", "r8", "r10", "out4", "out5", "out6", "out7", + : "memory", "out4", "out5", "out6", "out7", /* Non-stacked integer registers, minus r8, r10, r15. */ "r2", "r3", "r9", "r11", "r12", "r13", "r14", "r16", "r17", "r18", "r19", "r20", "r21", "r22", "r23", "r24", "r25", "r26", "r27", @@ -56,16 +57,41 @@ sys_futex0(int *addr, int op, int val) "f6", "f7", "f8", "f9", "f10", "f11", "f12", "f13", "f14", "f15", /* Branch registers. */ "b6"); + return r8 & r10; } static inline void futex_wait (int *addr, int val) { - sys_futex0 (addr, FUTEX_WAIT, val); + long err = sys_futex0 (addr, gomp_futex_wait, val); + if (__builtin_expect (err == ENOSYS, 0)) + { + gomp_futex_wait &= ~FUTEX_PRIVATE_FLAG; + gomp_futex_wake &= ~FUTEX_PRIVATE_FLAG; + sys_futex0 (addr, gomp_futex_wait, val); + } } static inline void futex_wake (int *addr, int count) { - sys_futex0 (addr, FUTEX_WAKE, count); + long err = sys_futex0 (addr, gomp_futex_wake, count); + if (__builtin_expect (err == ENOSYS, 0)) + { + gomp_futex_wait &= ~FUTEX_PRIVATE_FLAG; + gomp_futex_wake &= ~FUTEX_PRIVATE_FLAG; + sys_futex0 (addr, gomp_futex_wake, count); + } +} + +static inline void +cpu_relax (void) +{ + __asm volatile ("hint @pause" : : : "memory"); +} + +static inline void +atomic_write_barrier (void) +{ + __sync_synchronize (); } diff --git a/libgomp/config/linux/lock.c b/libgomp/config/linux/lock.c index 211f6007b43..a2e07d320fd 100644 --- a/libgomp/config/linux/lock.c +++ b/libgomp/config/linux/lock.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2005 Free Software Foundation, Inc. +/* Copyright (C) 2005, 2008 Free Software Foundation, Inc. Contributed by Richard Henderson <rth@redhat.com>. This file is part of the GNU OpenMP Library (libgomp). @@ -29,47 +29,109 @@ primitives. This implementation uses atomic instructions and the futex syscall. */ -#include "libgomp.h" #include <string.h> #include <unistd.h> #include <sys/syscall.h> -#include "futex.h" +#include "wait.h" /* The internal gomp_mutex_t and the external non-recursive omp_lock_t have the same form. Re-use it. */ void -omp_init_lock (omp_lock_t *lock) +gomp_init_lock_30 (omp_lock_t *lock) { gomp_mutex_init (lock); } void -omp_destroy_lock (omp_lock_t *lock) +gomp_destroy_lock_30 (omp_lock_t *lock) { gomp_mutex_destroy (lock); } void -omp_set_lock (omp_lock_t *lock) +gomp_set_lock_30 (omp_lock_t *lock) { gomp_mutex_lock (lock); } void -omp_unset_lock (omp_lock_t *lock) +gomp_unset_lock_30 (omp_lock_t *lock) { gomp_mutex_unlock (lock); } int -omp_test_lock (omp_lock_t *lock) +gomp_test_lock_30 (omp_lock_t *lock) { return __sync_bool_compare_and_swap (lock, 0, 1); } -/* The external recursive omp_nest_lock_t form requires additional work. */ +void +gomp_init_nest_lock_30 (omp_nest_lock_t *lock) +{ + memset (lock, '\0', sizeof (*lock)); +} + +void +gomp_destroy_nest_lock_30 (omp_nest_lock_t *lock) +{ +} + +void +gomp_set_nest_lock_30 (omp_nest_lock_t *lock) +{ + void *me = gomp_icv (true); + + if (lock->owner != me) + { + gomp_mutex_lock (&lock->lock); + lock->owner = me; + } + + lock->count++; +} + +void +gomp_unset_nest_lock_30 (omp_nest_lock_t *lock) +{ + if (--lock->count == 0) + { + lock->owner = NULL; + gomp_mutex_unlock (&lock->lock); + } +} + +int +gomp_test_nest_lock_30 (omp_nest_lock_t *lock) +{ + void *me = gomp_icv (true); + + if (lock->owner == me) + return ++lock->count; + + if (__sync_bool_compare_and_swap (&lock->lock, 0, 1)) + { + lock->owner = me; + lock->count = 1; + return 1; + } + + return 0; +} + +#ifdef LIBGOMP_GNU_SYMBOL_VERSIONING +/* gomp_mutex_* can be safely locked in one thread and + unlocked in another thread, so the OpenMP 2.5 and OpenMP 3.0 + non-nested locks can be the same. */ +strong_alias (gomp_init_lock_30, gomp_init_lock_25) +strong_alias (gomp_destroy_lock_30, gomp_destroy_lock_25) +strong_alias (gomp_set_lock_30, gomp_set_lock_25) +strong_alias (gomp_unset_lock_30, gomp_unset_lock_25) +strong_alias (gomp_test_lock_30, gomp_test_lock_25) + +/* The external recursive omp_nest_lock_25_t form requires additional work. */ /* We need an integer to uniquely identify this thread. Most generally this is the thread's TID, which ideally we'd get this straight from @@ -85,17 +147,17 @@ omp_test_lock (omp_lock_t *lock) always available directly. Make do with the gomp_thread pointer since it's handy. */ -#if !defined (HAVE_TLS) +# if !defined (HAVE_TLS) static inline int gomp_tid (void) { return syscall (SYS_gettid); } -#elif !defined(__LP64__) +# elif !defined(__LP64__) static inline int gomp_tid (void) { return (int) gomp_thread (); } -#else +# else static __thread int tid_cache; static inline int gomp_tid (void) { @@ -104,22 +166,22 @@ static inline int gomp_tid (void) tid_cache = tid = syscall (SYS_gettid); return tid; } -#endif +# endif void -omp_init_nest_lock (omp_nest_lock_t *lock) +gomp_init_nest_lock_25 (omp_nest_lock_25_t *lock) { memset (lock, 0, sizeof (lock)); } void -omp_destroy_nest_lock (omp_nest_lock_t *lock) +gomp_destroy_nest_lock_25 (omp_nest_lock_25_t *lock) { } void -omp_set_nest_lock (omp_nest_lock_t *lock) +gomp_set_nest_lock_25 (omp_nest_lock_25_t *lock) { int otid, tid = gomp_tid (); @@ -137,12 +199,12 @@ omp_set_nest_lock (omp_nest_lock_t *lock) return; } - futex_wait (&lock->owner, otid); + do_wait (&lock->owner, otid); } } void -omp_unset_nest_lock (omp_nest_lock_t *lock) +gomp_unset_nest_lock_25 (omp_nest_lock_25_t *lock) { /* ??? Validate that we own the lock here. */ @@ -154,7 +216,7 @@ omp_unset_nest_lock (omp_nest_lock_t *lock) } int -omp_test_nest_lock (omp_nest_lock_t *lock) +gomp_test_nest_lock_25 (omp_nest_lock_25_t *lock) { int otid, tid = gomp_tid (); @@ -170,6 +232,19 @@ omp_test_nest_lock (omp_nest_lock_t *lock) return 0; } +omp_lock_symver (omp_init_lock) +omp_lock_symver (omp_destroy_lock) +omp_lock_symver (omp_set_lock) +omp_lock_symver (omp_unset_lock) +omp_lock_symver (omp_test_lock) +omp_lock_symver (omp_init_nest_lock) +omp_lock_symver (omp_destroy_nest_lock) +omp_lock_symver (omp_set_nest_lock) +omp_lock_symver (omp_unset_nest_lock) +omp_lock_symver (omp_test_nest_lock) + +#else + ialias (omp_init_lock) ialias (omp_init_nest_lock) ialias (omp_destroy_lock) @@ -180,3 +255,5 @@ ialias (omp_unset_lock) ialias (omp_unset_nest_lock) ialias (omp_test_lock) ialias (omp_test_nest_lock) + +#endif diff --git a/libgomp/config/linux/mutex.c b/libgomp/config/linux/mutex.c index fa3dfd1cb03..36c362eb274 100644 --- a/libgomp/config/linux/mutex.c +++ b/libgomp/config/linux/mutex.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2005 Free Software Foundation, Inc. +/* Copyright (C) 2005, 2008 Free Software Foundation, Inc. Contributed by Richard Henderson <rth@redhat.com>. This file is part of the GNU OpenMP Library (libgomp). @@ -29,9 +29,10 @@ mechanism for libgomp. This type is private to the library. This implementation uses atomic instructions and the futex syscall. */ -#include "libgomp.h" -#include "futex.h" +#include "wait.h" +long int gomp_futex_wake = FUTEX_WAKE | FUTEX_PRIVATE_FLAG; +long int gomp_futex_wait = FUTEX_WAIT | FUTEX_PRIVATE_FLAG; void gomp_mutex_lock_slow (gomp_mutex_t *mutex) @@ -40,7 +41,7 @@ gomp_mutex_lock_slow (gomp_mutex_t *mutex) { int oldval = __sync_val_compare_and_swap (mutex, 1, 2); if (oldval != 0) - futex_wait (mutex, 2); + do_wait (mutex, 2); } while (!__sync_bool_compare_and_swap (mutex, 0, 2)); } diff --git a/libgomp/config/linux/omp-lock.h b/libgomp/config/linux/omp-lock.h index 350cba16056..e65aff7fce7 100644 --- a/libgomp/config/linux/omp-lock.h +++ b/libgomp/config/linux/omp-lock.h @@ -3,8 +3,10 @@ structures without polluting the namespace. When using the Linux futex primitive, non-recursive locks require - only one int. Recursive locks require we identify the owning thread - and so require two ints. */ + only one int. Recursive locks require we identify the owning task + and so require one int and a pointer. */ typedef int omp_lock_t; -typedef struct { int owner, count; } omp_nest_lock_t; +typedef struct { int lock, count; void *owner; } omp_nest_lock_t; +typedef int omp_lock_25_t; +typedef struct { int owner, count; } omp_nest_lock_25_t; diff --git a/libgomp/config/linux/powerpc/futex.h b/libgomp/config/linux/powerpc/futex.h index 20e03573783..c1e0d0f5651 100644 --- a/libgomp/config/linux/powerpc/futex.h +++ b/libgomp/config/linux/powerpc/futex.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2005 Free Software Foundation, Inc. +/* Copyright (C) 2005, 2008 Free Software Foundation, Inc. Contributed by Richard Henderson <rth@redhat.com>. This file is part of the GNU OpenMP Library (libgomp). @@ -28,10 +28,8 @@ /* Provide target-specific access to the futex system call. */ #include <sys/syscall.h> -#define FUTEX_WAIT 0 -#define FUTEX_WAKE 1 -static inline void +static inline long sys_futex0 (int *addr, int op, int val) { register long int r0 __asm__ ("r0"); @@ -50,21 +48,48 @@ sys_futex0 (int *addr, int op, int val) doesn't. It doesn't much matter for us. In the interest of unity, go ahead and clobber it always. */ - __asm volatile ("sc" + __asm volatile ("sc; mfcr %0" : "=r"(r0), "=r"(r3), "=r"(r4), "=r"(r5), "=r"(r6) : "r"(r0), "r"(r3), "r"(r4), "r"(r5), "r"(r6) : "r7", "r8", "r9", "r10", "r11", "r12", "cr0", "ctr", "memory"); + if (__builtin_expect (r0 & (1 << 28), 0)) + return r3; + return 0; } static inline void futex_wait (int *addr, int val) { - sys_futex0 (addr, FUTEX_WAIT, val); + long err = sys_futex0 (addr, gomp_futex_wait, val); + if (__builtin_expect (err == ENOSYS, 0)) + { + gomp_futex_wait &= ~FUTEX_PRIVATE_FLAG; + gomp_futex_wake &= ~FUTEX_PRIVATE_FLAG; + sys_futex0 (addr, gomp_futex_wait, val); + } } static inline void futex_wake (int *addr, int count) { - sys_futex0 (addr, FUTEX_WAKE, count); + long err = sys_futex0 (addr, gomp_futex_wake, count); + if (__builtin_expect (err == ENOSYS, 0)) + { + gomp_futex_wait &= ~FUTEX_PRIVATE_FLAG; + gomp_futex_wake &= ~FUTEX_PRIVATE_FLAG; + sys_futex0 (addr, gomp_futex_wake, count); + } +} + +static inline void +cpu_relax (void) +{ + __asm volatile ("" : : : "memory"); +} + +static inline void +atomic_write_barrier (void) +{ + __asm volatile ("eieio" : : : "memory"); } diff --git a/libgomp/config/linux/proc.c b/libgomp/config/linux/proc.c index 2267cfbd2d1..6a006f24aa3 100644 --- a/libgomp/config/linux/proc.c +++ b/libgomp/config/linux/proc.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2005, 2006, 2007 Free Software Foundation, Inc. +/* Copyright (C) 2005, 2006, 2007, 2008 Free Software Foundation, Inc. Contributed by Jakub Jelinek <jakub@redhat.com>. This file is part of the GNU OpenMP Library (libgomp). @@ -78,14 +78,14 @@ gomp_init_num_threads (void) if (pthread_getaffinity_np (pthread_self (), sizeof (cpuset), &cpuset) == 0) { /* Count only the CPUs this process can use. */ - gomp_nthreads_var = cpuset_popcount (&cpuset); - if (gomp_nthreads_var == 0) - gomp_nthreads_var = 1; + gomp_global_icv.nthreads_var = cpuset_popcount (&cpuset); + if (gomp_global_icv.nthreads_var == 0) + gomp_global_icv.nthreads_var = 1; return; } #endif #ifdef _SC_NPROCESSORS_ONLN - gomp_nthreads_var = sysconf (_SC_NPROCESSORS_ONLN); + gomp_global_icv.nthreads_var = sysconf (_SC_NPROCESSORS_ONLN); #endif } @@ -132,7 +132,7 @@ get_num_procs (void) #ifdef _SC_NPROCESSORS_ONLN return sysconf (_SC_NPROCESSORS_ONLN); #else - return gomp_nthreads_var; + return gomp_icv (false)->nthreads_var; #endif } @@ -146,11 +146,11 @@ get_num_procs (void) unsigned gomp_dynamic_max_threads (void) { - unsigned n_onln, loadavg; + unsigned n_onln, loadavg, nthreads_var = gomp_icv (false)->nthreads_var; n_onln = get_num_procs (); - if (n_onln > gomp_nthreads_var) - n_onln = gomp_nthreads_var; + if (n_onln > nthreads_var) + n_onln = nthreads_var; loadavg = 0; #ifdef HAVE_GETLOADAVG diff --git a/libgomp/config/linux/ptrlock.c b/libgomp/config/linux/ptrlock.c new file mode 100644 index 00000000000..8faa1b2287d --- /dev/null +++ b/libgomp/config/linux/ptrlock.c @@ -0,0 +1,70 @@ +/* Copyright (C) 2008 Free Software Foundation, Inc. + Contributed by Jakub Jelinek <jakub@redhat.com>. + + This file is part of the GNU OpenMP Library (libgomp). + + Libgomp is free software; you can redistribute it and/or modify it + under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation; either version 2.1 of the License, or + (at your option) any later version. + + Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for + more details. + + You should have received a copy of the GNU Lesser General Public License + along with libgomp; see the file COPYING.LIB. If not, write to the + Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +/* As a special exception, if you link this library with other files, some + of which are compiled with GCC, to produce an executable, this library + does not by itself cause the resulting executable to be covered by the + GNU General Public License. This exception does not however invalidate + any other reasons why the executable file might be covered by the GNU + General Public License. */ + +/* This is a Linux specific implementation of a mutex synchronization + mechanism for libgomp. This type is private to the library. This + implementation uses atomic instructions and the futex syscall. */ + +#include <endian.h> +#include <limits.h> +#include "wait.h" + +void * +gomp_ptrlock_get_slow (gomp_ptrlock_t *ptrlock) +{ + int *intptr; + __sync_bool_compare_and_swap (ptrlock, 1, 2); + + /* futex works on ints, not pointers. + But a valid work share pointer will be at least + 8 byte aligned, so it is safe to assume the low + 32-bits of the pointer won't contain values 1 or 2. */ + __asm volatile ("" : "=r" (intptr) : "0" (ptrlock)); +#if __BYTE_ORDER == __BIG_ENDIAN + if (sizeof (*ptrlock) > sizeof (int)) + intptr += (sizeof (*ptrlock) / sizeof (int)) - 1; +#endif + do + do_wait (intptr, 2); + while (*intptr == 2); + __asm volatile ("" : : : "memory"); + return *ptrlock; +} + +void +gomp_ptrlock_set_slow (gomp_ptrlock_t *ptrlock, void *ptr) +{ + int *intptr; + + *ptrlock = ptr; + __asm volatile ("" : "=r" (intptr) : "0" (ptrlock)); +#if __BYTE_ORDER == __BIG_ENDIAN + if (sizeof (*ptrlock) > sizeof (int)) + intptr += (sizeof (*ptrlock) / sizeof (int)) - 1; +#endif + futex_wake (intptr, INT_MAX); +} diff --git a/libgomp/config/linux/ptrlock.h b/libgomp/config/linux/ptrlock.h new file mode 100644 index 00000000000..bb5441676a4 --- /dev/null +++ b/libgomp/config/linux/ptrlock.h @@ -0,0 +1,65 @@ +/* Copyright (C) 2008 Free Software Foundation, Inc. + Contributed by Jakub Jelinek <jakub@redhat.com>. + + This file is part of the GNU OpenMP Library (libgomp). + + Libgomp is free software; you can redistribute it and/or modify it + under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation; either version 2.1 of the License, or + (at your option) any later version. + + Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for + more details. + + You should have received a copy of the GNU Lesser General Public License + along with libgomp; see the file COPYING.LIB. If not, write to the + Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +/* As a special exception, if you link this library with other files, some + of which are compiled with GCC, to produce an executable, this library + does not by itself cause the resulting executable to be covered by the + GNU General Public License. This exception does not however invalidate + any other reasons why the executable file might be covered by the GNU + General Public License. */ + +/* This is a Linux specific implementation of a mutex synchronization + mechanism for libgomp. This type is private to the library. This + implementation uses atomic instructions and the futex syscall. */ + +#ifndef GOMP_PTRLOCK_H +#define GOMP_PTRLOCK_H 1 + +typedef void *gomp_ptrlock_t; + +static inline void gomp_ptrlock_init (gomp_ptrlock_t *ptrlock, void *ptr) +{ + *ptrlock = ptr; +} + +extern void *gomp_ptrlock_get_slow (gomp_ptrlock_t *ptrlock); +static inline void *gomp_ptrlock_get (gomp_ptrlock_t *ptrlock) +{ + if ((uintptr_t) *ptrlock > 2) + return *ptrlock; + + if (__sync_bool_compare_and_swap (ptrlock, NULL, (uintptr_t) 1)) + return NULL; + + return gomp_ptrlock_get_slow (ptrlock); +} + +extern void gomp_ptrlock_set_slow (gomp_ptrlock_t *ptrlock, void *ptr); +static inline void gomp_ptrlock_set (gomp_ptrlock_t *ptrlock, void *ptr) +{ + if (!__sync_bool_compare_and_swap (ptrlock, (uintptr_t) 1, ptr)) + gomp_ptrlock_set_slow (ptrlock, ptr); +} + +static inline void gomp_ptrlock_destroy (gomp_ptrlock_t *ptrlock) +{ +} + +#endif /* GOMP_PTRLOCK_H */ diff --git a/libgomp/config/linux/s390/futex.h b/libgomp/config/linux/s390/futex.h index 9b3820c0d97..3c4e1fc5d25 100644 --- a/libgomp/config/linux/s390/futex.h +++ b/libgomp/config/linux/s390/futex.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2005 Free Software Foundation, Inc. +/* Copyright (C) 2005, 2008 Free Software Foundation, Inc. Contributed by Jakub Jelinek <jakub@redhat.com>. This file is part of the GNU OpenMP Library (libgomp). @@ -28,10 +28,8 @@ /* Provide target-specific access to the futex system call. */ #include <sys/syscall.h> -#define FUTEX_WAIT 0 -#define FUTEX_WAKE 1 -static inline void +static inline long sys_futex0 (int *addr, int op, int val) { register long int gpr2 __asm__ ("2"); @@ -49,16 +47,41 @@ sys_futex0 (int *addr, int op, int val) : "i" (SYS_futex), "0" (gpr2), "d" (gpr3), "d" (gpr4), "d" (gpr5) : "memory"); + return gpr2; } static inline void futex_wait (int *addr, int val) { - sys_futex0 (addr, FUTEX_WAIT, val); + long err = sys_futex0 (addr, gomp_futex_wait, val); + if (__builtin_expect (err == -ENOSYS, 0)) + { + gomp_futex_wait &= ~FUTEX_PRIVATE_FLAG; + gomp_futex_wake &= ~FUTEX_PRIVATE_FLAG; + sys_futex0 (addr, gomp_futex_wait, val); + } } static inline void futex_wake (int *addr, int count) { - sys_futex0 (addr, FUTEX_WAKE, count); + long err = sys_futex0 (addr, gomp_futex_wake, count); + if (__builtin_expect (err == -ENOSYS, 0)) + { + gomp_futex_wait &= ~FUTEX_PRIVATE_FLAG; + gomp_futex_wake &= ~FUTEX_PRIVATE_FLAG; + sys_futex0 (addr, gomp_futex_wake, count); + } +} + +static inline void +cpu_relax (void) +{ + __asm volatile ("" : : : "memory"); +} + +static inline void +atomic_write_barrier (void) +{ + __sync_synchronize (); } diff --git a/libgomp/config/linux/sem.c b/libgomp/config/linux/sem.c index 798e3f1f2c0..5615bc580ee 100644 --- a/libgomp/config/linux/sem.c +++ b/libgomp/config/linux/sem.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2005 Free Software Foundation, Inc. +/* Copyright (C) 2005, 2008 Free Software Foundation, Inc. Contributed by Richard Henderson <rth@redhat.com>. This file is part of the GNU OpenMP Library (libgomp). @@ -29,8 +29,7 @@ mechanism for libgomp. This type is private to the library. This implementation uses atomic instructions and the futex syscall. */ -#include "libgomp.h" -#include "futex.h" +#include "wait.h" void @@ -44,7 +43,7 @@ gomp_sem_wait_slow (gomp_sem_t *sem) if (__sync_bool_compare_and_swap (sem, val, val - 1)) return; } - futex_wait (sem, -1); + do_wait (sem, -1); } } diff --git a/libgomp/config/linux/sparc/futex.h b/libgomp/config/linux/sparc/futex.h index 7b1cc837956..b9bc387355f 100644 --- a/libgomp/config/linux/sparc/futex.h +++ b/libgomp/config/linux/sparc/futex.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2005 Free Software Foundation, Inc. +/* Copyright (C) 2005, 2008 Free Software Foundation, Inc. Contributed by Jakub Jelinek <jakub@redhat.com>. This file is part of the GNU OpenMP Library (libgomp). @@ -28,10 +28,8 @@ /* Provide target-specific access to the futex system call. */ #include <sys/syscall.h> -#define FUTEX_WAIT 0 -#define FUTEX_WAKE 1 -static inline void +static inline long sys_futex0 (int *addr, int op, int val) { register long int g1 __asm__ ("g1"); @@ -47,9 +45,9 @@ sys_futex0 (int *addr, int op, int val) o3 = 0; #ifdef __arch64__ -# define SYSCALL_STRING "ta\t0x6d" +# define SYSCALL_STRING "ta\t0x6d; bcs,a,pt %%xcc, 1f; sub %%g0, %%o0, %%o0; 1:" #else -# define SYSCALL_STRING "ta\t0x10" +# define SYSCALL_STRING "ta\t0x10; bcs,a 1f; sub %%g0, %%o0, %%o0; 1:" #endif __asm volatile (SYSCALL_STRING @@ -65,16 +63,49 @@ sys_futex0 (int *addr, int op, int val) "f48", "f50", "f52", "f54", "f56", "f58", "f60", "f62", #endif "cc", "memory"); + return o0; } static inline void futex_wait (int *addr, int val) { - sys_futex0 (addr, FUTEX_WAIT, val); + long err = sys_futex0 (addr, gomp_futex_wait, val); + if (__builtin_expect (err == ENOSYS, 0)) + { + gomp_futex_wait &= ~FUTEX_PRIVATE_FLAG; + gomp_futex_wake &= ~FUTEX_PRIVATE_FLAG; + sys_futex0 (addr, gomp_futex_wait, val); + } } static inline void futex_wake (int *addr, int count) { - sys_futex0 (addr, FUTEX_WAKE, count); + long err = sys_futex0 (addr, gomp_futex_wake, count); + if (__builtin_expect (err == ENOSYS, 0)) + { + gomp_futex_wait &= ~FUTEX_PRIVATE_FLAG; + gomp_futex_wake &= ~FUTEX_PRIVATE_FLAG; + sys_futex0 (addr, gomp_futex_wake, count); + } +} + +static inline void +cpu_relax (void) +{ +#if defined __arch64__ || defined __sparc_v9__ + __asm volatile ("membar #LoadLoad" : : : "memory"); +#else + __asm volatile ("" : : : "memory"); +#endif +} + +static inline void +atomic_write_barrier (void) +{ +#if defined __arch64__ || defined __sparc_v9__ + __asm volatile ("membar #StoreStore" : : : "memory"); +#else + __sync_synchronize (); +#endif } diff --git a/libgomp/config/linux/wait.h b/libgomp/config/linux/wait.h new file mode 100644 index 00000000000..21f0ab8756c --- /dev/null +++ b/libgomp/config/linux/wait.h @@ -0,0 +1,68 @@ +/* Copyright (C) 2008 Free Software Foundation, Inc. + Contributed by Jakub Jelinek <jakub@redhat.com>. + + This file is part of the GNU OpenMP Library (libgomp). + + Libgomp is free software; you can redistribute it and/or modify it + under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation; either version 2.1 of the License, or + (at your option) any later version. + + Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for + more details. + + You should have received a copy of the GNU Lesser General Public License + along with libgomp; see the file COPYING.LIB. If not, write to the + Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +/* As a special exception, if you link this library with other files, some + of which are compiled with GCC, to produce an executable, this library + does not by itself cause the resulting executable to be covered by the + GNU General Public License. This exception does not however invalidate + any other reasons why the executable file might be covered by the GNU + General Public License. */ + +/* This is a Linux specific implementation of a mutex synchronization + mechanism for libgomp. This type is private to the library. This + implementation uses atomic instructions and the futex syscall. */ + +#ifndef GOMP_WAIT_H +#define GOMP_WAIT_H 1 + +#include "libgomp.h" +#include <errno.h> + +#define FUTEX_WAIT 0 +#define FUTEX_WAKE 1 +#define FUTEX_PRIVATE_FLAG 128L + +#ifdef HAVE_ATTRIBUTE_VISIBILITY +# pragma GCC visibility push(hidden) +#endif + +extern long int gomp_futex_wait, gomp_futex_wake; + +#include "futex.h" + +static inline void do_wait (int *addr, int val) +{ + unsigned long long i, count = gomp_spin_count_var; + + if (__builtin_expect (gomp_managed_threads > gomp_available_cpus, 0)) + count = gomp_throttled_spin_count_var; + for (i = 0; i < count; i++) + if (__builtin_expect (*addr != val, 0)) + return; + else + cpu_relax (); + futex_wait (addr, val); +} + +#ifdef HAVE_ATTRIBUTE_VISIBILITY +# pragma GCC visibility pop +#endif + +#endif /* GOMP_WAIT_H */ diff --git a/libgomp/config/linux/x86/futex.h b/libgomp/config/linux/x86/futex.h index 4f9aac2ddbb..36af6aad93b 100644 --- a/libgomp/config/linux/x86/futex.h +++ b/libgomp/config/linux/x86/futex.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2005 Free Software Foundation, Inc. +/* Copyright (C) 2005, 2008 Free Software Foundation, Inc. Contributed by Richard Henderson <rth@redhat.com>. This file is part of the GNU OpenMP Library (libgomp). @@ -27,9 +27,6 @@ /* Provide target-specific access to the futex system call. */ -#define FUTEX_WAIT 0 -#define FUTEX_WAKE 1 - #ifdef __LP64__ # ifndef SYS_futex # define SYS_futex 202 @@ -38,14 +35,26 @@ static inline void futex_wait (int *addr, int val) { - register long r10 __asm__("%r10") = 0; + register long r10 __asm__("%r10"); long res; + r10 = 0; __asm volatile ("syscall" : "=a" (res) - : "0"(SYS_futex), "D" (addr), "S"(FUTEX_WAIT), - "d"(val), "r"(r10) + : "0" (SYS_futex), "D" (addr), "S" (gomp_futex_wait), + "d" (val), "r" (r10) : "r11", "rcx", "memory"); + if (__builtin_expect (res == -ENOSYS, 0)) + { + gomp_futex_wait &= ~FUTEX_PRIVATE_FLAG; + gomp_futex_wake &= ~FUTEX_PRIVATE_FLAG; + r10 = 0; + __asm volatile ("syscall" + : "=a" (res) + : "0" (SYS_futex), "D" (addr), "S" (gomp_futex_wait), + "d" (val), "r" (r10) + : "r11", "rcx", "memory"); + } } static inline void @@ -55,8 +64,19 @@ futex_wake (int *addr, int count) __asm volatile ("syscall" : "=a" (res) - : "0"(SYS_futex), "D" (addr), "S"(FUTEX_WAKE), "d"(count) + : "0" (SYS_futex), "D" (addr), "S" (gomp_futex_wake), + "d" (count) : "r11", "rcx", "memory"); + if (__builtin_expect (res == -ENOSYS, 0)) + { + gomp_futex_wait &= ~FUTEX_PRIVATE_FLAG; + gomp_futex_wake &= ~FUTEX_PRIVATE_FLAG; + __asm volatile ("syscall" + : "=a" (res) + : "0" (SYS_futex), "D" (addr), "S" (gomp_futex_wake), + "d" (count) + : "r11", "rcx", "memory"); + } } #else # ifndef SYS_futex @@ -65,7 +85,7 @@ futex_wake (int *addr, int count) # ifdef __PIC__ -static inline void +static inline long sys_futex0 (int *addr, int op, int val) { long res; @@ -77,11 +97,12 @@ sys_futex0 (int *addr, int op, int val) : "0"(SYS_futex), "r" (addr), "c"(op), "d"(val), "S"(0) : "memory"); + return res; } # else -static inline void +static inline long sys_futex0 (int *addr, int op, int val) { long res; @@ -91,6 +112,7 @@ sys_futex0 (int *addr, int op, int val) : "0"(SYS_futex), "b" (addr), "c"(op), "d"(val), "S"(0) : "memory"); + return res; } # endif /* __PIC__ */ @@ -98,13 +120,37 @@ sys_futex0 (int *addr, int op, int val) static inline void futex_wait (int *addr, int val) { - sys_futex0 (addr, FUTEX_WAIT, val); + long res = sys_futex0 (addr, gomp_futex_wait, val); + if (__builtin_expect (res == -ENOSYS, 0)) + { + gomp_futex_wait &= ~FUTEX_PRIVATE_FLAG; + gomp_futex_wake &= ~FUTEX_PRIVATE_FLAG; + sys_futex0 (addr, gomp_futex_wait, val); + } } static inline void futex_wake (int *addr, int count) { - sys_futex0 (addr, FUTEX_WAKE, count); + long res = sys_futex0 (addr, gomp_futex_wake, count); + if (__builtin_expect (res == -ENOSYS, 0)) + { + gomp_futex_wait &= ~FUTEX_PRIVATE_FLAG; + gomp_futex_wake &= ~FUTEX_PRIVATE_FLAG; + sys_futex0 (addr, gomp_futex_wake, count); + } } #endif /* __LP64__ */ + +static inline void +cpu_relax (void) +{ + __asm volatile ("rep; nop" : : : "memory"); +} + +static inline void +atomic_write_barrier (void) +{ + __sync_synchronize (); +} |