atomic ops:

    my_atomic_XX_t -> intXX, no implicit locking anymore
    simplified framework, support for requested cleanups

4 files changed, 84 insertions, 318 deletions

diff --git a/include/atomic/nolock.h b/include/atomic/nolock.h
index cf21a94c7de..458cb3e4f66 100644
--- a/include/atomic/nolock.h
+++ b/include/atomic/nolock.h
@@ -27,137 +27,9 @@
 #endif
 #endif
 
-#ifdef make_atomic_add_body8
+#ifdef make_atomic_add_body
 
-#ifdef HAVE_INLINE
-
-#define make_atomic_add(S)					\
-static inline uint ## S _my_atomic_add ## S(			\
-        my_atomic_ ## S ## _t *a, uint ## S v)			\
-{								\
-  make_atomic_add_body ## S;					\
-  return v;							\
-}
-
-#define make_atomic_swap(S)					\
-static inline uint ## S _my_atomic_swap ## S(			\
-        my_atomic_ ## S ## _t *a, uint ## S v)			\
-{								\
-  make_atomic_swap_body ## S;					\
-  return v;							\
-}
-
-#define make_atomic_cas(S)					\
-static inline uint _my_atomic_cas ## S(my_atomic_ ## S ## _t *a,\
-        uint ## S *cmp, uint ## S set)				\
-{								\
-  uint8 ret;							\
-  make_atomic_cas_body ## S;					\
-  return ret;							\
-}
-
-#define make_atomic_load(S)					\
-static inline uint ## S _my_atomic_load ## S(			\
-        my_atomic_ ## S ## _t *a)				\
-{								\
-  uint ## S ret;						\
-  make_atomic_load_body ## S;					\
-  return ret;							\
-}
-
-#define make_atomic_store(S)					\
-static inline void _my_atomic_store ## S(			\
-        my_atomic_ ## S ## _t *a, uint ## S v)			\
-{								\
-  make_atomic_store_body ## S;					\
-}
-
-#else /* no inline functions */
-
-#define make_atomic_add(S)					\
-extern uint ## S _my_atomic_add ## S(				\
-        my_atomic_ ## S ## _t *a, uint ## S v);
-
-#define make_atomic_swap(S)					\
-extern uint ## S _my_atomic_swap ## S(				\
-        my_atomic_ ## S ## _t *a, uint ## S v);
-
-#define make_atomic_cas(S)					\
-extern uint _my_atomic_cas ## S(my_atomic_ ## S ## _t *a,	\
-        uint ## S *cmp, uint ## S set);
-
-#define make_atomic_load(S)					\
-extern uint ## S _my_atomic_load ## S(				\
-        my_atomic_ ## S ## _t *a);
-
-#define make_atomic_store(S)					\
-extern void _my_atomic_store ## S(				\
-        my_atomic_ ## S ## _t *a, uint ## S v);
-
-#endif
-
-make_atomic_add( 8)
-make_atomic_add(16)
-make_atomic_add(32)
-
-make_atomic_cas( 8)
-make_atomic_cas(16)
-make_atomic_cas(32)
-
-make_atomic_load( 8)
-make_atomic_load(16)
-make_atomic_load(32)
-
-make_atomic_store( 8)
-make_atomic_store(16)
-make_atomic_store(32)
-
-make_atomic_swap( 8)
-make_atomic_swap(16)
-make_atomic_swap(32)
-
-#undef make_atomic_add_body8
-#undef make_atomic_cas_body8
-#undef make_atomic_load_body8
-#undef make_atomic_store_body8
-#undef make_atomic_swap_body8
-#undef make_atomic_add_body16
-#undef make_atomic_cas_body16
-#undef make_atomic_load_body16
-#undef make_atomic_store_body16
-#undef make_atomic_swap_body16
-#undef make_atomic_add_body32
-#undef make_atomic_cas_body32
-#undef make_atomic_load_body32
-#undef make_atomic_store_body32
-#undef make_atomic_swap_body32
-#undef make_atomic_add
-#undef make_atomic_cas
-#undef make_atomic_load
-#undef make_atomic_store
-#undef make_atomic_swap
-
-#define my_atomic_add8(a,v,L)  _my_atomic_add8(a,v)
-#define my_atomic_add16(a,v,L) _my_atomic_add16(a,v)
-#define my_atomic_add32(a,v,L) _my_atomic_add32(a,v)
-
-#define my_atomic_cas8(a,c,v,L)  _my_atomic_cas8(a,c,v)
-#define my_atomic_cas16(a,c,v,L) _my_atomic_cas16(a,c,v)
-#define my_atomic_cas32(a,c,v,L) _my_atomic_cas32(a,c,v)
-
-#define my_atomic_load8(a,L)  _my_atomic_load8(a)
-#define my_atomic_load16(a,L) _my_atomic_load16(a)
-#define my_atomic_load32(a,L) _my_atomic_load32(a)
-
-#define my_atomic_store8(a,v,L)  _my_atomic_store8(a,v)
-#define my_atomic_store16(a,v,L) _my_atomic_store16(a,v)
-#define my_atomic_store32(a,v,L) _my_atomic_store32(a,v)
-
-#define my_atomic_swap8(a,v,L)  _my_atomic_swap8(a,v)
-#define my_atomic_swap16(a,v,L) _my_atomic_swap16(a,v)
-#define my_atomic_swap32(a,v,L) _my_atomic_swap32(a,v)
-
-#define my_atomic_rwlock_t typedef int
+typedef struct { } my_atomic_rwlock_t;
 #define my_atomic_rwlock_destroy(name)
 #define my_atomic_rwlock_init(name)
 #define my_atomic_rwlock_rdlock(name)
diff --git a/include/atomic/rwlock.h b/include/atomic/rwlock.h
index ca5be29ab9b..12d0dd3f069 100644
--- a/include/atomic/rwlock.h
+++ b/include/atomic/rwlock.h
@@ -16,12 +16,6 @@
 
 typedef struct {pthread_rwlock_t rw;} my_atomic_rwlock_t;
 
-#ifdef MY_ATOMIC_EXTRA_DEBUG
-#define CHECK_RW if (rw) if (a->rw) assert(rw == a->rw); else a->rw=rw;
-#else
-#define CHECK_RW
-#endif
-
 #ifdef MY_ATOMIC_MODE_DUMMY
 /*
   the following can never be enabled by ./configure, one need to put #define in
@@ -36,6 +30,7 @@ typedef struct {pthread_rwlock_t rw;} my_atomic_rwlock_t;
 #define my_atomic_rwlock_wrlock(name)
 #define my_atomic_rwlock_rdunlock(name)
 #define my_atomic_rwlock_wrunlock(name)
+#define MY_ATOMIC_MODE "dummy (non-atomic)"
 #else
 #define my_atomic_rwlock_destroy(name)     pthread_rwlock_destroy(& (name)->rw)
 #define my_atomic_rwlock_init(name)        pthread_rwlock_init(& (name)->rw, 0)
@@ -43,119 +38,12 @@ typedef struct {pthread_rwlock_t rw;} my_atomic_rwlock_t;
 #define my_atomic_rwlock_wrlock(name)      pthread_rwlock_wrlock(& (name)->rw)
 #define my_atomic_rwlock_rdunlock(name)    pthread_rwlock_unlock(& (name)->rw)
 #define my_atomic_rwlock_wrunlock(name)    pthread_rwlock_unlock(& (name)->rw)
+#define MY_ATOMIC_MODE "rwlocks"
 #endif
 
-#ifdef HAVE_INLINE
-
-#define make_atomic_add(S)						\
-static inline uint ## S my_atomic_add ## S(				\
-        my_atomic_ ## S ## _t *a, uint ## S v, my_atomic_rwlock_t *rw)	\
-{									\
-  uint ## S ret;							\
-  CHECK_RW;								\
-  if (rw) my_atomic_rwlock_wrlock(rw);					\
-  ret= a->val;								\
-  a->val+= v;								\
-  if (rw) my_atomic_rwlock_wrunlock(rw);				\
-  return ret;								\
-}
-
-#define make_atomic_swap(S)						\
-static inline uint ## S my_atomic_swap ## S(				\
-        my_atomic_ ## S ## _t *a, uint ## S v, my_atomic_rwlock_t *rw)	\
-{									\
-  uint ## S ret;							\
-  CHECK_RW;								\
-  if (rw) my_atomic_rwlock_wrlock(rw);					\
-  ret= a->val;								\
-  a->val= v;								\
-  if (rw) my_atomic_rwlock_wrunlock(rw);				\
-  return ret;								\
-}
-
-#define make_atomic_cas(S)						\
-static inline uint my_atomic_cas ## S(my_atomic_ ## S ## _t *a,		\
-        uint ## S *cmp, uint ## S set, my_atomic_rwlock_t *rw)		\
-{									\
-  uint ret;								\
-  CHECK_RW;								\
-  if (rw) my_atomic_rwlock_wrlock(rw);					\
-  if (ret= (a->val == *cmp)) a->val= set; else *cmp=a->val;		\
-  if (rw) my_atomic_rwlock_wrunlock(rw);				\
-  return ret;								\
-}
-
-#define make_atomic_load(S)						\
-static inline uint ## S my_atomic_load ## S(				\
-        my_atomic_ ## S ## _t *a, my_atomic_rwlock_t *rw)		\
-{									\
-  uint ## S ret;							\
-  CHECK_RW;								\
-  if (rw) my_atomic_rwlock_wrlock(rw);					\
-  ret= a->val;								\
-  if (rw) my_atomic_rwlock_wrunlock(rw);				\
-  return ret;								\
-}
-
-#define make_atomic_store(S)						\
-static inline void my_atomic_store ## S(				\
-        my_atomic_ ## S ## _t *a, uint ## S v, my_atomic_rwlock_t *rw)	\
-{									\
-  CHECK_RW;								\
-  if (rw) my_atomic_rwlock_rdlock(rw);					\
-  (a)->val= (v);							\
-  if (rw) my_atomic_rwlock_rdunlock(rw);				\
-}
-
-#else /* no inline functions */
-
-#define make_atomic_add(S)						\
-extern uint ## S my_atomic_add ## S(					\
-        my_atomic_ ## S ## _t *a, uint ## S v, my_atomic_rwlock_t *rw);
-
-#define make_atomic_swap(S)						\
-extern uint ## S my_atomic_swap ## S(					\
-        my_atomic_ ## S ## _t *a, uint ## S v, my_atomic_rwlock_t *rw);
-
-#define make_atomic_cas(S)						\
-extern uint my_atomic_cas ## S(my_atomic_ ## S ## _t *a,		\
-        uint ## S *cmp, uint ## S set, my_atomic_rwlock_t *rw);
-
-#define make_atomic_load(S)						\
-extern uint ## S my_atomic_load ## S(					\
-        my_atomic_ ## S ## _t *a, my_atomic_rwlock_t *rw);
-
-#define make_atomic_store(S)						\
-extern void my_atomic_store ## S(					\
-        my_atomic_ ## S ## _t *a, uint ## S v, my_atomic_rwlock_t *rw);
-
-#endif
-
-make_atomic_add( 8)
-make_atomic_add(16)
-make_atomic_add(32)
-make_atomic_add(64)
-make_atomic_cas( 8)
-make_atomic_cas(16)
-make_atomic_cas(32)
-make_atomic_cas(64)
-make_atomic_load( 8)
-make_atomic_load(16)
-make_atomic_load(32)
-make_atomic_load(64)
-make_atomic_store( 8)
-make_atomic_store(16)
-make_atomic_store(32)
-make_atomic_store(64)
-make_atomic_swap( 8)
-make_atomic_swap(16)
-make_atomic_swap(32)
-make_atomic_swap(64)
-#undef make_atomic_add
-#undef make_atomic_cas
-#undef make_atomic_load
-#undef make_atomic_store
-#undef make_atomic_swap
-#undef CHECK_RW
-
+#define make_atomic_add_body(S)     int ## S sav; sav= *a; *a+= v; v=sav;
+#define make_atomic_swap_body(S)    int ## S sav; sav= *a; *a= v; v=sav;
+#define make_atomic_cas_body(S)     if ((ret= (*a == *cmp))) *a= set; else *cmp=*a;
+#define make_atomic_load_body(S)    ret= *a;
+#define make_atomic_store_body(S)   *a= v;
 
diff --git a/include/atomic/x86-gcc.h b/include/atomic/x86-gcc.h
index 7576db54d69..df6fcdc5ad2 100644
--- a/include/atomic/x86-gcc.h
+++ b/include/atomic/x86-gcc.h
@@ -16,44 +16,38 @@
 
 /*
   XXX 64-bit atomic operations can be implemented using
-  cmpxchg8b, if necessary
+  cmpxchg8b, if necessary. Though I've heard that not all 64-bit
+  architectures support double-word (128-bit) cas.
 */
 
+#define MY_ATOMIC_MODE "gcc-x86" ## LOCK
+
 /* fix -ansi errors while maintaining readability */
+#ifndef asm
 #define asm __asm__
+#endif
 
-#define make_atomic_add_body8					\
-  asm volatile (LOCK "xadd %0, %1;" : "+r" (v) , "+m" (a->val))
-#define make_atomic_swap_body8					\
-  asm volatile ("xchg %0, %1;" : "+r" (v) , "+m" (a->val))
-#define make_atomic_cas_body8					\
+#define make_atomic_add_body(S)					\
+  asm volatile (LOCK "xadd %0, %1;" : "+r" (v) , "+m" (*a))
+#define make_atomic_swap_body(S)				\
+  asm volatile ("xchg %0, %1;" : "+r" (v) , "+m" (*a))
+#define make_atomic_cas_body(S)					\
   asm volatile (LOCK "cmpxchg %3, %0; setz %2;"			\
-               : "+m" (a->val), "+a" (*cmp), "=q" (ret): "r" (set))
+               : "+m" (*a), "+a" (*cmp), "=q" (ret): "r" (set))
 
 #ifdef MY_ATOMIC_MODE_DUMMY
-#define make_atomic_load_body8   ret=a->val
-#define make_atomic_store_body8	 a->val=v
+#define make_atomic_load_body(S)   ret=*a
+#define make_atomic_store_body(S)  *a=v
 #else
 /*
   Actually 32-bit reads/writes are always atomic on x86
   But we add LOCK here anyway to force memory barriers
 */
-#define make_atomic_load_body8					\
+#define make_atomic_load_body(S)				\
   ret=0;							\
   asm volatile (LOCK "cmpxchg %2, %0"				\
-               : "+m" (a->val), "+a" (ret): "r" (ret))
-#define make_atomic_store_body8					\
-  asm volatile ("xchg %0, %1;" : "+m" (a->val) : "r" (v))
+               : "+m" (*a), "+a" (ret): "r" (ret))
+#define make_atomic_store_body(S)				\
+  asm volatile ("xchg %0, %1;" : "+m" (*a) : "r" (v))
 #endif
 
-#define make_atomic_add_body16   make_atomic_add_body8
-#define make_atomic_add_body32   make_atomic_add_body8
-#define make_atomic_cas_body16   make_atomic_cas_body8
-#define make_atomic_cas_body32   make_atomic_cas_body8
-#define make_atomic_load_body16  make_atomic_load_body8
-#define make_atomic_load_body32  make_atomic_load_body8
-#define make_atomic_store_body16 make_atomic_store_body8
-#define make_atomic_store_body32 make_atomic_store_body8
-#define make_atomic_swap_body16  make_atomic_swap_body8
-#define make_atomic_swap_body32  make_atomic_swap_body8
-
diff --git a/include/atomic/x86-msvc.h b/include/atomic/x86-msvc.h
index 19645551196..a7e14d0e926 100644
--- a/include/atomic/x86-msvc.h
+++ b/include/atomic/x86-msvc.h
@@ -23,63 +23,75 @@
 // (InterlockedCompareExchange, InterlockedCompareExchange16
 // InterlockedExchangeAdd, InterlockedExchange)
 
-#define make_atomic_add_body(REG)				\
-  _asm {							\
-    _asm mov   REG, v						\
-    _asm LOCK  xadd a->val, REG					\
-    _asm movzx v, REG						\
+#ifndef _atomic_h_cleanup_
+#define _atomic_h_cleanup_ "atomic/x86-msvc.h"
+
+#define MY_ATOMIC_MODE "msvc-x86" ## LOCK
+
+#define make_atomic_add_body(S)				\
+  _asm {						\
+    _asm mov   reg_ ## S, v				\
+    _asm LOCK  xadd *a, reg_ ## S			\
+    _asm movzx v, reg_ ## S				\
   }
-#define make_atomic_cas_body(AREG,REG2)				\
-  _asm {							\
-    _asm mov    AREG, *cmp					\
-    _asm mov    REG2, set					\
-    _asm LOCK cmpxchg a->val, REG2				\
-    _asm mov    *cmp, AREG					\
-    _asm setz   al						\
-    _asm movzx  ret, al						\
+#define make_atomic_cas_body(S)				\
+  _asm {						\
+    _asm mov    areg_ ## S, *cmp			\
+    _asm mov    reg2_ ## S, set				\
+    _asm LOCK cmpxchg *a, reg2_ ## S			\
+    _asm mov    *cmp, areg_ ## S			\
+    _asm setz   al					\
+    _asm movzx  ret, al					\
   }
-#define make_atomic_swap_body(REG)				\
-  _asm {							\
-    _asm mov    REG, v						\
-    _asm xchg   a->val, REG					\
-    _asm mov    v, REG						\
+#define make_atomic_swap_body(S)			\
+  _asm {						\
+    _asm mov    reg_ ## S, v				\
+    _asm xchg   *a, reg_ ## S				\
+    _asm mov    v, reg_ ## S				\
   }
 
 #ifdef MY_ATOMIC_MODE_DUMMY
-#define make_atomic_load_body(AREG,REG)   ret=a->val
-#define make_atomic_store_body(REG)	  a->val=v
+#define make_atomic_load_body(S)        ret=*a
+#define make_atomic_store_body(S)       *a=v
 #else
 /*
   Actually 32-bit reads/writes are always atomic on x86
   But we add LOCK here anyway to force memory barriers
 */
-#define make_atomic_load_body(AREG,REG2)			\
-  _asm {							\
-    _asm mov    AREG, 0						\
-    _asm mov    REG2, AREG					\
-    _asm LOCK cmpxchg a->val, REG2				\
-    _asm mov    ret, AREG					\
+#define make_atomic_load_body(S)			\
+  _asm {						\
+    _asm mov    areg_ ## S, 0				\
+    _asm mov    reg2_ ## S, areg_ ## S			\
+    _asm LOCK cmpxchg *a, reg2_ ## S			\
+    _asm mov    ret, areg_ ## S				\
   }
-#define make_atomic_store_body(REG)				\
-  _asm {							\
-    _asm mov    REG, v						\
-    _asm xchg   a->val, REG					\
+#define make_atomic_store_body(S)			\
+  _asm {						\
+    _asm mov    reg_ ## S, v				\
+    _asm xchg   *a, reg_ ## S				\
   }
 #endif
 
-#define make_atomic_add_body8	 make_atomic_add_body(al)
-#define make_atomic_add_body16   make_atomic_add_body(ax)
-#define make_atomic_add_body32   make_atomic_add_body(eax)
-#define make_atomic_cas_body8    make_atomic_cas_body(al, bl)
-#define make_atomic_cas_body16   make_atomic_cas_body(ax, bx)
-#define make_atomic_cas_body32   make_atomic_cas_body(eax, ebx)
-#define make_atomic_load_body8    make_atomic_load_body(al, bl)
-#define make_atomic_load_body16   make_atomic_load_body(ax, bx)
-#define make_atomic_load_body32   make_atomic_load_body(eax, ebx)
-#define make_atomic_store_body8  make_atomic_store_body(al)
-#define make_atomic_store_body16 make_atomic_store_body(ax)
-#define make_atomic_store_body32 make_atomic_store_body(eax)
-#define make_atomic_swap_body8   make_atomic_swap_body(al)
-#define make_atomic_swap_body16  make_atomic_swap_body(ax)
-#define make_atomic_swap_body32  make_atomic_swap_body(eax)
+#define reg_8           al
+#define reg_16          ax
+#define reg_32          eax
+#define areg_8          al
+#define areg_16         ax
+#define areg_32         eax
+#define reg2_8          bl
+#define reg2_16         bx
+#define reg2_32         ebx
+
+#else /* cleanup */
+
+#undef reg_8
+#undef reg_16
+#undef reg_32
+#undef areg_8
+#undef areg_16
+#undef areg_32
+#undef reg2_8
+#undef reg2_16
+#undef reg2_32
+#endif