Implement char/short/int-wide primitives using GCC built-in atomic/sync

* src/Makefile.am (EXTRA_DIST): Add gcc/generic-arithm.template,
gcc/generic-small.template entries.
* src/Makefile.am (nobase_private_HEADERS): Move gcc/generic-arithm.h,
atomic_ops/sysdeps/gcc/generic-small.h to BUILT_SOURCES.
* src/Makefile.am (atomic_ops/sysdeps/gcc/generic-arithm.h,
atomic_ops/sysdeps/gcc/generic-small.h): New rules (to generate
AO_char/short/int_X primitives included from gcc/generic.h file.
* src/atomic_ops/sysdeps/gcc/generic-arithm.template: New template
file (based on code moved from generic-arithm.h).
* src/atomic_ops/sysdeps/gcc/generic-small.template: New template file
(based on code moved from generic-small.h).
* src/atomic_ops/sysdeps/gcc/generic-arithm.h: Regenerate.
* src/atomic_ops/sysdeps/gcc/generic-small.h: Likewise.

5 files changed, 817 insertions, 12 deletions

diff --git a/src/Makefile.am b/src/Makefile.am
index db1b83d..f611ac4 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -17,6 +17,8 @@ EXTRA_DIST = Makefile.msft atomic_ops/sysdeps/README \
         atomic_ops/generalize-arithm.template \
         atomic_ops/generalize-small.template \
         atomic_ops/sysdeps/ao_t_is_int.template \
+        atomic_ops/sysdeps/gcc/generic-arithm.template \
+        atomic_ops/sysdeps/gcc/generic-small.template \
         atomic_ops/sysdeps/loadstore/acquire_release_volatile.template \
         atomic_ops/sysdeps/loadstore/atomic_load.template \
         atomic_ops/sysdeps/loadstore/atomic_store.template \
@@ -27,6 +29,8 @@ EXTRA_DIST = Makefile.msft atomic_ops/sysdeps/README \
 BUILT_SOURCES = atomic_ops/generalize-arithm.h \
         atomic_ops/generalize-small.h \
         atomic_ops/sysdeps/ao_t_is_int.h \
+        atomic_ops/sysdeps/gcc/generic-arithm.h \
+        atomic_ops/sysdeps/gcc/generic-small.h \
         atomic_ops/sysdeps/loadstore/acquire_release_volatile.h \
         atomic_ops/sysdeps/loadstore/atomic_load.h \
         atomic_ops/sysdeps/loadstore/atomic_store.h \
@@ -68,8 +72,6 @@ nobase_private_HEADERS = atomic_ops/ao_version.h \
           atomic_ops/sysdeps/gcc/arm.h \
           atomic_ops/sysdeps/gcc/avr32.h \
           atomic_ops/sysdeps/gcc/cris.h \
-          atomic_ops/sysdeps/gcc/generic-arithm.h \
-          atomic_ops/sysdeps/gcc/generic-small.h \
           atomic_ops/sysdeps/gcc/generic.h \
           atomic_ops/sysdeps/gcc/hexagon.h \
           atomic_ops/sysdeps/gcc/hppa.h \
@@ -120,6 +122,20 @@ atomic_ops/sysdeps/ao_t_is_int.h: atomic_ops/sysdeps/ao_t_is_int.template
 	sed -e s:XBAR:write:g $? >> $@
 	sed -e s:XBAR:read:g $? >> $@
 
+atomic_ops/sysdeps/gcc/generic-arithm.h: \
+        atomic_ops/sysdeps/gcc/generic-arithm.template
+	sed -e s:XSIZE:char:g -e s:XCTYPE:unsigned/**/char:g $? > $@
+	sed -e s:XSIZE:short:g -e s:XCTYPE:unsigned/**/short:g $? >> $@
+	sed -e s:XSIZE:int:g -e s:XCTYPE:unsigned:g $? >> $@
+	sed -e s:XSIZE_::g -e s:XCTYPE:AO_t:g $? >> $@
+
+atomic_ops/sysdeps/gcc/generic-small.h: \
+        atomic_ops/sysdeps/gcc/generic-small.template
+	sed -e s:XSIZE:char:g -e s:XCTYPE:unsigned/**/char:g $? > $@
+	sed -e s:XSIZE:short:g -e s:XCTYPE:unsigned/**/short:g $? >> $@
+	sed -e s:XSIZE:int:g -e s:XCTYPE:unsigned:g $? >> $@
+	sed -e s:XSIZE_::g -e s:XCTYPE:AO_t:g $? >> $@
+
 atomic_ops/sysdeps/loadstore/ordered_loads_only.h: \
         atomic_ops/sysdeps/loadstore/ordered_loads_only.template
 	sed -e s:XSIZE:char:g -e s:XCTYPE:unsigned/**/char:g $? > $@
diff --git a/src/atomic_ops/sysdeps/gcc/generic-arithm.h b/src/atomic_ops/sysdeps/gcc/generic-arithm.h
index ad703a8..00b58b9 100644
--- a/src/atomic_ops/sysdeps/gcc/generic-arithm.h
+++ b/src/atomic_ops/sysdeps/gcc/generic-arithm.h
@@ -15,6 +15,390 @@
  *
  */
 
+AO_INLINE unsigned/**/char
+AO_char_fetch_and_add(volatile unsigned/**/char *addr, unsigned/**/char incr)
+{
+  return __atomic_fetch_add(addr, incr, __ATOMIC_RELAXED);
+}
+#define AO_HAVE_char_fetch_and_add
+
+AO_INLINE unsigned/**/char
+AO_char_fetch_and_add_acquire(volatile unsigned/**/char *addr, unsigned/**/char incr)
+{
+  return __atomic_fetch_add(addr, incr, __ATOMIC_ACQUIRE);
+}
+#define AO_HAVE_char_fetch_and_add_acquire
+
+AO_INLINE unsigned/**/char
+AO_char_fetch_and_add_release(volatile unsigned/**/char *addr, unsigned/**/char incr)
+{
+  return __atomic_fetch_add(addr, incr, __ATOMIC_RELEASE);
+}
+#define AO_HAVE_char_fetch_and_add_release
+
+AO_INLINE unsigned/**/char
+AO_char_fetch_and_add_full(volatile unsigned/**/char *addr, unsigned/**/char incr)
+{
+  return __atomic_fetch_add(addr, incr, __ATOMIC_SEQ_CST);
+}
+#define AO_HAVE_char_fetch_and_add_full
+
+AO_INLINE void
+AO_char_and(volatile unsigned/**/char *addr, unsigned/**/char value)
+{
+  (void)__atomic_and_fetch(addr, value, __ATOMIC_RELAXED);
+}
+#define AO_HAVE_char_and
+
+AO_INLINE void
+AO_char_and_acquire(volatile unsigned/**/char *addr, unsigned/**/char value)
+{
+  (void)__atomic_and_fetch(addr, value, __ATOMIC_ACQUIRE);
+}
+#define AO_HAVE_char_and_acquire
+
+AO_INLINE void
+AO_char_and_release(volatile unsigned/**/char *addr, unsigned/**/char value)
+{
+  (void)__atomic_and_fetch(addr, value, __ATOMIC_RELEASE);
+}
+#define AO_HAVE_char_and_release
+
+AO_INLINE void
+AO_char_and_full(volatile unsigned/**/char *addr, unsigned/**/char value)
+{
+  (void)__atomic_and_fetch(addr, value, __ATOMIC_SEQ_CST);
+}
+#define AO_HAVE_char_and_full
+
+AO_INLINE void
+AO_char_or(volatile unsigned/**/char *addr, unsigned/**/char value)
+{
+  (void)__atomic_or_fetch(addr, value, __ATOMIC_RELAXED);
+}
+#define AO_HAVE_char_or
+
+AO_INLINE void
+AO_char_or_acquire(volatile unsigned/**/char *addr, unsigned/**/char value)
+{
+  (void)__atomic_or_fetch(addr, value, __ATOMIC_ACQUIRE);
+}
+#define AO_HAVE_char_or_acquire
+
+AO_INLINE void
+AO_char_or_release(volatile unsigned/**/char *addr, unsigned/**/char value)
+{
+  (void)__atomic_or_fetch(addr, value, __ATOMIC_RELEASE);
+}
+#define AO_HAVE_char_or_release
+
+AO_INLINE void
+AO_char_or_full(volatile unsigned/**/char *addr, unsigned/**/char value)
+{
+  (void)__atomic_or_fetch(addr, value, __ATOMIC_SEQ_CST);
+}
+#define AO_HAVE_char_or_full
+
+AO_INLINE void
+AO_char_xor(volatile unsigned/**/char *addr, unsigned/**/char value)
+{
+  (void)__atomic_xor_fetch(addr, value, __ATOMIC_RELAXED);
+}
+#define AO_HAVE_char_xor
+
+AO_INLINE void
+AO_char_xor_acquire(volatile unsigned/**/char *addr, unsigned/**/char value)
+{
+  (void)__atomic_xor_fetch(addr, value, __ATOMIC_ACQUIRE);
+}
+#define AO_HAVE_char_xor_acquire
+
+AO_INLINE void
+AO_char_xor_release(volatile unsigned/**/char *addr, unsigned/**/char value)
+{
+  (void)__atomic_xor_fetch(addr, value, __ATOMIC_RELEASE);
+}
+#define AO_HAVE_char_xor_release
+
+AO_INLINE void
+AO_char_xor_full(volatile unsigned/**/char *addr, unsigned/**/char value)
+{
+  (void)__atomic_xor_fetch(addr, value, __ATOMIC_SEQ_CST);
+}
+#define AO_HAVE_char_xor_full
+/*
+ * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
+ * Copyright (c) 1996-1999 by Silicon Graphics.  All rights reserved.
+ * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P.
+ *
+ *
+ * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
+ * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
+ *
+ * Permission is hereby granted to use or copy this program
+ * for any purpose, provided the above notices are retained on all copies.
+ * Permission to modify the code and to distribute modified code is granted,
+ * provided the above notices are retained, and a notice that the code was
+ * modified is included with the above copyright notice.
+ *
+ */
+
+AO_INLINE unsigned/**/short
+AO_short_fetch_and_add(volatile unsigned/**/short *addr, unsigned/**/short incr)
+{
+  return __atomic_fetch_add(addr, incr, __ATOMIC_RELAXED);
+}
+#define AO_HAVE_short_fetch_and_add
+
+AO_INLINE unsigned/**/short
+AO_short_fetch_and_add_acquire(volatile unsigned/**/short *addr, unsigned/**/short incr)
+{
+  return __atomic_fetch_add(addr, incr, __ATOMIC_ACQUIRE);
+}
+#define AO_HAVE_short_fetch_and_add_acquire
+
+AO_INLINE unsigned/**/short
+AO_short_fetch_and_add_release(volatile unsigned/**/short *addr, unsigned/**/short incr)
+{
+  return __atomic_fetch_add(addr, incr, __ATOMIC_RELEASE);
+}
+#define AO_HAVE_short_fetch_and_add_release
+
+AO_INLINE unsigned/**/short
+AO_short_fetch_and_add_full(volatile unsigned/**/short *addr, unsigned/**/short incr)
+{
+  return __atomic_fetch_add(addr, incr, __ATOMIC_SEQ_CST);
+}
+#define AO_HAVE_short_fetch_and_add_full
+
+AO_INLINE void
+AO_short_and(volatile unsigned/**/short *addr, unsigned/**/short value)
+{
+  (void)__atomic_and_fetch(addr, value, __ATOMIC_RELAXED);
+}
+#define AO_HAVE_short_and
+
+AO_INLINE void
+AO_short_and_acquire(volatile unsigned/**/short *addr, unsigned/**/short value)
+{
+  (void)__atomic_and_fetch(addr, value, __ATOMIC_ACQUIRE);
+}
+#define AO_HAVE_short_and_acquire
+
+AO_INLINE void
+AO_short_and_release(volatile unsigned/**/short *addr, unsigned/**/short value)
+{
+  (void)__atomic_and_fetch(addr, value, __ATOMIC_RELEASE);
+}
+#define AO_HAVE_short_and_release
+
+AO_INLINE void
+AO_short_and_full(volatile unsigned/**/short *addr, unsigned/**/short value)
+{
+  (void)__atomic_and_fetch(addr, value, __ATOMIC_SEQ_CST);
+}
+#define AO_HAVE_short_and_full
+
+AO_INLINE void
+AO_short_or(volatile unsigned/**/short *addr, unsigned/**/short value)
+{
+  (void)__atomic_or_fetch(addr, value, __ATOMIC_RELAXED);
+}
+#define AO_HAVE_short_or
+
+AO_INLINE void
+AO_short_or_acquire(volatile unsigned/**/short *addr, unsigned/**/short value)
+{
+  (void)__atomic_or_fetch(addr, value, __ATOMIC_ACQUIRE);
+}
+#define AO_HAVE_short_or_acquire
+
+AO_INLINE void
+AO_short_or_release(volatile unsigned/**/short *addr, unsigned/**/short value)
+{
+  (void)__atomic_or_fetch(addr, value, __ATOMIC_RELEASE);
+}
+#define AO_HAVE_short_or_release
+
+AO_INLINE void
+AO_short_or_full(volatile unsigned/**/short *addr, unsigned/**/short value)
+{
+  (void)__atomic_or_fetch(addr, value, __ATOMIC_SEQ_CST);
+}
+#define AO_HAVE_short_or_full
+
+AO_INLINE void
+AO_short_xor(volatile unsigned/**/short *addr, unsigned/**/short value)
+{
+  (void)__atomic_xor_fetch(addr, value, __ATOMIC_RELAXED);
+}
+#define AO_HAVE_short_xor
+
+AO_INLINE void
+AO_short_xor_acquire(volatile unsigned/**/short *addr, unsigned/**/short value)
+{
+  (void)__atomic_xor_fetch(addr, value, __ATOMIC_ACQUIRE);
+}
+#define AO_HAVE_short_xor_acquire
+
+AO_INLINE void
+AO_short_xor_release(volatile unsigned/**/short *addr, unsigned/**/short value)
+{
+  (void)__atomic_xor_fetch(addr, value, __ATOMIC_RELEASE);
+}
+#define AO_HAVE_short_xor_release
+
+AO_INLINE void
+AO_short_xor_full(volatile unsigned/**/short *addr, unsigned/**/short value)
+{
+  (void)__atomic_xor_fetch(addr, value, __ATOMIC_SEQ_CST);
+}
+#define AO_HAVE_short_xor_full
+/*
+ * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
+ * Copyright (c) 1996-1999 by Silicon Graphics.  All rights reserved.
+ * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P.
+ *
+ *
+ * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
+ * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
+ *
+ * Permission is hereby granted to use or copy this program
+ * for any purpose, provided the above notices are retained on all copies.
+ * Permission to modify the code and to distribute modified code is granted,
+ * provided the above notices are retained, and a notice that the code was
+ * modified is included with the above copyright notice.
+ *
+ */
+
+AO_INLINE unsigned
+AO_int_fetch_and_add(volatile unsigned *addr, unsigned incr)
+{
+  return __atomic_fetch_add(addr, incr, __ATOMIC_RELAXED);
+}
+#define AO_HAVE_int_fetch_and_add
+
+AO_INLINE unsigned
+AO_int_fetch_and_add_acquire(volatile unsigned *addr, unsigned incr)
+{
+  return __atomic_fetch_add(addr, incr, __ATOMIC_ACQUIRE);
+}
+#define AO_HAVE_int_fetch_and_add_acquire
+
+AO_INLINE unsigned
+AO_int_fetch_and_add_release(volatile unsigned *addr, unsigned incr)
+{
+  return __atomic_fetch_add(addr, incr, __ATOMIC_RELEASE);
+}
+#define AO_HAVE_int_fetch_and_add_release
+
+AO_INLINE unsigned
+AO_int_fetch_and_add_full(volatile unsigned *addr, unsigned incr)
+{
+  return __atomic_fetch_add(addr, incr, __ATOMIC_SEQ_CST);
+}
+#define AO_HAVE_int_fetch_and_add_full
+
+AO_INLINE void
+AO_int_and(volatile unsigned *addr, unsigned value)
+{
+  (void)__atomic_and_fetch(addr, value, __ATOMIC_RELAXED);
+}
+#define AO_HAVE_int_and
+
+AO_INLINE void
+AO_int_and_acquire(volatile unsigned *addr, unsigned value)
+{
+  (void)__atomic_and_fetch(addr, value, __ATOMIC_ACQUIRE);
+}
+#define AO_HAVE_int_and_acquire
+
+AO_INLINE void
+AO_int_and_release(volatile unsigned *addr, unsigned value)
+{
+  (void)__atomic_and_fetch(addr, value, __ATOMIC_RELEASE);
+}
+#define AO_HAVE_int_and_release
+
+AO_INLINE void
+AO_int_and_full(volatile unsigned *addr, unsigned value)
+{
+  (void)__atomic_and_fetch(addr, value, __ATOMIC_SEQ_CST);
+}
+#define AO_HAVE_int_and_full
+
+AO_INLINE void
+AO_int_or(volatile unsigned *addr, unsigned value)
+{
+  (void)__atomic_or_fetch(addr, value, __ATOMIC_RELAXED);
+}
+#define AO_HAVE_int_or
+
+AO_INLINE void
+AO_int_or_acquire(volatile unsigned *addr, unsigned value)
+{
+  (void)__atomic_or_fetch(addr, value, __ATOMIC_ACQUIRE);
+}
+#define AO_HAVE_int_or_acquire
+
+AO_INLINE void
+AO_int_or_release(volatile unsigned *addr, unsigned value)
+{
+  (void)__atomic_or_fetch(addr, value, __ATOMIC_RELEASE);
+}
+#define AO_HAVE_int_or_release
+
+AO_INLINE void
+AO_int_or_full(volatile unsigned *addr, unsigned value)
+{
+  (void)__atomic_or_fetch(addr, value, __ATOMIC_SEQ_CST);
+}
+#define AO_HAVE_int_or_full
+
+AO_INLINE void
+AO_int_xor(volatile unsigned *addr, unsigned value)
+{
+  (void)__atomic_xor_fetch(addr, value, __ATOMIC_RELAXED);
+}
+#define AO_HAVE_int_xor
+
+AO_INLINE void
+AO_int_xor_acquire(volatile unsigned *addr, unsigned value)
+{
+  (void)__atomic_xor_fetch(addr, value, __ATOMIC_ACQUIRE);
+}
+#define AO_HAVE_int_xor_acquire
+
+AO_INLINE void
+AO_int_xor_release(volatile unsigned *addr, unsigned value)
+{
+  (void)__atomic_xor_fetch(addr, value, __ATOMIC_RELEASE);
+}
+#define AO_HAVE_int_xor_release
+
+AO_INLINE void
+AO_int_xor_full(volatile unsigned *addr, unsigned value)
+{
+  (void)__atomic_xor_fetch(addr, value, __ATOMIC_SEQ_CST);
+}
+#define AO_HAVE_int_xor_full
+/*
+ * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
+ * Copyright (c) 1996-1999 by Silicon Graphics.  All rights reserved.
+ * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P.
+ *
+ *
+ * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
+ * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
+ *
+ * Permission is hereby granted to use or copy this program
+ * for any purpose, provided the above notices are retained on all copies.
+ * Permission to modify the code and to distribute modified code is granted,
+ * provided the above notices are retained, and a notice that the code was
+ * modified is included with the above copyright notice.
+ *
+ */
+
 AO_INLINE AO_t
 AO_fetch_and_add(volatile AO_t *addr, AO_t incr)
 {
@@ -126,5 +510,3 @@ AO_xor_full(volatile AO_t *addr, AO_t value)
   (void)__atomic_xor_fetch(addr, value, __ATOMIC_SEQ_CST);
 }
 #define AO_HAVE_xor_full
-
-/* TODO: Add AO_char/short/int_ primitives (via template header). */
diff --git a/src/atomic_ops/sysdeps/gcc/generic-arithm.template b/src/atomic_ops/sysdeps/gcc/generic-arithm.template
new file mode 100644
index 0000000..fb7b818
--- /dev/null
+++ b/src/atomic_ops/sysdeps/gcc/generic-arithm.template
@@ -0,0 +1,128 @@
+/*
+ * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
+ * Copyright (c) 1996-1999 by Silicon Graphics.  All rights reserved.
+ * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P.
+ *
+ *
+ * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
+ * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
+ *
+ * Permission is hereby granted to use or copy this program
+ * for any purpose, provided the above notices are retained on all copies.
+ * Permission to modify the code and to distribute modified code is granted,
+ * provided the above notices are retained, and a notice that the code was
+ * modified is included with the above copyright notice.
+ *
+ */
+
+AO_INLINE XCTYPE
+AO_XSIZE_fetch_and_add(volatile XCTYPE *addr, XCTYPE incr)
+{
+  return __atomic_fetch_add(addr, incr, __ATOMIC_RELAXED);
+}
+#define AO_HAVE_XSIZE_fetch_and_add
+
+AO_INLINE XCTYPE
+AO_XSIZE_fetch_and_add_acquire(volatile XCTYPE *addr, XCTYPE incr)
+{
+  return __atomic_fetch_add(addr, incr, __ATOMIC_ACQUIRE);
+}
+#define AO_HAVE_XSIZE_fetch_and_add_acquire
+
+AO_INLINE XCTYPE
+AO_XSIZE_fetch_and_add_release(volatile XCTYPE *addr, XCTYPE incr)
+{
+  return __atomic_fetch_add(addr, incr, __ATOMIC_RELEASE);
+}
+#define AO_HAVE_XSIZE_fetch_and_add_release
+
+AO_INLINE XCTYPE
+AO_XSIZE_fetch_and_add_full(volatile XCTYPE *addr, XCTYPE incr)
+{
+  return __atomic_fetch_add(addr, incr, __ATOMIC_SEQ_CST);
+}
+#define AO_HAVE_XSIZE_fetch_and_add_full
+
+AO_INLINE void
+AO_XSIZE_and(volatile XCTYPE *addr, XCTYPE value)
+{
+  (void)__atomic_and_fetch(addr, value, __ATOMIC_RELAXED);
+}
+#define AO_HAVE_XSIZE_and
+
+AO_INLINE void
+AO_XSIZE_and_acquire(volatile XCTYPE *addr, XCTYPE value)
+{
+  (void)__atomic_and_fetch(addr, value, __ATOMIC_ACQUIRE);
+}
+#define AO_HAVE_XSIZE_and_acquire
+
+AO_INLINE void
+AO_XSIZE_and_release(volatile XCTYPE *addr, XCTYPE value)
+{
+  (void)__atomic_and_fetch(addr, value, __ATOMIC_RELEASE);
+}
+#define AO_HAVE_XSIZE_and_release
+
+AO_INLINE void
+AO_XSIZE_and_full(volatile XCTYPE *addr, XCTYPE value)
+{
+  (void)__atomic_and_fetch(addr, value, __ATOMIC_SEQ_CST);
+}
+#define AO_HAVE_XSIZE_and_full
+
+AO_INLINE void
+AO_XSIZE_or(volatile XCTYPE *addr, XCTYPE value)
+{
+  (void)__atomic_or_fetch(addr, value, __ATOMIC_RELAXED);
+}
+#define AO_HAVE_XSIZE_or
+
+AO_INLINE void
+AO_XSIZE_or_acquire(volatile XCTYPE *addr, XCTYPE value)
+{
+  (void)__atomic_or_fetch(addr, value, __ATOMIC_ACQUIRE);
+}
+#define AO_HAVE_XSIZE_or_acquire
+
+AO_INLINE void
+AO_XSIZE_or_release(volatile XCTYPE *addr, XCTYPE value)
+{
+  (void)__atomic_or_fetch(addr, value, __ATOMIC_RELEASE);
+}
+#define AO_HAVE_XSIZE_or_release
+
+AO_INLINE void
+AO_XSIZE_or_full(volatile XCTYPE *addr, XCTYPE value)
+{
+  (void)__atomic_or_fetch(addr, value, __ATOMIC_SEQ_CST);
+}
+#define AO_HAVE_XSIZE_or_full
+
+AO_INLINE void
+AO_XSIZE_xor(volatile XCTYPE *addr, XCTYPE value)
+{
+  (void)__atomic_xor_fetch(addr, value, __ATOMIC_RELAXED);
+}
+#define AO_HAVE_XSIZE_xor
+
+AO_INLINE void
+AO_XSIZE_xor_acquire(volatile XCTYPE *addr, XCTYPE value)
+{
+  (void)__atomic_xor_fetch(addr, value, __ATOMIC_ACQUIRE);
+}
+#define AO_HAVE_XSIZE_xor_acquire
+
+AO_INLINE void
+AO_XSIZE_xor_release(volatile XCTYPE *addr, XCTYPE value)
+{
+  (void)__atomic_xor_fetch(addr, value, __ATOMIC_RELEASE);
+}
+#define AO_HAVE_XSIZE_xor_release
+
+AO_INLINE void
+AO_XSIZE_xor_full(volatile XCTYPE *addr, XCTYPE value)
+{
+  (void)__atomic_xor_fetch(addr, value, __ATOMIC_SEQ_CST);
+}
+#define AO_HAVE_XSIZE_xor_full
diff --git a/src/atomic_ops/sysdeps/gcc/generic-small.h b/src/atomic_ops/sysdeps/gcc/generic-small.h
index a3e375c..72f4a5b 100644
--- a/src/atomic_ops/sysdeps/gcc/generic-small.h
+++ b/src/atomic_ops/sysdeps/gcc/generic-small.h
@@ -15,6 +15,216 @@
  *
  */
 
+AO_INLINE unsigned/**/char
+AO_char_load(const volatile unsigned/**/char *addr)
+{
+  return __atomic_load_n(addr, __ATOMIC_RELAXED);
+}
+#define AO_HAVE_char_load
+
+AO_INLINE unsigned/**/char
+AO_char_load_acquire(const volatile unsigned/**/char *addr)
+{
+  return __atomic_load_n(addr, __ATOMIC_ACQUIRE);
+}
+#define AO_HAVE_char_load_acquire
+
+/* char_load_full is generalized using load and nop_full, so that      */
+/* char_load_read is defined using load and nop_read.                  */
+/* char_store_full definition is omitted similar to load_full reason.  */
+
+AO_INLINE void
+AO_char_store(volatile unsigned/**/char *addr, unsigned/**/char value)
+{
+  __atomic_store_n(addr, value, __ATOMIC_RELAXED);
+}
+#define AO_HAVE_char_store
+
+AO_INLINE void
+AO_char_store_release(volatile unsigned/**/char *addr, unsigned/**/char value)
+{
+  __atomic_store_n(addr, value, __ATOMIC_RELEASE);
+}
+#define AO_HAVE_char_store_release
+
+AO_INLINE unsigned/**/char
+AO_char_fetch_compare_and_swap(volatile unsigned/**/char *addr,
+                                unsigned/**/char old_val, unsigned/**/char new_val)
+{
+  return __sync_val_compare_and_swap(addr, old_val, new_val
+                                     /* empty protection list */);
+}
+#define AO_HAVE_char_fetch_compare_and_swap
+
+/* TODO: Add CAS _acquire/release/full primitives. */
+
+#ifndef AO_GENERALIZE_ASM_BOOL_CAS
+  AO_INLINE int
+  AO_char_compare_and_swap(volatile unsigned/**/char *addr,
+                            unsigned/**/char old_val, unsigned/**/char new_val)
+  {
+    return __sync_bool_compare_and_swap(addr, old_val, new_val
+                                        /* empty protection list */);
+  }
+# define AO_HAVE_char_compare_and_swap
+#endif /* !AO_GENERALIZE_ASM_BOOL_CAS */
+/*
+ * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
+ * Copyright (c) 1996-1999 by Silicon Graphics.  All rights reserved.
+ * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P.
+ *
+ *
+ * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
+ * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
+ *
+ * Permission is hereby granted to use or copy this program
+ * for any purpose, provided the above notices are retained on all copies.
+ * Permission to modify the code and to distribute modified code is granted,
+ * provided the above notices are retained, and a notice that the code was
+ * modified is included with the above copyright notice.
+ *
+ */
+
+AO_INLINE unsigned/**/short
+AO_short_load(const volatile unsigned/**/short *addr)
+{
+  return __atomic_load_n(addr, __ATOMIC_RELAXED);
+}
+#define AO_HAVE_short_load
+
+AO_INLINE unsigned/**/short
+AO_short_load_acquire(const volatile unsigned/**/short *addr)
+{
+  return __atomic_load_n(addr, __ATOMIC_ACQUIRE);
+}
+#define AO_HAVE_short_load_acquire
+
+/* short_load_full is generalized using load and nop_full, so that      */
+/* short_load_read is defined using load and nop_read.                  */
+/* short_store_full definition is omitted similar to load_full reason.  */
+
+AO_INLINE void
+AO_short_store(volatile unsigned/**/short *addr, unsigned/**/short value)
+{
+  __atomic_store_n(addr, value, __ATOMIC_RELAXED);
+}
+#define AO_HAVE_short_store
+
+AO_INLINE void
+AO_short_store_release(volatile unsigned/**/short *addr, unsigned/**/short value)
+{
+  __atomic_store_n(addr, value, __ATOMIC_RELEASE);
+}
+#define AO_HAVE_short_store_release
+
+AO_INLINE unsigned/**/short
+AO_short_fetch_compare_and_swap(volatile unsigned/**/short *addr,
+                                unsigned/**/short old_val, unsigned/**/short new_val)
+{
+  return __sync_val_compare_and_swap(addr, old_val, new_val
+                                     /* empty protection list */);
+}
+#define AO_HAVE_short_fetch_compare_and_swap
+
+/* TODO: Add CAS _acquire/release/full primitives. */
+
+#ifndef AO_GENERALIZE_ASM_BOOL_CAS
+  AO_INLINE int
+  AO_short_compare_and_swap(volatile unsigned/**/short *addr,
+                            unsigned/**/short old_val, unsigned/**/short new_val)
+  {
+    return __sync_bool_compare_and_swap(addr, old_val, new_val
+                                        /* empty protection list */);
+  }
+# define AO_HAVE_short_compare_and_swap
+#endif /* !AO_GENERALIZE_ASM_BOOL_CAS */
+/*
+ * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
+ * Copyright (c) 1996-1999 by Silicon Graphics.  All rights reserved.
+ * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P.
+ *
+ *
+ * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
+ * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
+ *
+ * Permission is hereby granted to use or copy this program
+ * for any purpose, provided the above notices are retained on all copies.
+ * Permission to modify the code and to distribute modified code is granted,
+ * provided the above notices are retained, and a notice that the code was
+ * modified is included with the above copyright notice.
+ *
+ */
+
+AO_INLINE unsigned
+AO_int_load(const volatile unsigned *addr)
+{
+  return __atomic_load_n(addr, __ATOMIC_RELAXED);
+}
+#define AO_HAVE_int_load
+
+AO_INLINE unsigned
+AO_int_load_acquire(const volatile unsigned *addr)
+{
+  return __atomic_load_n(addr, __ATOMIC_ACQUIRE);
+}
+#define AO_HAVE_int_load_acquire
+
+/* int_load_full is generalized using load and nop_full, so that      */
+/* int_load_read is defined using load and nop_read.                  */
+/* int_store_full definition is omitted similar to load_full reason.  */
+
+AO_INLINE void
+AO_int_store(volatile unsigned *addr, unsigned value)
+{
+  __atomic_store_n(addr, value, __ATOMIC_RELAXED);
+}
+#define AO_HAVE_int_store
+
+AO_INLINE void
+AO_int_store_release(volatile unsigned *addr, unsigned value)
+{
+  __atomic_store_n(addr, value, __ATOMIC_RELEASE);
+}
+#define AO_HAVE_int_store_release
+
+AO_INLINE unsigned
+AO_int_fetch_compare_and_swap(volatile unsigned *addr,
+                                unsigned old_val, unsigned new_val)
+{
+  return __sync_val_compare_and_swap(addr, old_val, new_val
+                                     /* empty protection list */);
+}
+#define AO_HAVE_int_fetch_compare_and_swap
+
+/* TODO: Add CAS _acquire/release/full primitives. */
+
+#ifndef AO_GENERALIZE_ASM_BOOL_CAS
+  AO_INLINE int
+  AO_int_compare_and_swap(volatile unsigned *addr,
+                            unsigned old_val, unsigned new_val)
+  {
+    return __sync_bool_compare_and_swap(addr, old_val, new_val
+                                        /* empty protection list */);
+  }
+# define AO_HAVE_int_compare_and_swap
+#endif /* !AO_GENERALIZE_ASM_BOOL_CAS */
+/*
+ * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
+ * Copyright (c) 1996-1999 by Silicon Graphics.  All rights reserved.
+ * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P.
+ *
+ *
+ * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
+ * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
+ *
+ * Permission is hereby granted to use or copy this program
+ * for any purpose, provided the above notices are retained on all copies.
+ * Permission to modify the code and to distribute modified code is granted,
+ * provided the above notices are retained, and a notice that the code was
+ * modified is included with the above copyright notice.
+ *
+ */
+
 AO_INLINE AO_t
 AO_load(const volatile AO_t *addr)
 {
@@ -29,8 +239,9 @@ AO_load_acquire(const volatile AO_t *addr)
 }
 #define AO_HAVE_load_acquire
 
-/* AO_load_full is generalized using AO_load and AO_nop_full, so that   */
-/* AO_load_read is defined using AO_load and AO_nop_read.               */
+/* load_full is generalized using load and nop_full, so that      */
+/* load_read is defined using load and nop_read.                  */
+/* store_full definition is omitted similar to load_full reason.  */
 
 AO_INLINE void
 AO_store(volatile AO_t *addr, AO_t value)
@@ -46,10 +257,9 @@ AO_store_release(volatile AO_t *addr, AO_t value)
 }
 #define AO_HAVE_store_release
 
-/* AO_store_full definition is omitted similar to AO_load_full reason.  */
-
 AO_INLINE AO_t
-AO_fetch_compare_and_swap(volatile AO_t *addr, AO_t old_val, AO_t new_val)
+AO_fetch_compare_and_swap(volatile AO_t *addr,
+                                AO_t old_val, AO_t new_val)
 {
   return __sync_val_compare_and_swap(addr, old_val, new_val
                                      /* empty protection list */);
@@ -60,12 +270,11 @@ AO_fetch_compare_and_swap(volatile AO_t *addr, AO_t old_val, AO_t new_val)
 
 #ifndef AO_GENERALIZE_ASM_BOOL_CAS
   AO_INLINE int
-  AO_compare_and_swap(volatile AO_t *addr, AO_t old_val, AO_t new_val)
+  AO_compare_and_swap(volatile AO_t *addr,
+                            AO_t old_val, AO_t new_val)
   {
     return __sync_bool_compare_and_swap(addr, old_val, new_val
                                         /* empty protection list */);
   }
 # define AO_HAVE_compare_and_swap
 #endif /* !AO_GENERALIZE_ASM_BOOL_CAS */
-
-/* TODO: Add AO_char/short/int_ primitives (via template header). */
diff --git a/src/atomic_ops/sysdeps/gcc/generic-small.template b/src/atomic_ops/sysdeps/gcc/generic-small.template
new file mode 100644
index 0000000..9685acf
--- /dev/null
+++ b/src/atomic_ops/sysdeps/gcc/generic-small.template
@@ -0,0 +1,70 @@
+/*
+ * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
+ * Copyright (c) 1996-1999 by Silicon Graphics.  All rights reserved.
+ * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P.
+ *
+ *
+ * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
+ * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
+ *
+ * Permission is hereby granted to use or copy this program
+ * for any purpose, provided the above notices are retained on all copies.
+ * Permission to modify the code and to distribute modified code is granted,
+ * provided the above notices are retained, and a notice that the code was
+ * modified is included with the above copyright notice.
+ *
+ */
+
+AO_INLINE XCTYPE
+AO_XSIZE_load(const volatile XCTYPE *addr)
+{
+  return __atomic_load_n(addr, __ATOMIC_RELAXED);
+}
+#define AO_HAVE_XSIZE_load
+
+AO_INLINE XCTYPE
+AO_XSIZE_load_acquire(const volatile XCTYPE *addr)
+{
+  return __atomic_load_n(addr, __ATOMIC_ACQUIRE);
+}
+#define AO_HAVE_XSIZE_load_acquire
+
+/* XSIZE_load_full is generalized using load and nop_full, so that      */
+/* XSIZE_load_read is defined using load and nop_read.                  */
+/* XSIZE_store_full definition is omitted similar to load_full reason.  */
+
+AO_INLINE void
+AO_XSIZE_store(volatile XCTYPE *addr, XCTYPE value)
+{
+  __atomic_store_n(addr, value, __ATOMIC_RELAXED);
+}
+#define AO_HAVE_XSIZE_store
+
+AO_INLINE void
+AO_XSIZE_store_release(volatile XCTYPE *addr, XCTYPE value)
+{
+  __atomic_store_n(addr, value, __ATOMIC_RELEASE);
+}
+#define AO_HAVE_XSIZE_store_release
+
+AO_INLINE XCTYPE
+AO_XSIZE_fetch_compare_and_swap(volatile XCTYPE *addr,
+                                XCTYPE old_val, XCTYPE new_val)
+{
+  return __sync_val_compare_and_swap(addr, old_val, new_val
+                                     /* empty protection list */);
+}
+#define AO_HAVE_XSIZE_fetch_compare_and_swap
+
+/* TODO: Add CAS _acquire/release/full primitives. */
+
+#ifndef AO_GENERALIZE_ASM_BOOL_CAS
+  AO_INLINE int
+  AO_XSIZE_compare_and_swap(volatile XCTYPE *addr,
+                            XCTYPE old_val, XCTYPE new_val)
+  {
+    return __sync_bool_compare_and_swap(addr, old_val, new_val
+                                        /* empty protection list */);
+  }
+# define AO_HAVE_XSIZE_compare_and_swap
+#endif /* !AO_GENERALIZE_ASM_BOOL_CAS */