Workaround missing built-in double-wide primitives for AArch64 in gcc-4.7

* src/atomic_ops/sysdeps/gcc/aarch64.h (AO_double_load,
AO_double_load_acquire, AO_double_store, AO_double_store_release,
AO_double_compare_and_swap, AO_double_compare_and_swap_acquire,
AO_double_compare_and_swap_release): Define using inline assembly
(because GCC 4.7-2013.01 has no support of 16-byte atomic operations).
* src/atomic_ops/sysdeps/gcc/generic.h (AO_double_load,
AO_double_load_acquire, AO_double_store, AO_double_store_release,
AO_double_compare_and_swap): Do not define if the corresponding
AO_HAVE_double_X macro is defined.

2 files changed, 199 insertions, 40 deletions

diff --git a/src/atomic_ops/sysdeps/gcc/aarch64.h b/src/atomic_ops/sysdeps/gcc/aarch64.h
index 7408e07..7c2db2f 100644
--- a/src/atomic_ops/sysdeps/gcc/aarch64.h
+++ b/src/atomic_ops/sysdeps/gcc/aarch64.h
@@ -28,4 +28,152 @@
 # define AO_HAVE_nop_write
 #endif
 
+#ifdef AO_HAVE_DOUBLE_PTR_STORAGE
+
+  AO_INLINE AO_double_t
+  AO_double_load(const volatile AO_double_t *addr)
+  {
+    AO_double_t result;
+    int status;
+
+    do {
+      __asm__ __volatile__("//AO_double_load\n"
+      "       ldxp  %0, %1, [%3]\n"
+      "       stxp %w2, %0, %1, [%3]"
+      : "=&r" (result.AO_val1), "=&r" (result.AO_val2), "=&r" (status)
+      : "r" (addr)
+      );
+    } while (status);
+    return result;
+  }
+# define AO_HAVE_double_load
+
+  AO_INLINE AO_double_t
+  AO_double_load_acquire(const volatile AO_double_t *addr)
+  {
+    AO_double_t result;
+    int status;
+
+    do {
+      __asm__ __volatile__("//AO_double_load_acquire\n"
+      "       ldaxp  %0, %1, [%3]\n"
+      "       stxp %w2, %0, %1, [%3]"
+      : "=&r" (result.AO_val1), "=&r" (result.AO_val2), "=&r" (status)
+      : "r" (addr)
+      );
+    } while (status);
+    return result;
+  }
+# define AO_HAVE_double_load_acquire
+
+  AO_INLINE void
+  AO_double_store(volatile AO_double_t *addr, AO_double_t value)
+  {
+    AO_double_t old_val;
+    int status;
+
+    do {
+      __asm__ __volatile__("//AO_double_store\n"
+      "       ldxp  %0, %1, %3\n"
+      "       stxp %w2, %4, %5, %3"
+      : "=&r" (old_val.AO_val1), "=&r" (old_val.AO_val2), "=&r" (status), "+Q" (*addr)
+      : "r" (value.AO_val1), "r" (value.AO_val2)
+      );
+    } while (status);
+  }
+# define AO_HAVE_double_store
+
+  AO_INLINE void
+  AO_double_store_release(volatile AO_double_t *addr, AO_double_t value)
+  {
+    AO_double_t old_val;
+    int status;
+
+    do {
+      __asm__ __volatile__("//AO_double_store\n"
+      "       ldxp  %0, %1, %3\n"
+      "       stlxp %w2, %4, %5, %3"
+      : "=&r" (old_val.AO_val1), "=&r" (old_val.AO_val2), "=&r" (status), "+Q" (*addr)
+      : "r" (value.AO_val1), "r" (value.AO_val2)
+      );
+    } while (status);
+  }
+# define AO_HAVE_double_store_release
+
+  AO_INLINE int
+  AO_double_compare_and_swap(volatile AO_double_t *addr,
+                             AO_double_t old_val, AO_double_t new_val)
+  {
+    AO_double_t tmp;
+    int result = 1;
+
+    do {
+      __asm__ __volatile__("//AO_double_compare_and_swap\n"
+        "       ldxp  %0, %1, [%2]\n"
+        : "=&r" (tmp.AO_val1), "=&r" (tmp.AO_val2)
+        : "r" (addr)
+        );
+      if (tmp.AO_val1 != old_val.AO_val1 || tmp.AO_val2 != old_val.AO_val2)
+        break;
+      __asm__ __volatile__(
+        "       stxp %w0, %2, %3, %1\n"
+        : "=&r" (result), "+Q" (*addr)
+        : "r" (new_val.AO_val1), "r" (new_val.AO_val2), "r" (addr)
+      );
+    } while (AO_EXPECT_FALSE(result));
+    return !result;
+  }
+# define AO_HAVE_double_compare_and_swap
+
+  AO_INLINE int
+  AO_double_compare_and_swap_acquire(volatile AO_double_t *addr,
+                             AO_double_t old_val, AO_double_t new_val)
+  {
+    AO_double_t tmp;
+    int result = 1;
+
+    do {
+      __asm__ __volatile__("//AO_double_compare_and_swap_acquire\n"
+        "       ldaxp  %0, %1, [%2]\n"
+        : "=&r" (tmp.AO_val1), "=&r" (tmp.AO_val2)
+        : "r" (addr)
+        );
+      if (tmp.AO_val1 != old_val.AO_val1 || tmp.AO_val2 != old_val.AO_val2)
+        break;
+      __asm__ __volatile__(
+        "       stxp %w0, %2, %3, %1\n"
+        : "=&r" (result), "+Q" (*addr)
+        : "r" (new_val.AO_val1), "r" (new_val.AO_val2), "r" (addr)
+      );
+    } while (AO_EXPECT_FALSE(result));
+    return !result;
+  }
+# define AO_HAVE_double_compare_and_swap_acquire
+
+  AO_INLINE int
+  AO_double_compare_and_swap_release(volatile AO_double_t *addr,
+                             AO_double_t old_val, AO_double_t new_val)
+  {
+    AO_double_t tmp;
+    int result = 1;
+
+    do {
+      __asm__ __volatile__("//AO_double_compare_and_swap_release\n"
+        "       ldxp  %0, %1, [%2]\n"
+        : "=&r" (tmp.AO_val1), "=&r" (tmp.AO_val2)
+        : "r" (addr)
+        );
+      if (tmp.AO_val1 != old_val.AO_val1 || tmp.AO_val2 != old_val.AO_val2)
+        break;
+      __asm__ __volatile__(
+        "       stlxp %w0, %2, %3, %1\n"
+        : "=&r" (result), "+Q" (*addr)
+        : "r" (new_val.AO_val1), "r" (new_val.AO_val2), "r" (addr)
+      );
+    } while (AO_EXPECT_FALSE(result));
+    return !result;
+  }
+# define AO_HAVE_double_compare_and_swap_release
+#endif
+
 #include "generic.h"
diff --git a/src/atomic_ops/sysdeps/gcc/generic.h b/src/atomic_ops/sysdeps/gcc/generic.h
index 92887f5..de79edb 100644
--- a/src/atomic_ops/sysdeps/gcc/generic.h
+++ b/src/atomic_ops/sysdeps/gcc/generic.h
@@ -92,52 +92,63 @@
 #endif /* !AO_PREFER_GENERALIZED */
 
 #ifdef AO_HAVE_DOUBLE_PTR_STORAGE
-  AO_INLINE AO_double_t
-  AO_double_load(const volatile AO_double_t *addr)
-  {
-    AO_double_t result;
 
-    result.AO_whole = __atomic_load_n(&addr->AO_whole, __ATOMIC_RELAXED);
-    return result;
-  }
-# define AO_HAVE_double_load
+# ifndef AO_HAVE_double_load
+    AO_INLINE AO_double_t
+    AO_double_load(const volatile AO_double_t *addr)
+    {
+      AO_double_t result;
 
-  AO_INLINE AO_double_t
-  AO_double_load_acquire(const volatile AO_double_t *addr)
-  {
-    AO_double_t result;
+      result.AO_whole = __atomic_load_n(&addr->AO_whole, __ATOMIC_RELAXED);
+      return result;
+    }
+#   define AO_HAVE_double_load
+# endif
 
-    result.AO_whole = __atomic_load_n(&addr->AO_whole, __ATOMIC_ACQUIRE);
-    return result;
-  }
-# define AO_HAVE_double_load_acquire
+# ifndef AO_HAVE_double_load_acquire
+    AO_INLINE AO_double_t
+    AO_double_load_acquire(const volatile AO_double_t *addr)
+    {
+      AO_double_t result;
 
-  AO_INLINE void
-  AO_double_store(volatile AO_double_t *addr, AO_double_t value)
-  {
-    __atomic_store_n(&addr->AO_whole, value.AO_whole, __ATOMIC_RELAXED);
-  }
-# define AO_HAVE_double_store
+      result.AO_whole = __atomic_load_n(&addr->AO_whole, __ATOMIC_ACQUIRE);
+      return result;
+    }
+#   define AO_HAVE_double_load_acquire
+# endif
 
-  AO_INLINE void
-  AO_double_store_release(volatile AO_double_t *addr, AO_double_t value)
-  {
-    __atomic_store_n(&addr->AO_whole, value.AO_whole, __ATOMIC_RELEASE);
-  }
-# define AO_HAVE_double_store_release
+# ifndef AO_HAVE_double_store
+    AO_INLINE void
+    AO_double_store(volatile AO_double_t *addr, AO_double_t value)
+    {
+      __atomic_store_n(&addr->AO_whole, value.AO_whole, __ATOMIC_RELAXED);
+    }
+#   define AO_HAVE_double_store
+# endif
 
-  AO_INLINE int
-  AO_double_compare_and_swap(volatile AO_double_t *addr,
-                             AO_double_t old_val, AO_double_t new_val)
-  {
-    return (int)__atomic_compare_exchange_n(&addr->AO_whole,
-                                &old_val.AO_whole /* p_expected */,
-                                new_val.AO_whole /* desired */,
-                                0 /* is_weak: false */,
-                                __ATOMIC_RELAXED /* success */,
-                                __ATOMIC_RELAXED /* failure */);
-  }
-# define AO_HAVE_double_compare_and_swap
+# ifndef AO_HAVE_double_store_release
+    AO_INLINE void
+    AO_double_store_release(volatile AO_double_t *addr, AO_double_t value)
+    {
+      __atomic_store_n(&addr->AO_whole, value.AO_whole, __ATOMIC_RELEASE);
+    }
+#   define AO_HAVE_double_store_release
+# endif
+
+# ifndef AO_HAVE_double_compare_and_swap
+    AO_INLINE int
+    AO_double_compare_and_swap(volatile AO_double_t *addr,
+                               AO_double_t old_val, AO_double_t new_val)
+    {
+      return (int)__atomic_compare_exchange_n(&addr->AO_whole,
+                                  &old_val.AO_whole /* p_expected */,
+                                  new_val.AO_whole /* desired */,
+                                  0 /* is_weak: false */,
+                                  __ATOMIC_RELAXED /* success */,
+                                  __ATOMIC_RELAXED /* failure */);
+    }
+#   define AO_HAVE_double_compare_and_swap
+# endif
 
   /* TODO: Add double CAS _acquire/release/full primitives. */
 #endif /* AO_HAVE_DOUBLE_PTR_STORAGE */