ovs-atomic: Expose atomic exchange operation.

The atomic exchange operation is a useful primitive that should be
available as well.  Most compilers already expose or offer a way
to use it, but a single symbol needs to be defined.

Signed-off-by: Gaetan Rivet <grive@u256.net>
Reviewed-by: Eli Britstein <elibr@nvidia.com>
Reviewed-by: Maxime Coquelin <maxime.coquelin@redhat.com>
Signed-off-by: Ilya Maximets <i.maximets@ovn.org>

10 files changed, 71 insertions, 1 deletions

diff --git a/lib/ovs-atomic-c++.h b/lib/ovs-atomic-c++.h
index d47b8dd39..8605fa9d3 100644
--- a/lib/ovs-atomic-c++.h
+++ b/lib/ovs-atomic-c++.h
@@ -29,6 +29,9 @@ using std::atomic_compare_exchange_strong_explicit;
 using std::atomic_compare_exchange_weak;
 using std::atomic_compare_exchange_weak_explicit;
 
+using std::atomic_exchange;
+using std::atomic_exchange_explicit;
+
 #define atomic_read(SRC, DST) \
     atomic_read_explicit(SRC, DST, memory_order_seq_cst)
 #define atomic_read_explicit(SRC, DST, ORDER)   \
diff --git a/lib/ovs-atomic-clang.h b/lib/ovs-atomic-clang.h
index 34cc2faa7..cdf02a512 100644
--- a/lib/ovs-atomic-clang.h
+++ b/lib/ovs-atomic-clang.h
@@ -67,6 +67,11 @@ typedef enum {
 #define atomic_compare_exchange_weak_explicit(DST, EXP, SRC, ORD1, ORD2) \
     __c11_atomic_compare_exchange_weak(DST, EXP, SRC, ORD1, ORD2)
 
+#define atomic_exchange(RMW, ARG) \
+    atomic_exchange_explicit(RMW, ARG, memory_order_seq_cst)
+#define atomic_exchange_explicit(RMW, ARG, ORDER) \
+    __c11_atomic_exchange(RMW, ARG, ORDER)
+
 #define atomic_add(RMW, ARG, ORIG) \
     atomic_add_explicit(RMW, ARG, ORIG, memory_order_seq_cst)
 #define atomic_sub(RMW, ARG, ORIG) \
diff --git a/lib/ovs-atomic-gcc4+.h b/lib/ovs-atomic-gcc4+.h
index 25bcf20a0..f9accde1a 100644
--- a/lib/ovs-atomic-gcc4+.h
+++ b/lib/ovs-atomic-gcc4+.h
@@ -128,6 +128,11 @@ atomic_signal_fence(memory_order order)
 #define atomic_compare_exchange_weak_explicit   \
     atomic_compare_exchange_strong_explicit
 
+#define atomic_exchange_explicit(DST, SRC, ORDER) \
+    __sync_lock_test_and_set(DST, SRC)
+#define atomic_exchange(DST, SRC) \
+    atomic_exchange_explicit(DST, SRC, memory_order_seq_cst)
+
 #define atomic_op__(RMW, OP, ARG, ORIG)                     \
     ({                                                      \
         typeof(RMW) rmw__ = (RMW);                          \
diff --git a/lib/ovs-atomic-gcc4.7+.h b/lib/ovs-atomic-gcc4.7+.h
index 4c197ebe0..846e05775 100644
--- a/lib/ovs-atomic-gcc4.7+.h
+++ b/lib/ovs-atomic-gcc4.7+.h
@@ -61,6 +61,11 @@ typedef enum {
 #define atomic_compare_exchange_weak_explicit(DST, EXP, SRC, ORD1, ORD2) \
     __atomic_compare_exchange_n(DST, EXP, SRC, true, ORD1, ORD2)
 
+#define atomic_exchange_explicit(DST, SRC, ORDER) \
+    __atomic_exchange_n(DST, SRC, ORDER)
+#define atomic_exchange(DST, SRC) \
+    atomic_exchange_explicit(DST, SRC, memory_order_seq_cst)
+
 #define atomic_add(RMW, OPERAND, ORIG) \
         atomic_add_explicit(RMW, OPERAND, ORIG, memory_order_seq_cst)
 #define atomic_sub(RMW, OPERAND, ORIG) \
diff --git a/lib/ovs-atomic-i586.h b/lib/ovs-atomic-i586.h
index 9a385ce84..35a0959ff 100644
--- a/lib/ovs-atomic-i586.h
+++ b/lib/ovs-atomic-i586.h
@@ -400,6 +400,11 @@ atomic_signal_fence(memory_order order)
 #define atomic_compare_exchange_weak_explicit   \
     atomic_compare_exchange_strong_explicit
 
+#define atomic_exchange_explicit(RMW, ARG, ORDER) \
+    atomic_exchange__(RMW, ARG, ORDER)
+#define atomic_exchange(RMW, ARG) \
+    atomic_exchange_explicit(RMW, ARG, memory_order_seq_cst)
+
 #define atomic_add__(RMW, ARG, CLOB)            \
     asm volatile("lock; xadd %0,%1 ; "          \
                  "# atomic_add__     "          \
diff --git a/lib/ovs-atomic-locked.h b/lib/ovs-atomic-locked.h
index f8f0ba2a5..bf38c4a43 100644
--- a/lib/ovs-atomic-locked.h
+++ b/lib/ovs-atomic-locked.h
@@ -31,6 +31,15 @@ void atomic_unlock__(void *);
          atomic_unlock__(DST),                          \
          false)))
 
+#define atomic_exchange_locked(DST, SRC)     \
+    ({                                       \
+        atomic_lock__(DST);                  \
+        typeof(*(DST)) __tmp = *(DST);       \
+        *(DST) = SRC;                        \
+        atomic_unlock__(DST);                \
+        __tmp;                               \
+    })
+
 #define atomic_op_locked_add +=
 #define atomic_op_locked_sub -=
 #define atomic_op_locked_or  |=
diff --git a/lib/ovs-atomic-msvc.h b/lib/ovs-atomic-msvc.h
index 9def887d3..fb8cd03bd 100644
--- a/lib/ovs-atomic-msvc.h
+++ b/lib/ovs-atomic-msvc.h
@@ -345,6 +345,28 @@ atomic_signal_fence(memory_order order)
 #define atomic_compare_exchange_weak_explicit \
         atomic_compare_exchange_strong_explicit
 
+/* While intrinsics offering different memory ordering
+ * are available in MSVC C compiler, they are not defined
+ * in the C++ compiler. Ignore for compatibility.
+ *
+ * Use nested ternary operators as the GNU extension ({})
+ * is not available.
+ */
+
+#define atomic_exchange_explicit(DST, SRC, ORDER) \
+    ((sizeof *(DST) == 1) ? \
+        _InterlockedExchange8((char volatile *) DST, SRC) \
+    : (sizeof *(DST) == 2) ? \
+        _InterlockedExchange16((short volatile *) DST, SRC) \
+    : (sizeof *(DST) == 4) ? \
+        _InterlockedExchange((long int volatile *) DST, SRC) \
+    : (sizeof *(DST) == 8) ? \
+        _InterlockedExchange64((__int64 volatile *) DST, SRC) \
+    : (abort(), 0))
+
+#define atomic_exchange(DST, SRC) \
+        atomic_exchange_explicit(DST, SRC, memory_order_seq_cst)
+
 /* MSVCs c++ compiler implements c11 atomics and looking through its
  * implementation (in xatomic.h), orders are ignored for x86 platform.
  * Do the same here. */
diff --git a/lib/ovs-atomic-pthreads.h b/lib/ovs-atomic-pthreads.h
index 12234e79e..570a67fe4 100644
--- a/lib/ovs-atomic-pthreads.h
+++ b/lib/ovs-atomic-pthreads.h
@@ -77,6 +77,11 @@ atomic_signal_fence(memory_order order OVS_UNUSED)
 #define atomic_compare_exchange_weak_explicit   \
     atomic_compare_exchange_strong_explicit
 
+#define atomic_exchange(DST, SRC) \
+    atomic_exchange_locked(DST, SRC)
+#define atomic_exchange_explicit(DST, SRC, ORDER) \
+    ((void) (ORDER), atomic_exchange(DST, SRC))
+
 #define atomic_add(RMW, ARG, ORIG) atomic_op_locked(RMW, add, ARG, ORIG)
 #define atomic_sub(RMW, ARG, ORIG) atomic_op_locked(RMW, sub, ARG, ORIG)
 #define atomic_or( RMW, ARG, ORIG) atomic_op_locked(RMW, or, ARG, ORIG)
diff --git a/lib/ovs-atomic-x86_64.h b/lib/ovs-atomic-x86_64.h
index 1e7d42707..3bdaf2f08 100644
--- a/lib/ovs-atomic-x86_64.h
+++ b/lib/ovs-atomic-x86_64.h
@@ -274,6 +274,11 @@ atomic_signal_fence(memory_order order)
 #define atomic_compare_exchange_weak_explicit   \
     atomic_compare_exchange_strong_explicit
 
+#define atomic_exchange_explicit(RMW, ARG, ORDER) \
+    atomic_exchange__(RMW, ARG, ORDER)
+#define atomic_exchange(RMW, ARG) \
+    atomic_exchange_explicit(RMW, ARG, memory_order_seq_cst)
+
 #define atomic_add__(RMW, ARG, CLOB)            \
     asm volatile("lock; xadd %0,%1 ; "          \
                  "# atomic_add__     "          \
diff --git a/lib/ovs-atomic.h b/lib/ovs-atomic.h
index 11fa19268..8fdce0cf8 100644
--- a/lib/ovs-atomic.h
+++ b/lib/ovs-atomic.h
@@ -210,7 +210,7 @@
  * In this section, A is an atomic type and C is the corresponding non-atomic
  * type.
  *
- * The "store" and "compare_exchange" primitives match C11:
+ * The "store", "exchange", and "compare_exchange" primitives match C11:
  *
  *     void atomic_store(A *object, C value);
  *     void atomic_store_explicit(A *object, C value, memory_order);
@@ -244,6 +244,12 @@
  *         efficiently, so it should be used if the application will need to
  *         loop anyway.
  *
+ *     C atomic_exchange(A *object, C desired);
+ *     C atomic_exchange_explicit(A *object, C desired, memory_order);
+ *
+ *         Atomically stores 'desired' into '*object', returning the value
+ *         previously held.
+ *
  * The following primitives differ from the C11 ones (and have different names)
  * because there does not appear to be a way to implement the standard
  * primitives in standard C: