Merge remote-tracking branch 'lightening/master'

6 files changed, 286 insertions, 12 deletions

diff --git a/libguile/lightening/lightening.h b/libguile/lightening/lightening.h
index 32e3e3acc..d9e8e1f79 100644
--- a/libguile/lightening/lightening.h
+++ b/libguile/lightening/lightening.h
@@ -477,6 +477,11 @@ jit_load_args_3(jit_state_t *_jit, jit_operand_t a, jit_operand_t b,
           M(_FGo_, ldxi_f)		\
           M(_FGG_, ldxr_d)		\
           M(_FGo_, ldxi_d)		\
+					\
+          M(_GG__, ldr_atomic)		\
+          M(_GG__, str_atomic)		\
+          M(_GGG_, swap_atomic)		\
+          M(_GGGG, cas_atomic)		\
   					\
           M(_GG__, str_c)		\
           M(_pG__, sti_c)		\
diff --git a/libguile/lightening/lightening/aarch64-cpu.c b/libguile/lightening/lightening/aarch64-cpu.c
index d24262012..39ca06a34 100644
--- a/libguile/lightening/lightening/aarch64-cpu.c
+++ b/libguile/lightening/lightening/aarch64-cpu.c
@@ -221,6 +221,10 @@ oxxrs(jit_state_t *_jit, int32_t Op,
 #define A64_MUL                       0x1b007c00
 #define A64_SMULH                     0x9b407c00
 #define A64_UMULH                     0x9bc07c00
+#define A64_LDAR                      0xc8dffc00
+#define A64_STLR                      0xc89ffc00
+#define A64_LDAXR                     0xc85ffc00
+#define A64_STLXR                     0xc800fc00
 #define A64_STRBI                     0x39000000
 #define A64_LDRBI                     0x39400000
 #define A64_LDRSBI                    0x39800000
@@ -647,6 +651,30 @@ REV(jit_state_t *_jit, int32_t Rd, int32_t Rn)
 }
 
 static void
+LDAR(jit_state_t *_jit, int32_t Rt, int32_t Rn) 
+{
+  return o_xx(_jit, A64_LDAR, Rt, Rn);
+}
+
+static void
+STLR(jit_state_t *_jit, int32_t Rt, int32_t Rn) 
+{
+  return o_xx(_jit, A64_STLR, Rt, Rn);
+}
+
+static void
+LDAXR(jit_state_t *_jit, int32_t Rt, int32_t Rn) 
+{
+  return o_xx(_jit, A64_LDAXR, Rt, Rn);
+}
+
+static void
+STLXR(jit_state_t *_jit, int32_t Rt, int32_t Rn, int32_t Rm)
+{
+  return oxxx(_jit, A64_STLXR, Rt, Rn, Rm);
+}
+
+static void
 LDRSB(jit_state_t *_jit, int32_t Rt, int32_t Rn, int32_t Rm) 
 {
   return oxxx(_jit, A64_LDRSB,Rt,Rn,Rm);
@@ -2465,3 +2493,50 @@ patch_jmp_without_veneer(jit_state_t *_jit, uint32_t *loc)
 {
   patch_jmp_offset(loc, _jit->pc.ui - loc);
 }
+
+static void
+ldr_atomic(jit_state_t *_jit, int32_t dst, int32_t loc)
+{
+  LDAR(_jit, dst, loc);
+}
+
+static void
+str_atomic(jit_state_t *_jit, int32_t loc, int32_t val)
+{
+  STLR(_jit, val, loc);
+}
+
+static void
+swap_atomic(jit_state_t *_jit, int32_t dst, int32_t loc, int32_t val)
+{
+  void *retry = jit_address(_jit);
+  int32_t result = jit_gpr_regno(get_temp_gpr(_jit));
+  int32_t val_or_tmp = dst == val ? jit_gpr_regno(get_temp_gpr(_jit)) : val;
+  movr(_jit, val_or_tmp, val);
+  LDAXR(_jit, dst, loc);
+  STLXR(_jit, val_or_tmp, loc, result);
+  jit_patch_there(_jit, bnei(_jit, result, 0), retry);
+  if (dst == val) unget_temp_gpr(_jit);
+  unget_temp_gpr(_jit);
+}
+
+static void
+cas_atomic(jit_state_t *_jit, int32_t dst, int32_t loc, int32_t expected,
+           int32_t desired)
+{
+  int32_t dst_or_tmp;
+  if (dst == loc || dst == expected || dst == expected)
+    dst_or_tmp = jit_gpr_regno(get_temp_gpr(_jit));
+  else
+    dst_or_tmp = dst;
+  void *retry = jit_address(_jit);
+  LDAXR(_jit, dst_or_tmp, loc);
+  jit_reloc_t bad = bner(_jit, dst_or_tmp, expected);
+  int result = jit_gpr_regno(get_temp_gpr(_jit));
+  STLXR(_jit, desired, loc, result);
+  jit_patch_there(_jit, bnei(_jit, result, 0), retry);
+  unget_temp_gpr(_jit);
+  jit_patch_here(_jit, bad);
+  movr(_jit, dst, dst_or_tmp);
+  unget_temp_gpr(_jit);
+}
diff --git a/libguile/lightening/lightening/arm-cpu.c b/libguile/lightening/lightening/arm-cpu.c
index 6d91f2f87..72f5d282e 100644
--- a/libguile/lightening/lightening/arm-cpu.c
+++ b/libguile/lightening/lightening/arm-cpu.c
@@ -179,6 +179,9 @@
 #define THUMB2_STRI                   0xf8400c00
 #define THUMB2_LDM_W                  0x00200000
 #define THUMB2_PUSH                   0xe92d0000
+#define THUMB_DMB                     0xf3bf8f50
+#define THUMB_LDREX                   0xe8500f00
+#define THUMB_STREX                   0xe8400000
 
 #define _NOREG (jit_gpr_regno(_PC))
 
@@ -1030,6 +1033,26 @@ T2_BLXI(jit_state_t *_jit)
   return tb(_jit, THUMB2_BLXI);
 }
 
+enum dmb_option { DMB_ISH = 0xb };
+static void
+T1_DMB(jit_state_t *_jit, enum dmb_option option)
+{
+  emit_wide_thumb(_jit, THUMB_DMB|_u4(option));
+}
+
+static void
+T1_LDREX(jit_state_t *_jit, int32_t rt, int32_t rn, int8_t offset)
+{
+  emit_wide_thumb(_jit, THUMB_LDREX|(_u4(rn)<<16)|(_u4(rt)<<12)|_u8(offset));
+}
+
+static void
+T1_STREX(jit_state_t *_jit, int32_t rd, int32_t rt, int32_t rn, int8_t offset)
+{
+  emit_wide_thumb
+    (_jit, THUMB_STREX|(_u4(rn)<<16)|(_u4(rt)<<12)|(_u4(rd)<<8)|_u8(offset));
+}
+
 static void
 T1_LDRSB(jit_state_t *_jit, int32_t rt, int32_t rn, int32_t rm)
 {
@@ -1171,7 +1194,7 @@ T1_LDR(jit_state_t *_jit, int32_t rt, int32_t rn, int32_t rm)
 static void
 T2_LDR(jit_state_t *_jit, int32_t rt, int32_t rn, int32_t rm)
 {
-  return torxr(_jit, THUMB2_LDR,rn,rt,rm);
+  emit_u16_with_pool(_jit, THUMB_LDR|(_u3(rm)<<6)|(_u3(rn)<<3)|_u3(rt));
 }
 
 static void
@@ -2975,3 +2998,58 @@ emit_veneer(jit_state_t *_jit, jit_pointer_t target)
   emit_u16(_jit, THUMB_MOV|((_u4(rd)&8)<<4)|(_u4(tmp)<<3)|(rd&7));
   emit_u32(_jit, (uint32_t) target);
 }
+
+static void
+ldr_atomic(jit_state_t *_jit, int32_t dst, int32_t loc)
+{
+  T1_DMB(_jit, DMB_ISH);
+  ldr_i(_jit, dst, loc);
+  T1_DMB(_jit, DMB_ISH);
+}
+
+static void
+str_atomic(jit_state_t *_jit, int32_t loc, int32_t val)
+{
+  T1_DMB(_jit, DMB_ISH);
+  str_i(_jit, loc, val);
+  T1_DMB(_jit, DMB_ISH);
+}
+
+static void
+swap_atomic(jit_state_t *_jit, int32_t dst, int32_t loc, int32_t val)
+{
+  int32_t result = jit_gpr_regno(get_temp_gpr(_jit));
+  int32_t val_or_tmp = dst == val ? jit_gpr_regno(get_temp_gpr(_jit)) : val;
+  movr(_jit, val_or_tmp, val);
+  T1_DMB(_jit, DMB_ISH);
+  void *retry = jit_address(_jit);
+  T1_LDREX(_jit, dst, loc, 0);
+  T1_STREX(_jit, result, val_or_tmp, loc, 0);
+  jit_patch_there(_jit, bnei(_jit, result, 0), retry);
+  T1_DMB(_jit, DMB_ISH);
+  if (dst == val) unget_temp_gpr(_jit);
+  unget_temp_gpr(_jit);
+}
+
+static void
+cas_atomic(jit_state_t *_jit, int32_t dst, int32_t loc, int32_t expected,
+           int32_t desired)
+{
+  int32_t dst_or_tmp;
+  if (dst == loc || dst == expected || dst == expected)
+    dst_or_tmp = jit_gpr_regno(get_temp_gpr(_jit));
+  else
+    dst_or_tmp = dst;
+  T1_DMB(_jit, DMB_ISH);
+  void *retry = jit_address(_jit);
+  T1_LDREX(_jit, dst_or_tmp, loc, 0);
+  jit_reloc_t bad = bner(_jit, dst_or_tmp, expected);
+  int result = jit_gpr_regno(get_temp_gpr(_jit));
+  T1_STREX(_jit, result, desired, loc, 0);
+  jit_patch_there(_jit, bnei(_jit, result, 0), retry);
+  unget_temp_gpr(_jit);
+  jit_patch_here(_jit, bad);
+  T1_DMB(_jit, DMB_ISH);
+  movr(_jit, dst, dst_or_tmp);
+  unget_temp_gpr(_jit);
+}
diff --git a/libguile/lightening/lightening/lightening.c b/libguile/lightening/lightening/lightening.c
index 4f03851d7..ca5708f0a 100644
--- a/libguile/lightening/lightening/lightening.c
+++ b/libguile/lightening/lightening/lightening.c
@@ -96,10 +96,6 @@ static struct jit_literal_pool* alloc_literal_pool(jit_state_t *_jit,
                                                    size_t capacity);
 static void reset_literal_pool(jit_state_t *_jit,
                                struct jit_literal_pool *pool);
-static void grow_literal_pool(jit_state_t *_jit);
-static jit_bool_t add_literal_pool_entry(jit_state_t *_jit,
-                                         struct jit_literal_pool_entry entry,
-                                         uint32_t max_offset);
 static jit_bool_t add_pending_literal(jit_state_t *_jit, jit_reloc_t src,
                                       uint8_t max_offset_bits);
 static void remove_pending_literal(jit_state_t *_jit, jit_reloc_t src);
@@ -1279,9 +1275,11 @@ static jit_bool_t
 add_literal_pool_entry(jit_state_t *_jit, struct jit_literal_pool_entry entry,
                        uint32_t max_offset)
 {
+  if (_jit->overflow)
+    return 1;
+
   if (max_offset <= literal_pool_byte_size(_jit->pool)) {
     emit_literal_pool(_jit, GUARD_NEEDED);
-    ASSERT(_jit->pool->size == 0);
     return 0;
   }
 
diff --git a/libguile/lightening/lightening/x86-cpu.c b/libguile/lightening/lightening/x86-cpu.c
index ab0da684e..28ec73886 100644
--- a/libguile/lightening/lightening/x86-cpu.c
+++ b/libguile/lightening/lightening/x86-cpu.c
@@ -610,6 +610,30 @@ xchgr(jit_state_t *_jit, int32_t r0, int32_t r1)
 }
 
 static void
+xchgrm(jit_state_t *_jit, int32_t val_and_dst, int32_t loc)
+{
+  rex(_jit, 0, WIDE, val_and_dst, _NOREG, loc);
+  ic(_jit, 0x87);
+  rx(_jit, val_and_dst, 0, loc, _NOREG, _SCL1);
+}
+
+static void
+lock(jit_state_t *_jit)
+{
+  ic(_jit, 0xf0);
+}
+
+static void
+cmpxchgmr(jit_state_t *_jit, int32_t loc, int32_t desired)
+{
+  lock(_jit);
+  rex(_jit, 0, WIDE, desired, _NOREG, loc);
+  ic(_jit, 0x0f);
+  ic(_jit, 0xb1);
+  rx(_jit, desired, 0, loc, _NOREG, _SCL1);
+}
+
+static void
 testr(jit_state_t *_jit, int32_t r0, int32_t r1)
 {
   rex(_jit, 0, WIDE, r1, _NOREG, r0);
@@ -2634,3 +2658,98 @@ retval_l(jit_state_t *_jit, int32_t r0)
   movr(_jit, r0, _RAX_REGNO);
 }
 #endif
+
+static void
+mfence(jit_state_t *_jit)
+{
+  ic(_jit, 0x0f);
+  ic(_jit, 0xae);
+  ic(_jit, 0xf0);
+}
+
+static void
+ldr_atomic(jit_state_t *_jit, int32_t dst, int32_t loc)
+{
+#if __X64
+  ldr_l(_jit, dst, loc);
+#else
+  ldr_i(_jit, dst, loc);
+#endif
+}
+
+static void
+str_atomic(jit_state_t *_jit, int32_t loc, int32_t val)
+{
+#if __X64
+  str_l(_jit, loc, val);
+#else
+  str_i(_jit, loc, val);
+#endif
+  mfence(_jit);
+}
+
+static void
+swap_atomic(jit_state_t *_jit, int32_t dst, int32_t loc, int32_t val)
+{
+  if (dst == val) {
+    xchgrm(_jit, dst, loc);
+  } else {
+    int32_t tmp = jit_gpr_regno(get_temp_gpr(_jit));
+    movr(_jit, tmp, val);
+    xchgrm(_jit, tmp, loc);
+    movr(_jit, dst, tmp);
+    unget_temp_gpr(_jit);
+  }
+}
+
+static void
+cas_atomic(jit_state_t *_jit, int32_t dst, int32_t loc, int32_t expected,
+           int32_t desired)
+{
+  ASSERT(loc != expected);
+  ASSERT(loc != desired);
+
+  if (dst == jit_gpr_regno(_RAX)) {
+    if (loc == dst) {
+      int32_t tmp = jit_gpr_regno(get_temp_gpr(_jit));
+      movr(_jit, tmp ,loc);
+      movr(_jit, dst, expected);
+      cmpxchgmr(_jit, tmp, desired);
+      unget_temp_gpr(_jit);
+    } else {
+      movr(_jit, dst, expected);
+      cmpxchgmr(_jit, loc, desired);
+    }
+  } else if (loc == jit_gpr_regno(_RAX)) {
+    int32_t tmp = jit_gpr_regno(get_temp_gpr(_jit));
+    movr(_jit, tmp, loc);
+    movr(_jit, jit_gpr_regno(_RAX), expected);
+    cmpxchgmr(_jit, tmp, desired);
+    movr(_jit, dst, jit_gpr_regno(_RAX));
+    movr(_jit, loc, tmp);
+    unget_temp_gpr(_jit);
+  } else if (expected == jit_gpr_regno(_RAX)) {
+    int32_t tmp = jit_gpr_regno(get_temp_gpr(_jit));
+    movr(_jit, tmp, expected);
+    cmpxchgmr(_jit, loc, desired);
+    movr(_jit, dst, jit_gpr_regno(_RAX));
+    movr(_jit, expected, tmp);
+    unget_temp_gpr(_jit);
+  } else if (desired == jit_gpr_regno(_RAX)) {
+    int32_t tmp = jit_gpr_regno(get_temp_gpr(_jit));
+    movr(_jit, tmp, desired);
+    movr(_jit, jit_gpr_regno(_RAX), expected);
+    cmpxchgmr(_jit, loc, tmp);
+    movr(_jit, dst, jit_gpr_regno(_RAX));
+    movr(_jit, desired, tmp);
+    unget_temp_gpr(_jit);
+  } else {
+    int32_t tmp = jit_gpr_regno(get_temp_gpr(_jit));
+    movr(_jit, tmp, jit_gpr_regno(_RAX));
+    movr(_jit, jit_gpr_regno(_RAX), expected);
+    cmpxchgmr(_jit, loc, desired);
+    movr(_jit, dst, jit_gpr_regno(_RAX));
+    movr(_jit, jit_gpr_regno(_RAX), tmp);
+    unget_temp_gpr(_jit);
+  }
+}
diff --git a/libguile/lightening/tests/Makefile b/libguile/lightening/tests/Makefile
index 2a048501d..81279720d 100644
--- a/libguile/lightening/tests/Makefile
+++ b/libguile/lightening/tests/Makefile
@@ -1,6 +1,5 @@
 TESTS=$(sort $(basename $(wildcard *.c)))
-TARGETS=native ia32 aarch64
-ALL_TARGETS=$(TARGETS) armv7
+TARGETS=native ia32 aarch64 armv7
 
 # Suitable values of cross-compiler variables for Debian:
 #
@@ -55,9 +54,9 @@ test-armv7-%: CC = $(CC_ARMv7)
 test-armv7-%: %.c lightening-armv7.o test.h
 	$(CC) $(CFLAGS) $(CPPFLAGS) -I.. -o $@ lightening-armv7.o $<
 
-.PRECIOUS: $(foreach TARGET,$(ALL_TARGETS),$(addprefix test-$(TARGET)-,$(TESTS)))
-.PRECIOUS: $(foreach TARGET,$(ALL_TARGETS),lightening-$(TARGET).o)
+.PRECIOUS: $(foreach TARGET,$(TARGETS),$(addprefix test-$(TARGET)-,$(TESTS)))
+.PRECIOUS: $(foreach TARGET,$(TARGETS),lightening-$(TARGET).o)
 
 clean:
-	rm -f $(foreach TARGET,$(ALL_TARGETS),$(addprefix test-$(TARGET)-,$(TESTS)))
-	rm -f $(foreach TARGET,$(ALL_TARGETS),lightening-$(TARGET).o)
+	rm -f $(foreach TARGET,$(TARGETS),$(addprefix test-$(TARGET)-,$(TESTS)))
+	rm -f $(foreach TARGET,$(TARGETS),lightening-$(TARGET).o)