Merge from upstream lightening

2 files changed, 412 insertions, 349 deletions

diff --git a/libguile/lightening/lightening.h b/libguile/lightening/lightening.h
index 20a14412b..4fcec669b 100644
--- a/libguile/lightening/lightening.h
+++ b/libguile/lightening/lightening.h
@@ -156,9 +156,9 @@ typedef struct jit_operand
   union
   {
     intptr_t imm;
-    jit_gpr_t gpr;
+    struct { jit_gpr_t gpr; ptrdiff_t addend; } gpr;
     jit_fpr_t fpr;
-    struct { jit_gpr_t base; ptrdiff_t offset; } mem;
+    struct { jit_gpr_t base; ptrdiff_t offset; ptrdiff_t addend; } mem;
   } loc;
 } jit_operand_t;
 
@@ -169,9 +169,17 @@ jit_operand_imm (enum jit_operand_abi abi, intptr_t imm)
 }
 
 static inline jit_operand_t
+jit_operand_gpr_with_addend (enum jit_operand_abi abi, jit_gpr_t gpr,
+                             ptrdiff_t addend)
+{
+  return (jit_operand_t){ abi, JIT_OPERAND_KIND_GPR,
+      { .gpr = { gpr, addend } } };
+}
+
+static inline jit_operand_t
 jit_operand_gpr (enum jit_operand_abi abi, jit_gpr_t gpr)
 {
-  return (jit_operand_t){ abi, JIT_OPERAND_KIND_GPR, { .gpr = gpr } };
+  return jit_operand_gpr_with_addend (abi, gpr, 0);
 }
 
 static inline jit_operand_t
@@ -181,16 +189,34 @@ jit_operand_fpr (enum jit_operand_abi abi, jit_fpr_t fpr)
 }
 
 static inline jit_operand_t
+jit_operand_mem_with_addend (enum jit_operand_abi abi, jit_gpr_t base,
+                             ptrdiff_t offset, ptrdiff_t addend)
+{
+  return (jit_operand_t){ abi, JIT_OPERAND_KIND_MEM,
+      { .mem = { base, offset, addend } } };
+}
+
+static inline jit_operand_t
 jit_operand_mem (enum jit_operand_abi abi, jit_gpr_t base, ptrdiff_t offset)
 {
-  return (jit_operand_t){ abi, JIT_OPERAND_KIND_MEM, { .mem = { base, offset } } };
+  return jit_operand_mem_with_addend (abi, base, offset, 0);
 }
 
-typedef union jit_anyreg
+static inline jit_operand_t
+jit_operand_addi (jit_operand_t op, ptrdiff_t addend)
 {
-  jit_gpr_t gpr;
-  jit_fpr_t fpr;
-} jit_anyreg_t;
+  switch (op.kind) {
+  case JIT_OPERAND_KIND_GPR:
+    return jit_operand_gpr_with_addend (op.abi, op.loc.gpr.gpr,
+                                        op.loc.gpr.addend + addend);
+  case JIT_OPERAND_KIND_MEM:
+    return jit_operand_mem_with_addend (op.abi, op.loc.mem.base,
+                                        op.loc.mem.offset,
+                                        op.loc.mem.addend + addend);
+  default:
+    abort ();
+  }
+}
 
 JIT_API jit_bool_t init_jit(void);
 
@@ -212,6 +238,9 @@ JIT_API void jit_patch_there(jit_state_t*, jit_reloc_t, jit_pointer_t);
 JIT_API jit_bool_t jit_gpr_is_callee_save (jit_state_t*, jit_gpr_t);
 JIT_API jit_bool_t jit_fpr_is_callee_save (jit_state_t*, jit_fpr_t);
 
+JIT_API void jit_move_operands (jit_state_t *_jit, jit_operand_t *dst,
+                                jit_operand_t *src, size_t argc);
+
 /* Note that all functions that take jit_operand_t args[] use the args
    as scratch space while shuffling values into position.  */
 JIT_API void jit_calli(jit_state_t *, jit_pointer_t f,
diff --git a/libguile/lightening/lightening/x86.c b/libguile/lightening/lightening/x86.c
index b8d1150d9..3b89c0ca4 100644
--- a/libguile/lightening/lightening/x86.c
+++ b/libguile/lightening/lightening/x86.c
@@ -376,107 +376,6 @@ is_gpr_arg(enum jit_operand_abi arg)
   return !is_fpr_arg(arg);
 }
 
-static const jit_gpr_t abi_gpr_args[] = {
-#if __X32
-  /* No GPRs in args.  */
-#elif __CYGWIN__
-  JIT_GPR(_RCX), JIT_GPR(_RDX), JIT_GPR(_R8), JIT_GPR(_R9)
-#else
-  JIT_GPR(_RDI), JIT_GPR(_RSI), JIT_GPR(_RDX), JIT_GPR(_RCX), JIT_GPR(_R8), JIT_GPR(_R9)
-#endif
-};
-
-static const jit_fpr_t abi_fpr_args[] = {
-#if __X32
-  /* No FPRs in args.  */
-#elif __CYGWIN__
-  JIT_FPR(_XMM0), JIT_FPR(_XMM1), JIT_FPR(_XMM2), JIT_FPR(_XMM3)
-#else
-  JIT_FPR(_XMM0), JIT_FPR(_XMM1), JIT_FPR(_XMM2), JIT_FPR(_XMM3), JIT_FPR(_XMM4), JIT_FPR(_XMM5), JIT_FPR(_XMM6), JIT_FPR(_XMM7)
-#endif
-};
-
-static const int abi_gpr_arg_count = sizeof(abi_gpr_args) / sizeof(abi_gpr_args[0]);
-static const int abi_fpr_arg_count = sizeof(abi_fpr_args) / sizeof(abi_fpr_args[0]);
-
-struct abi_arg_iterator
-{
-  const jit_operand_t *args;
-  size_t argc;
-
-  size_t arg_idx;
-  size_t gpr_idx;
-  size_t fpr_idx;
-  size_t stack_size;
-};
-
-static size_t
-jit_operand_abi_sizeof(enum jit_operand_abi abi)
-{
-  switch (abi) {
-  case JIT_OPERAND_ABI_UINT8:
-  case JIT_OPERAND_ABI_INT8:
-    return 1;
-  case JIT_OPERAND_ABI_UINT16:
-  case JIT_OPERAND_ABI_INT16:
-    return 2;
-  case JIT_OPERAND_ABI_UINT32:
-  case JIT_OPERAND_ABI_INT32:
-    return 4;
-  case JIT_OPERAND_ABI_UINT64:
-  case JIT_OPERAND_ABI_INT64:
-    return 8;
-  case JIT_OPERAND_ABI_POINTER:
-    return CHOOSE_32_64(4, 8);
-  case JIT_OPERAND_ABI_FLOAT:
-    return 4;
-  case JIT_OPERAND_ABI_DOUBLE:
-    return 8;
-  default:
-    abort();
-  }
-}
-
-static size_t
-round_size_up_to_words(size_t bytes)
-{
-  size_t word_size = CHOOSE_32_64(4, 8);
-  size_t words = (bytes + word_size - 1) / word_size;
-  return words * word_size;
-}
-
-static void
-reset_abi_arg_iterator(struct abi_arg_iterator *iter, size_t argc,
-                       const jit_operand_t *args)
-{
-  memset(iter, 0, sizeof *iter);
-  iter->argc = argc;
-  iter->args = args;
-}
-
-static void
-next_abi_arg(struct abi_arg_iterator *iter, jit_operand_t *arg)
-{
-  ASSERT(iter->arg_idx < iter->argc);
-  enum jit_operand_abi abi = iter->args[iter->arg_idx].abi;
-  if (is_gpr_arg(abi) && iter->gpr_idx < abi_gpr_arg_count) {
-    *arg = jit_operand_gpr (abi, abi_gpr_args[iter->gpr_idx++]);
-#ifdef __CYGWIN__
-    iter->fpr_idx++;
-#endif
-  } else if (is_fpr_arg(abi) && iter->fpr_idx < abi_fpr_arg_count) {
-    *arg = jit_operand_fpr (abi, abi_fpr_args[iter->fpr_idx++]);
-#ifdef __CYGWIN__
-    iter->gpr_idx++;
-#endif
-  } else {
-    *arg = jit_operand_mem (abi, JIT_GPR(_RSP), iter->stack_size);
-    size_t bytes = jit_operand_abi_sizeof (abi);
-    iter->stack_size += round_size_up_to_words (bytes);
-  }
-  iter->arg_idx++;
-}
-
 static void
 abi_imm_to_gpr(jit_state_t *_jit, enum jit_operand_abi abi, jit_gpr_t dst,
                intptr_t imm)
@@ -521,7 +420,7 @@ abi_imm_to_gpr(jit_state_t *_jit, enum jit_operand_abi abi, jit_gpr_t dst,
 
 static void
 abi_gpr_to_mem(jit_state_t *_jit, enum jit_operand_abi abi,
-               jit_gpr_t src, jit_gpr_t base, ptrdiff_t offset)
+               jit_gpr_t base, ptrdiff_t offset, jit_gpr_t src)
 {
   switch (abi) {
   case JIT_OPERAND_ABI_UINT8:
@@ -553,7 +452,7 @@ abi_gpr_to_mem(jit_state_t *_jit, enum jit_operand_abi abi,
 
 static void
 abi_fpr_to_mem(jit_state_t *_jit, enum jit_operand_abi abi,
-               jit_fpr_t src, jit_gpr_t base, ptrdiff_t offset)
+               jit_gpr_t base, ptrdiff_t offset, jit_fpr_t src)
 {
   switch (abi) {
   case JIT_OPERAND_ABI_FLOAT:
@@ -621,245 +520,441 @@ abi_mem_to_fpr(jit_state_t *_jit, enum jit_operand_abi abi,
 }
 
 static void
-store_mem_abi_arg(jit_state_t *_jit, jit_operand_t *arg, jit_gpr_t base,
-                  ptrdiff_t offset)
+abi_imm_to_mem(jit_state_t *_jit, enum jit_operand_abi abi, jit_gpr_t base,
+               ptrdiff_t offset, intmax_t imm)
 {
-  switch (arg->kind) {
-  case JIT_OPERAND_KIND_GPR:
-    abi_gpr_to_mem(_jit, arg->abi, arg->loc.gpr, base, offset);
-    break;
+  ASSERT(!is_fpr_arg(abi));
 
-  case JIT_OPERAND_KIND_FPR:
-    abi_fpr_to_mem(_jit, arg->abi, arg->loc.fpr, base, offset);
-    break;
-
-  case JIT_OPERAND_KIND_MEM:
-    if (is_gpr_arg(arg->abi)) {
-      jit_gpr_t tmp = get_temp_gpr(_jit);
-      abi_mem_to_gpr(_jit, arg->abi, tmp, arg->loc.mem.base,
-                     arg->loc.mem.offset);
-      abi_gpr_to_mem(_jit, arg->abi, tmp, base, offset);
-      unget_temp_gpr(_jit);
-    } else {
-      jit_fpr_t tmp = get_temp_xpr(_jit);
-      abi_mem_to_fpr(_jit, arg->abi, tmp, arg->loc.mem.base,
-                     arg->loc.mem.offset);
-      abi_fpr_to_mem(_jit, arg->abi, tmp, base, offset);
-      unget_temp_xpr(_jit);
-    }
-    break;
+  jit_gpr_t tmp = get_temp_gpr(_jit);
+  abi_imm_to_gpr(_jit, abi, tmp, imm);
+  abi_gpr_to_mem(_jit, abi, base, offset, tmp);
+  unget_temp_gpr(_jit);
+}
 
-  case JIT_OPERAND_KIND_IMM: {
-    if (is_gpr_arg(arg->abi)) {
-      jit_gpr_t tmp = get_temp_gpr(_jit);
-      abi_imm_to_gpr(_jit, arg->abi, tmp, arg->loc.imm);
-      abi_gpr_to_mem(_jit, arg->abi, tmp, base, offset);
-      unget_temp_gpr(_jit);
-    } else {
-      /* Floating-point immediates not supported yet.  */
-      abort ();
-    }
-    break;
+static void
+abi_mem_to_mem(jit_state_t *_jit, enum jit_operand_abi abi, jit_gpr_t base,
+               ptrdiff_t offset, jit_gpr_t src_base, ptrdiff_t src_offset)
+{
+  if (is_gpr_arg (abi)) {
+    jit_gpr_t tmp = get_temp_gpr(_jit);
+    abi_mem_to_gpr(_jit, abi, tmp, src_base, src_offset);
+    abi_gpr_to_mem(_jit, abi, base, offset, tmp);
+    unget_temp_gpr(_jit);
+  } else {
+    jit_fpr_t tmp = get_temp_xpr(_jit);
+    abi_mem_to_fpr(_jit, abi, tmp, src_base, src_offset);
+    abi_fpr_to_mem(_jit, abi, base, offset, tmp);
+    unget_temp_xpr(_jit);
   }
+}
+
+#define MOVE_KIND(a, b) ((((int) a) << 4) | ((int) b))
+
+#define MOVE_KIND_ENUM(a, b) \
+  MOVE_##a##_TO_##b = MOVE_KIND(JIT_OPERAND_KIND_##a, JIT_OPERAND_KIND_##b)
+enum move_kind {
+  MOVE_KIND_ENUM(IMM, GPR),
+  MOVE_KIND_ENUM(GPR, GPR),
+  MOVE_KIND_ENUM(MEM, GPR),
+  MOVE_KIND_ENUM(FPR, FPR),
+  MOVE_KIND_ENUM(MEM, FPR),
+  MOVE_KIND_ENUM(IMM, MEM),
+  MOVE_KIND_ENUM(GPR, MEM),
+  MOVE_KIND_ENUM(FPR, MEM),
+  MOVE_KIND_ENUM(MEM, MEM)
+};
+#undef MOVE_KIND_ENUM
+
+static void
+move_operand(jit_state_t *_jit, jit_operand_t dst, jit_operand_t src)
+{
+  switch (MOVE_KIND (src.kind, dst.kind)) {
+  case MOVE_IMM_TO_GPR:
+    return abi_imm_to_gpr(_jit, src.abi, dst.loc.gpr.gpr, src.loc.imm);
+
+  case MOVE_GPR_TO_GPR:
+    return jit_movr(_jit, dst.loc.gpr.gpr, src.loc.gpr.gpr);
+
+  case MOVE_MEM_TO_GPR:
+    return abi_mem_to_gpr(_jit, src.abi, dst.loc.gpr.gpr, src.loc.mem.base,
+                          src.loc.mem.offset);
+
+  case MOVE_FPR_TO_FPR:
+    return jit_movr_d(_jit, dst.loc.fpr, src.loc.fpr);
+
+  case MOVE_MEM_TO_FPR:
+    return abi_mem_to_fpr(_jit, src.abi, dst.loc.fpr, src.loc.mem.base,
+                          src.loc.mem.offset);
+
+  case MOVE_IMM_TO_MEM:
+    return abi_imm_to_mem(_jit, src.abi, dst.loc.mem.base, dst.loc.mem.offset,
+                          src.loc.imm);
+
+  case MOVE_GPR_TO_MEM:
+    return abi_gpr_to_mem(_jit, src.abi, dst.loc.mem.base, dst.loc.mem.offset,
+                          src.loc.gpr.gpr);
+
+  case MOVE_FPR_TO_MEM:
+    return abi_fpr_to_mem(_jit, src.abi, dst.loc.mem.base, dst.loc.mem.offset,
+                          src.loc.fpr);
+
+  case MOVE_MEM_TO_MEM:
+    return abi_mem_to_mem(_jit, src.abi, dst.loc.mem.base, dst.loc.mem.offset,
+                          src.loc.mem.base, src.loc.mem.offset);
 
   default:
     abort();
   }
+}
+
+// A direct transliteration of "Tilting at windmills with Coq: formal
+// verification of a compilation algorithm for parallel moves" by
+// Laurence Rideau, Bernard Paul Serpette, and Xavier Leroy:
+// https://xavierleroy.org/publi/parallel-move.pdf
 
-  arg->kind = JIT_OPERAND_KIND_MEM;
-  arg->loc.mem.base = base;
-  arg->loc.mem.offset = offset;
+enum move_status { TO_MOVE, BEING_MOVED, MOVED };
+
+static inline int
+already_in_place(jit_operand_t src, jit_operand_t dst)
+{
+  switch (MOVE_KIND(src.kind, dst.kind)) {
+  case MOVE_GPR_TO_GPR:
+    return jit_same_gprs (src.loc.gpr.gpr, dst.loc.gpr.gpr);
+  case MOVE_FPR_TO_FPR:
+    return jit_same_fprs (src.loc.fpr, dst.loc.fpr);
+  case MOVE_MEM_TO_MEM:
+    return jit_same_gprs (src.loc.mem.base, dst.loc.mem.base) &&
+      src.loc.mem.offset == dst.loc.mem.offset;
+  default:
+    return 0;
+  }
 }
 
-static void
-shuffle_gpr_arg(jit_state_t *_jit, jit_gpr_t dst, size_t argc,
-                jit_operand_t *args, size_t idx)
+static inline int
+write_would_clobber(jit_operand_t src, jit_operand_t dst)
 {
-  ASSERT(args[idx].kind == JIT_OPERAND_KIND_GPR);
+  if (already_in_place (src, dst))
+    return 1;
 
-  if (rn(args[idx].loc.gpr) == rn(dst))
-    return;
+  if (MOVE_KIND(src.kind, dst.kind) == MOVE_MEM_TO_GPR)
+    return jit_same_gprs(src.loc.mem.base, dst.loc.gpr.gpr);
 
-  /* Arg in a reg but it's not the right one.  See if this reg
-     holds some other arg, and swap if so.  */
-  for (size_t j=idx+1; j<argc; j++)
-    if (args[j].kind == JIT_OPERAND_KIND_GPR && rn(args[j].loc.gpr) == rn(dst))
-      {
-        xchgr(_jit, rn(args[idx].loc.gpr), rn(dst));
-        args[j].loc.gpr = args[idx].loc.gpr;
-        args[idx].loc.gpr = dst;
-        /* Could be this register holds a value for more than one argument;
-           update subsequent args if any.  */
-        for (size_t k=j+1; k<argc; k++)
-          if (args[k].kind == JIT_OPERAND_KIND_GPR && rn(args[k].loc.gpr) == rn(dst))
-            args[k].loc.gpr = args[j].loc.gpr;
-        return;
-      }
+  return 0;
+}
 
-  /* Arg in reg, but it's not the right one, and the desired reg
-     is free.  */
-  jit_movr(_jit, dst, args[idx].loc.gpr);
-  args[idx].loc.gpr = dst;
+static inline ptrdiff_t
+operand_addend(jit_operand_t op)
+{
+  switch (op.kind) {
+  case JIT_OPERAND_KIND_GPR:
+    return op.loc.gpr.addend;
+  case JIT_OPERAND_KIND_MEM:
+    return op.loc.mem.addend;
+  default:
+    abort();
+  }
 }
 
 static void
-shuffle_fpr_arg(jit_state_t *_jit, jit_fpr_t dst, size_t argc,
-                jit_operand_t *args, size_t idx)
+move_one(jit_state_t *_jit, jit_operand_t *dst, jit_operand_t *src,
+         size_t argc, enum move_status *status, size_t i)
 {
-  ASSERT(args[idx].kind == JIT_OPERAND_KIND_FPR);
+  int tmp_gpr = 0, tmp_fpr = 0;
 
-  if (rn(args[idx].loc.fpr) == rn(dst))
+  if (already_in_place(src[i], dst[i]))
     return;
 
-  /* Arg in a reg but it's not the right one.  See if this reg
-     holds some other arg, and swap if so.  */
-  for (size_t j=idx+1; j<argc; j++)
-    if (args[j].kind == JIT_OPERAND_KIND_FPR && rn(args[j].loc.fpr) == rn(dst))
-      {
-        jit_fpr_t tmp = get_temp_xpr(_jit);
-        jit_movr_d (_jit, tmp, args[idx].loc.fpr);
-        jit_movr_d (_jit, args[idx].loc.fpr, dst);
-        jit_movr_d (_jit, dst, tmp);
-        unget_temp_xpr(_jit);
-        args[j].loc.fpr = args[idx].loc.fpr;
-        args[idx].loc.fpr = dst;
-        /* Could be this register holds a value for more than one argument;
-           update subsequent args if any.  */
-        for (size_t k=j+1; k<argc; k++)
-          if (args[k].kind == JIT_OPERAND_KIND_FPR && rn(args[k].loc.fpr) == rn(dst))
-            args[k].loc.fpr = args[j].loc.fpr;
-        return;
+  status[i] = BEING_MOVED;
+  for (size_t j = 0; j < argc; j++) {
+    if (write_would_clobber(src[j], dst[i])) {
+      switch (status[j]) {
+      case TO_MOVE:
+        move_one(_jit, dst, src, argc, status, j);
+        break;
+      case BEING_MOVED: {
+        jit_operand_t tmp;
+        if (is_fpr_arg (src[j].kind)) {
+          tmp_fpr = 1;
+          tmp = jit_operand_fpr(src[j].abi, get_temp_xpr(_jit));
+        } else {
+          tmp_gpr = 1;
+          /* Preserve addend, if any, from source operand, to be applied
+             at the end.  */
+          tmp = jit_operand_gpr_with_addend(src[j].abi, get_temp_gpr(_jit),
+                                            operand_addend(src[j]));
+        }
+        move_operand (_jit, tmp, src[j]);
+        src[j] = tmp;
+        break;
       }
+      case MOVED:
+        break;
+      default:
+        abort ();
+      }
+    }
+  }
 
-  /* Arg in reg, but it's not the right one, and the desired reg
-     is free.  */
-  jit_movr_d(_jit, dst, args[idx].loc.fpr);
-  args[idx].loc.fpr = dst;
+  move_operand (_jit, dst[i], src[i]);
+  status[i] = MOVED;
+  if (tmp_gpr)
+    unget_temp_gpr(_jit);
+  else if (tmp_fpr)
+    unget_temp_xpr(_jit);
 }
 
 static void
-prepare_args(jit_state_t *_jit, size_t argc, jit_operand_t args[])
+apply_addend(jit_state_t *_jit, jit_operand_t dst, jit_operand_t src)
 {
-  jit_operand_t scratch;
-  struct abi_arg_iterator iter;
-  
-  // Compute stack arg size.
-  reset_abi_arg_iterator(&iter, argc, args);
-  for (size_t i = 0; i < argc; i++)
-    next_abi_arg(&iter, &scratch);
-
-  // Put all ABI memory arguments in place.  We do this first because it might
-  // free up some registers.
-  if (iter.stack_size)
-    {
-      size_t stack_size = iter.stack_size;
-      subi(_jit, _RSP_REGNO, _RSP_REGNO, stack_size);
-      reset_abi_arg_iterator(&iter, argc, args);
-      for (size_t i = 0; i < argc; i++) {
-        next_abi_arg(&iter, &scratch);
-        if (scratch.kind == JIT_OPERAND_KIND_MEM)
-          store_mem_abi_arg(_jit, &args[i], scratch.loc.mem.base,
-                            scratch.loc.mem.offset);
-      }
+  switch (MOVE_KIND(src.kind, dst.kind)) {
+  case MOVE_GPR_TO_GPR:
+  case MOVE_MEM_TO_GPR:
+    if (operand_addend(src))
+      jit_addi(_jit, dst.loc.gpr.gpr, dst.loc.gpr.gpr, operand_addend(src));
+    break;
+  case MOVE_GPR_TO_MEM:
+  case MOVE_MEM_TO_MEM:
+    if (operand_addend(src)) {
+      jit_gpr_t tmp = get_temp_gpr(_jit);
+      abi_mem_to_gpr(_jit, dst.abi, tmp, dst.loc.mem.base, dst.loc.mem.offset);
+      jit_addi(_jit, tmp, tmp, operand_addend(src));
+      abi_gpr_to_mem(_jit, dst.abi, dst.loc.mem.base, dst.loc.mem.offset, tmp);
     }
+    break;
+  default:
+    break;
+  }
+}
 
-  // We move on now to the ABI register arguments.  All args whose values are in
-  // registers are ABI register arguments, but they might not be the right
-  // register for the correponding ABI argument.  Note that there may be ABI
-  // register arguments whose values are still in memory or as immediates; we
-  // will load them later.
-  reset_abi_arg_iterator(&iter, argc, args);
-  for (size_t i = 0; i < argc; i++)
-    {
-      next_abi_arg(&iter, &scratch);
-      switch (scratch.kind) {
+/* Preconditions: No dest operand is IMM.  No dest operand aliases
+   another dest operand.  No dest MEM operand uses a base register which
+   is used as a dest GPR.  No dst operand has an addend.  The registers
+   returned by get_temp_gpr and get_temp_fpr do not appear in source or
+   dest args.  */
+void
+jit_move_operands(jit_state_t *_jit, jit_operand_t *dst, jit_operand_t *src,
+                  size_t argc)
+{
+  // Check preconditions, except the condition about tmp registers.
+  {
+    uint64_t src_gprs = 0;
+    uint64_t dst_gprs = 0;
+    uint64_t dst_fprs = 0;
+    uint64_t dst_mem_base_gprs = 0;
+    for (size_t i = 0; i < argc; i++) {
+      switch (src[i].kind) {
       case JIT_OPERAND_KIND_GPR:
-        if (args[i].kind == JIT_OPERAND_KIND_GPR)
-          shuffle_gpr_arg(_jit, scratch.loc.gpr, argc, args, i);
+        src_gprs |= 1ULL << rn(src[i].loc.gpr.gpr);
         break;
-        
       case JIT_OPERAND_KIND_FPR:
-        if (args[i].kind == JIT_OPERAND_KIND_FPR)
-          shuffle_fpr_arg(_jit, scratch.loc.fpr, argc, args, i);
+      case JIT_OPERAND_KIND_IMM:
+      case JIT_OPERAND_KIND_MEM:
         break;
-
       default:
+        abort();
+      }
+      switch (dst[i].kind) {
+      case JIT_OPERAND_KIND_GPR: {
+        ASSERT(dst[i].loc.gpr.addend == 0);
+        uint64_t bit = 1ULL << rn(dst[i].loc.gpr.gpr);
+        ASSERT((dst_gprs & bit) == 0);
+        dst_gprs |= bit;
         break;
       }
-    }
-  
-  // The only thing that's left is ABI register arguments whose values are still
-  // in memory or immediates; load them now.
-  reset_abi_arg_iterator(&iter, argc, args);
-  for (size_t i = 0; i < argc; i++)
-    {
-      next_abi_arg(&iter, &scratch);
-      switch (scratch.kind) {
-      case JIT_OPERAND_KIND_GPR:
-        if (args[i].kind == JIT_OPERAND_KIND_MEM) {
-          abi_mem_to_gpr(_jit, args[i].abi, scratch.loc.gpr,
-                         args[i].loc.mem.base, args[i].loc.mem.offset);
-          args[i].kind = JIT_OPERAND_KIND_GPR;
-          args[i].loc.gpr = scratch.loc.gpr;
-        } else if (args[i].kind == JIT_OPERAND_KIND_IMM) {
-          abi_imm_to_gpr(_jit, args[i].abi, scratch.loc.gpr, args[i].loc.imm);
-          args[i].kind = JIT_OPERAND_KIND_GPR;
-          args[i].loc.gpr = scratch.loc.gpr;
-        }
+      case JIT_OPERAND_KIND_FPR: {
+        uint64_t bit = 1ULL << rn(dst[i].loc.fpr);
+        ASSERT((dst_fprs & bit) == 0);
+        dst_fprs |= bit;
         break;
-        
-      case JIT_OPERAND_KIND_FPR:
-        if (args[i].kind == JIT_OPERAND_KIND_MEM) {
-          abi_mem_to_fpr(_jit, args[i].abi, scratch.loc.fpr,
-                         args[i].loc.mem.base, args[i].loc.mem.offset);
-          args[i].kind = JIT_OPERAND_KIND_FPR;
-          args[i].loc.fpr = scratch.loc.fpr;
-        } else if (args[i].kind == JIT_OPERAND_KIND_IMM) {
-          /* Currently unsupported.  */
-          abort ();
-        }
+      }
+      case JIT_OPERAND_KIND_MEM: {
+        ASSERT(dst[i].loc.mem.addend == 0);
+        uint64_t bit = 1ULL << rn(dst[i].loc.mem.base);
+        dst_mem_base_gprs |= bit;
         break;
-
+      }
+      case JIT_OPERAND_KIND_IMM:
       default:
+        abort();
         break;
       }
     }
+    ASSERT(((src_gprs | dst_gprs) & dst_mem_base_gprs) == 0);
+  }
+
+  enum move_status status[argc];
+  for (size_t i = 0; i < argc; i++)
+    status[i] = TO_MOVE;
+  for (size_t i = 0; i < argc; i++)
+    if (status[i] == TO_MOVE)
+      move_one(_jit, dst, src, argc, status, i);
+
+  // Apply addends at the end.  We could do it earlier in some cases but
+  // at least at the end we know that an in-place increment of one
+  // operand won't alias another.
+  for (size_t i = 0; i < argc; i++)
+    if (status[i] == TO_MOVE)
+      apply_addend(_jit, dst[i], src[i]);
+}
+
+static const jit_gpr_t abi_gpr_args[] = {
+#if __X32
+  /* No GPRs in args.  */
+#elif __CYGWIN__
+  JIT_GPR(_RCX), JIT_GPR(_RDX), JIT_GPR(_R8), JIT_GPR(_R9)
+#else
+  JIT_GPR(_RDI), JIT_GPR(_RSI), JIT_GPR(_RDX), JIT_GPR(_RCX), JIT_GPR(_R8), JIT_GPR(_R9)
+#endif
+};
+
+static const jit_fpr_t abi_fpr_args[] = {
+#if __X32
+  /* No FPRs in args.  */
+#elif __CYGWIN__
+  JIT_FPR(_XMM0), JIT_FPR(_XMM1), JIT_FPR(_XMM2), JIT_FPR(_XMM3)
+#else
+  JIT_FPR(_XMM0), JIT_FPR(_XMM1), JIT_FPR(_XMM2), JIT_FPR(_XMM3), JIT_FPR(_XMM4), JIT_FPR(_XMM5), JIT_FPR(_XMM6), JIT_FPR(_XMM7)
+#endif
+};
+
+static const int abi_gpr_arg_count = sizeof(abi_gpr_args) / sizeof(abi_gpr_args[0]);
+static const int abi_fpr_arg_count = sizeof(abi_fpr_args) / sizeof(abi_fpr_args[0]);
+
+struct abi_arg_iterator
+{
+  const jit_operand_t *args;
+  size_t argc;
+
+  size_t arg_idx;
+  size_t gpr_idx;
+  size_t fpr_idx;
+  size_t stack_size;
+};
+
+static size_t
+jit_operand_abi_sizeof(enum jit_operand_abi abi)
+{
+  switch (abi) {
+  case JIT_OPERAND_ABI_UINT8:
+  case JIT_OPERAND_ABI_INT8:
+    return 1;
+  case JIT_OPERAND_ABI_UINT16:
+  case JIT_OPERAND_ABI_INT16:
+    return 2;
+  case JIT_OPERAND_ABI_UINT32:
+  case JIT_OPERAND_ABI_INT32:
+    return 4;
+  case JIT_OPERAND_ABI_UINT64:
+  case JIT_OPERAND_ABI_INT64:
+    return 8;
+  case JIT_OPERAND_ABI_POINTER:
+    return CHOOSE_32_64(4, 8);
+  case JIT_OPERAND_ABI_FLOAT:
+    return 4;
+  case JIT_OPERAND_ABI_DOUBLE:
+    return 8;
+  default:
+    abort();
+  }
+}
+
+static size_t
+round_size_up_to_words(size_t bytes)
+{
+  size_t word_size = CHOOSE_32_64(4, 8);
+  size_t words = (bytes + word_size - 1) / word_size;
+  return words * word_size;
 }
 
 static void
-cleanup_stack_after_call(jit_state_t *_jit, size_t argc,
-                         const jit_operand_t args[])
+reset_abi_arg_iterator(struct abi_arg_iterator *iter, size_t argc,
+                       const jit_operand_t *args)
 {
-  jit_operand_t scratch;
-  struct abi_arg_iterator iter;
+  memset(iter, 0, sizeof *iter);
+  iter->argc = argc;
+  iter->args = args;
+}
 
-  // Compute stack arg size.
+static void
+next_abi_arg(struct abi_arg_iterator *iter, jit_operand_t *arg)
+{
+  ASSERT(iter->arg_idx < iter->argc);
+  enum jit_operand_abi abi = iter->args[iter->arg_idx].abi;
+  if (is_gpr_arg(abi) && iter->gpr_idx < abi_gpr_arg_count) {
+    *arg = jit_operand_gpr (abi, abi_gpr_args[iter->gpr_idx++]);
+#ifdef __CYGWIN__
+    iter->fpr_idx++;
+#endif
+  } else if (is_fpr_arg(abi) && iter->fpr_idx < abi_fpr_arg_count) {
+    *arg = jit_operand_fpr (abi, abi_fpr_args[iter->fpr_idx++]);
+#ifdef __CYGWIN__
+    iter->gpr_idx++;
+#endif
+  } else {
+    *arg = jit_operand_mem (abi, JIT_GPR(_RSP), iter->stack_size);
+    size_t bytes = jit_operand_abi_sizeof (abi);
+    iter->stack_size += round_size_up_to_words (bytes);
+  }
+  iter->arg_idx++;
+}
+
+static size_t
+prepare_call_args(jit_state_t *_jit, size_t argc, jit_operand_t args[])
+{
+  jit_operand_t dst[argc];
+  struct abi_arg_iterator iter;
+  
+  // Compute shuffle destinations and space for spilled arguments.
   reset_abi_arg_iterator(&iter, argc, args);
   for (size_t i = 0; i < argc; i++)
-    next_abi_arg(&iter, &scratch);
+    next_abi_arg(&iter, &dst[i]);
 
+  // Reserve space for spilled arguments, and fix up SP-relative
+  // operands.
   if (iter.stack_size)
-    jit_addi(_jit, JIT_SP, JIT_SP, iter.stack_size);
+    {
+      jit_subi(_jit, JIT_SP, JIT_SP, iter.stack_size);
+      for (size_t i = 0; i < argc; i++) {
+        switch(args[i].kind) {
+        case JIT_OPERAND_KIND_GPR:
+          if (jit_same_gprs (args[i].loc.mem.base, JIT_SP))
+            args[i].loc.gpr.addend += iter.stack_size;
+          break;
+        case JIT_OPERAND_KIND_MEM:
+          if (jit_same_gprs (args[i].loc.mem.base, JIT_SP))
+            args[i].loc.mem.offset += iter.stack_size;
+          break;
+        default:
+          break;
+        }
+      }
+    }
+
+  jit_move_operands(_jit, dst, args, argc);
+
+  return iter.stack_size;
 }
 
 void
 jit_calli(jit_state_t *_jit, jit_pointer_t f, size_t argc, jit_operand_t args[])
 {
-  prepare_args(_jit, argc, args);
+  size_t spill_size = prepare_call_args(_jit, argc, args);
 
   calli(_jit, (jit_word_t)f);
 
-  cleanup_stack_after_call(_jit, argc, args);
+  if (spill_size)
+    jit_addi(_jit, JIT_SP, JIT_SP, spill_size);
 }
 
 void
 jit_callr(jit_state_t *_jit, jit_gpr_t f, size_t argc, jit_operand_t args[])
 {
-  prepare_args(_jit, argc, args);
+  size_t spill_size = prepare_call_args(_jit, argc, args);
 
   callr(_jit, rn(f));
 
-  cleanup_stack_after_call(_jit, argc, args);
+  if (spill_size)
+    jit_addi(_jit, JIT_SP, JIT_SP, spill_size);
 }
 
 void
@@ -872,78 +967,17 @@ jit_locate_args(jit_state_t *_jit, size_t argc, jit_operand_t args[])
     next_abi_arg(&iter, &args[i]);
 }
 
-/* Precondition: args are distinct locations.  No JIT_OPERAND_KIND_MEM
-   element of args aliases stack-spilled args.  */
+/* Precondition: args are distinct locations of type GPR or FPR.  All
+   addends of arg operands are zero.  No GPR arg is SP.  */
 void
 jit_load_args(jit_state_t *_jit, size_t argc, jit_operand_t args[])
 {
-  jit_operand_t scratch[argc];
-
-  memcpy(scratch, args, sizeof(scratch[0]) * argc);
+  jit_operand_t src[argc];
 
-  jit_locate_args(_jit, argc, scratch);
-
-  /* First shuffle any arguments that are already in registers into
-     position.  */
-  for (size_t i = 0; i < argc; i++) {
-    switch (scratch[i].kind) {
-    case JIT_OPERAND_KIND_IMM:
-      abort();
-    case JIT_OPERAND_KIND_GPR:
-      switch (args[i].kind) {
-      case JIT_OPERAND_KIND_GPR:
-        shuffle_gpr_arg(_jit, args[i].loc.gpr, argc, scratch, i);
-        break;
-      case JIT_OPERAND_KIND_MEM:
-        store_mem_abi_arg(_jit, &scratch[i], args[i].loc.mem.base,
-                          args[i].loc.mem.offset);
-        break;
-      default:
-        abort();
-      }
-      break;
-    case JIT_OPERAND_KIND_FPR:
-      switch (args[i].kind) {
-      case JIT_OPERAND_KIND_FPR:
-        shuffle_fpr_arg(_jit, args[i].loc.fpr, argc, scratch, i);
-        break;
-      case JIT_OPERAND_KIND_MEM:
-        store_mem_abi_arg(_jit, &scratch[i], args[i].loc.mem.base,
-                          args[i].loc.mem.offset);
-        break;
-      default:
-        abort();
-      }
-      break;
-    case JIT_OPERAND_KIND_MEM:
-      break;
-    default:
-      abort();
-    }
-  }
+  memcpy(src, args, sizeof(src[0]) * argc);
 
-  /* Now shuffle spilled arguments from memory into place (memory or
-     registers).  */
-  for (size_t i = 0; i < argc; i++) {
-    if (args[i].kind == JIT_OPERAND_KIND_MEM) {
-      switch (args[i].kind)  {
-      case JIT_OPERAND_KIND_GPR:
-        abi_mem_to_gpr(_jit, args[i].kind, args[i].loc.gpr,
-                       scratch[i].loc.mem.base, scratch[i].loc.mem.offset);
-        break;
-      case JIT_OPERAND_KIND_FPR:
-        abi_mem_to_fpr(_jit, args[i].kind, args[i].loc.fpr,
-                       scratch[i].loc.mem.base, scratch[i].loc.mem.offset);
-        break;
-      case JIT_OPERAND_KIND_MEM:
-        store_mem_abi_arg(_jit, &scratch[i], args[i].loc.mem.base,
-                          args[i].loc.mem.offset);
-        break;
-      default:
-        abort();
-      }
-    }
-  }
+  jit_locate_args(_jit, argc, src);
+  jit_move_operands(_jit, args, src, argc);
 }
 
 void