summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMike Pall <mike>2017-02-20 03:43:10 +0100
committerMike Pall <mike>2017-02-20 03:43:10 +0100
commita25c0b99b84558887887b8e298409dcf8605e5e3 (patch)
tree8cb7b1db3cb0cd4f6cdd59540d39d986b502e471
parent4416e885d28c0f49d2c7bb3f9630ab23c22fbc9a (diff)
downloadluajit2-a25c0b99b84558887887b8e298409dcf8605e5e3.tar.gz
MIPS64, part 2: Add MIPS64 hard-float JIT compiler backend.
Contributed by Djordje Kovacevic and Stefan Pejic from RT-RK.com. Sponsored by Cisco Systems, Inc.
Diffstat
-rw-r--r--Makefile3
-rw-r--r--src/jit/dis_mips.lua45
-rw-r--r--src/jit/dis_mips64.lua17
-rw-r--r--src/jit/dis_mips64el.lua17
-rw-r--r--src/jit/dump.lua2
-rw-r--r--src/lj_arch.h4
-rw-r--r--src/lj_asm_mips.h673
-rw-r--r--src/lj_emit_mips.h107
-rw-r--r--src/lj_jit.h8
-rw-r--r--src/lj_mcode.c6
-rw-r--r--src/lj_snap.c3
-rw-r--r--src/lj_target_mips.h90
-rw-r--r--src/lj_trace.c11
-rw-r--r--src/vm_mips64.dasc237
14 files changed, 1024 insertions, 199 deletions
diff --git a/Makefile b/Makefile
index 02d649ea..489d7e75 100644
--- a/Makefile
+++ b/Makefile
@@ -87,7 +87,8 @@ FILE_PC= luajit.pc
FILES_INC= lua.h lualib.h lauxlib.h luaconf.h lua.hpp luajit.h
FILES_JITLIB= bc.lua bcsave.lua dump.lua p.lua v.lua zone.lua \
dis_x86.lua dis_x64.lua dis_arm.lua dis_arm64.lua \
- dis_ppc.lua dis_mips.lua dis_mipsel.lua vmdef.lua
+ dis_ppc.lua dis_mips.lua dis_mipsel.lua dis_mips64.lua \
+ dis_mips64el.lua vmdef.lua
ifeq (,$(findstring Windows,$(OS)))
HOST_SYS:= $(shell uname -s)
diff --git a/src/jit/dis_mips.lua b/src/jit/dis_mips.lua
index fdd3d353..a12b8e62 100644
--- a/src/jit/dis_mips.lua
+++ b/src/jit/dis_mips.lua
@@ -34,15 +34,17 @@ local map_special = {
"jrS", "jalrD1S", "movzDST", "movnDST",
"syscallY", "breakY", false, "sync",
"mfhiD", "mthiS", "mfloD", "mtloS",
- false, false, false, false,
+ "dsllvDST", false, "dsrlvDST", "dsravDST",
"multST", "multuST", "divST", "divuST",
- false, false, false, false,
+ "dmultST", "dmultuST", "ddivST", "ddivuST",
"addDST", "addu|moveDST0", "subDST", "subu|neguDS0T",
"andDST", "or|moveDST0", "xorDST", "nor|notDST0",
false, false, "sltDST", "sltuDST",
- false, false, false, false,
+ "daddDST", "dadduDST", "dsubDST", "dsubuDST",
"tgeSTZ", "tgeuSTZ", "tltSTZ", "tltuSTZ",
- "teqSTZ", false, "tneSTZ",
+ "teqSTZ", false, "tneSTZ", false,
+ "dsllDTA", false, "dsrlDTA", "dsraDTA",
+ "dsll32DTA", false, "dsrl32DTA", "dsra32DTA",
}
local map_special2 = {
@@ -60,11 +62,17 @@ local map_bshfl = {
[24] = "sehDT",
}
+local map_dbshfl = {
+ shift = 6, mask = 31,
+ [2] = "dsbhDT",
+ [5] = "dshdDT",
+}
+
local map_special3 = {
shift = 0, mask = 63,
- [0] = "extTSAK", [4] = "insTSAL",
- [32] = map_bshfl,
- [59] = "rdhwrTD",
+ [0] = "extTSAK", [1] = "dextmTSAP", [3] = "dextTSAK",
+ [4] = "insTSAL", [6] = "dinsuTSEQ", [7] = "dinsTSAL",
+ [32] = map_bshfl, [36] = map_dbshfl, [59] = "rdhwrTD",
}
local map_regimm = {
@@ -178,8 +186,8 @@ local map_cop1bc = {
local map_cop1 = {
shift = 21, mask = 31,
- [0] = "mfc1TG", false, "cfc1TG", "mfhc1TG",
- "mtc1TG", false, "ctc1TG", "mthc1TG",
+ [0] = "mfc1TG", "dmfc1TG", "cfc1TG", "mfhc1TG",
+ "mtc1TG", "dmtc1TG", "ctc1TG", "mthc1TG",
map_cop1bc, false, false, false,
false, false, false, false,
map_cop1s, map_cop1d, false, false,
@@ -213,16 +221,16 @@ local map_pri = {
"andiTSU", "ori|liTS0U", "xoriTSU", "luiTU",
map_cop0, map_cop1, false, map_cop1x,
"beql|beqzlST0B", "bnel|bnezlST0B", "blezlSB", "bgtzlSB",
- false, false, false, false,
+ "daddiTSI", "daddiuTSI", false, false,
map_special2, "jalxJ", false, map_special3,
"lbTSO", "lhTSO", "lwlTSO", "lwTSO",
"lbuTSO", "lhuTSO", "lwrTSO", false,
"sbTSO", "shTSO", "swlTSO", "swTSO",
false, false, "swrTSO", "cacheNSO",
"llTSO", "lwc1HSO", "lwc2TSO", "prefNSO",
- false, "ldc1HSO", "ldc2TSO", false,
+ false, "ldc1HSO", "ldc2TSO", "ldTSO",
"scTSO", "swc1HSO", "swc2TSO", false,
- false, "sdc1HSO", "sdc2TSO", false,
+ false, "sdc1HSO", "sdc2TSO", "sdTSO",
}
------------------------------------------------------------------------------
@@ -306,6 +314,8 @@ local function disass_ins(ctx)
x = "f"..band(rshift(op, 21), 31)
elseif p == "A" then
x = band(rshift(op, 6), 31)
+ elseif p == "E" then
+ x = band(rshift(op, 6), 31) + 32
elseif p == "M" then
x = band(rshift(op, 11), 31)
elseif p == "N" then
@@ -315,8 +325,12 @@ local function disass_ins(ctx)
if x == 0 then x = nil end
elseif p == "K" then
x = band(rshift(op, 11), 31) + 1
+ elseif p == "P" then
+ x = band(rshift(op, 11), 31) + 33
elseif p == "L" then
x = band(rshift(op, 11), 31) - last + 1
+ elseif p == "Q" then
+ x = band(rshift(op, 11), 31) - last + 33
elseif p == "I" then
x = arshift(lshift(op, 16), 16)
elseif p == "U" then
@@ -330,11 +344,12 @@ local function disass_ins(ctx)
elseif p == "B" then
x = ctx.addr + ctx.pos + arshift(lshift(op, 16), 16)*4 + 4
ctx.rel = x
- x = "0x"..tohex(x)
+ x = format("0x%08x", x)
elseif p == "J" then
- x = band(ctx.addr + ctx.pos, 0xf0000000) + band(op, 0x03ffffff)*4
+ local a = ctx.addr + ctx.pos
+ x = a - band(a, 0x0fffffff) + band(op, 0x03ffffff)*4
ctx.rel = x
- x = "0x"..tohex(x)
+ x = format("0x%08x", x)
elseif p == "V" then
x = band(rshift(op, 8), 7)
if x == 0 then x = nil end
diff --git a/src/jit/dis_mips64.lua b/src/jit/dis_mips64.lua
new file mode 100644
index 00000000..c4374928
--- /dev/null
+++ b/src/jit/dis_mips64.lua
@@ -0,0 +1,17 @@
+----------------------------------------------------------------------------
+-- LuaJIT MIPS64 disassembler wrapper module.
+--
+-- Copyright (C) 2005-2017 Mike Pall. All rights reserved.
+-- Released under the MIT license. See Copyright Notice in luajit.h
+----------------------------------------------------------------------------
+-- This module just exports the big-endian functions from the
+-- MIPS disassembler module. All the interesting stuff is there.
+------------------------------------------------------------------------------
+
+local dis_mips = require((string.match(..., ".*%.") or "").."dis_mips")
+return {
+ create = dis_mips.create,
+ disass = dis_mips.disass,
+ regname = dis_mips.regname
+}
+
diff --git a/src/jit/dis_mips64el.lua b/src/jit/dis_mips64el.lua
new file mode 100644
index 00000000..2b1470af
--- /dev/null
+++ b/src/jit/dis_mips64el.lua
@@ -0,0 +1,17 @@
+----------------------------------------------------------------------------
+-- LuaJIT MIPS64EL disassembler wrapper module.
+--
+-- Copyright (C) 2005-2017 Mike Pall. All rights reserved.
+-- Released under the MIT license. See Copyright Notice in luajit.h
+----------------------------------------------------------------------------
+-- This module just exports the little-endian functions from the
+-- MIPS disassembler module. All the interesting stuff is there.
+------------------------------------------------------------------------------
+
+local dis_mips = require((string.match(..., ".*%.") or "").."dis_mips")
+return {
+ create = dis_mips.create_el,
+ disass = dis_mips.disass_el,
+ regname = dis_mips.regname
+}
+
diff --git a/src/jit/dump.lua b/src/jit/dump.lua
index 898ce9a1..2bea652b 100644
--- a/src/jit/dump.lua
+++ b/src/jit/dump.lua
@@ -85,7 +85,7 @@ local nexitsym = 0
local function fillsymtab_tr(tr, nexit)
local t = {}
symtabmt.__index = t
- if jit.arch == "mips" or jit.arch == "mipsel" then
+ if jit.arch:sub(1, 4) == "mips" then
t[traceexitstub(tr, 0)] = "exit"
return
end
diff --git a/src/lj_arch.h b/src/lj_arch.h
index 001111da..9bf6f481 100644
--- a/src/lj_arch.h
+++ b/src/lj_arch.h
@@ -332,10 +332,12 @@
#define LJ_ARCH_BITS 32
#define LJ_TARGET_MIPS32 1
#else
+#if LJ_ABI_SOFTFP || !LJ_ARCH_HASFPU
+#define LJ_ARCH_NOJIT 1 /* NYI */
+#endif
#define LJ_ARCH_BITS 64
#define LJ_TARGET_MIPS64 1
#define LJ_TARGET_GC64 1
-#define LJ_ARCH_NOJIT 1 /* NYI */
#endif
#define LJ_TARGET_MIPS 1
#define LJ_TARGET_EHRETREG 4
diff --git a/src/lj_asm_mips.h b/src/lj_asm_mips.h
index dd821c70..affe7d89 100644
--- a/src/lj_asm_mips.h
+++ b/src/lj_asm_mips.h
@@ -23,7 +23,7 @@ static Reg ra_alloc1z(ASMState *as, IRRef ref, RegSet allow)
{
Reg r = IR(ref)->r;
if (ra_noreg(r)) {
- if (!(allow & RSET_FPR) && irref_isk(ref) && IR(ref)->i == 0)
+ if (!(allow & RSET_FPR) && irref_isk(ref) && get_kval(IR(ref)) == 0)
return RID_ZERO;
r = ra_allocref(as, ref, allow);
} else {
@@ -166,9 +166,9 @@ static Reg asm_fuseahuref(ASMState *as, IRRef ref, int32_t *ofsp, RegSet allow)
} else if (ir->o == IR_UREFC) {
if (irref_isk(ir->op1)) {
GCfunc *fn = ir_kfunc(IR(ir->op1));
- int32_t ofs = i32ptr(&gcref(fn->l.uvptr[(ir->op2 >> 8)])->uv.tv);
- int32_t jgl = (intptr_t)J2G(as->J);
- if ((uint32_t)(ofs-jgl) < 65536) {
+ intptr_t ofs = (intptr_t)&gcref(fn->l.uvptr[(ir->op2 >> 8)])->uv.tv;
+ intptr_t jgl = (intptr_t)J2G(as->J);
+ if ((uintptr_t)(ofs-jgl) < 65536) {
*ofsp = ofs-jgl-32768;
return RID_JGL;
} else {
@@ -190,20 +190,21 @@ static void asm_fusexref(ASMState *as, MIPSIns mi, Reg rt, IRRef ref,
Reg base;
if (ra_noreg(ir->r) && canfuse(as, ir)) {
if (ir->o == IR_ADD) {
- int32_t ofs2;
- if (irref_isk(ir->op2) && (ofs2 = ofs + IR(ir->op2)->i, checki16(ofs2))) {
+ intptr_t ofs2;
+ if (irref_isk(ir->op2) && (ofs2 = ofs + get_kval(IR(ir->op2)),
+ checki16(ofs2))) {
ref = ir->op1;
- ofs = ofs2;
+ ofs = (int32_t)ofs2;
}
} else if (ir->o == IR_STRREF) {
- int32_t ofs2 = 65536;
+ intptr_t ofs2 = 65536;
lua_assert(ofs == 0);
ofs = (int32_t)sizeof(GCstr);
if (irref_isk(ir->op2)) {
- ofs2 = ofs + IR(ir->op2)->i;
+ ofs2 = ofs + get_kval(IR(ir->op2));
ref = ir->op1;
} else if (irref_isk(ir->op1)) {
- ofs2 = ofs + IR(ir->op1)->i;
+ ofs2 = ofs + get_kval(IR(ir->op1));
ref = ir->op2;
}
if (!checki16(ofs2)) {
@@ -211,7 +212,7 @@ static void asm_fusexref(ASMState *as, MIPSIns mi, Reg rt, IRRef ref,
Reg right, left = ra_alloc2(as, ir, allow);
right = (left >> 8); left &= 255;
emit_hsi(as, mi, rt, RID_TMP, ofs);
- emit_dst(as, MIPSI_ADDU, RID_TMP, left, right);
+ emit_dst(as, MIPSI_AADDU, RID_TMP, left, right);
return;
}
ofs = ofs2;
@@ -227,7 +228,7 @@ static void asm_fusexref(ASMState *as, MIPSIns mi, Reg rt, IRRef ref,
static void asm_gencall(ASMState *as, const CCallInfo *ci, IRRef *args)
{
uint32_t n, nargs = CCI_XNARGS(ci);
- int32_t ofs = 16;
+ int32_t ofs = LJ_32 ? 16 : 0;
#if LJ_SOFTFP
Reg gpr = REGARG_FIRSTGPR;
#else
@@ -249,15 +250,15 @@ static void asm_gencall(ASMState *as, const CCallInfo *ci, IRRef *args)
!(ci->flags & CCI_VARARG)) {
lua_assert(rset_test(as->freeset, fpr)); /* Already evicted. */
ra_leftov(as, fpr, ref);
- fpr += 2;
- gpr += irt_isnum(ir->t) ? 2 : 1;
+ fpr += LJ_32 ? 2 : 1;
+ gpr += (LJ_32 && irt_isnum(ir->t)) ? 2 : 1;
} else
#endif
{
-#if !LJ_SOFTFP
+#if LJ_32 && !LJ_SOFTFP
fpr = REGARG_LASTFPR+1;
#endif
- if (irt_isnum(ir->t)) gpr = (gpr+1) & ~1;
+ if (LJ_32 && irt_isnum(ir->t)) gpr = (gpr+1) & ~1;
if (gpr <= REGARG_LASTGPR) {
lua_assert(rset_test(as->freeset, gpr)); /* Already evicted. */
#if !LJ_SOFTFP
@@ -269,35 +270,55 @@ static void asm_gencall(ASMState *as, const CCallInfo *ci, IRRef *args)
r = ra_alloc1(as, ref, RSET_FPR);
as->freeset |= (of & RSET_RANGE(REGARG_FIRSTGPR, REGARG_LASTGPR+1));
if (irt_isnum(ir->t)) {
+#if LJ_32
emit_tg(as, MIPSI_MFC1, gpr+(LJ_BE?0:1), r+1);
emit_tg(as, MIPSI_MFC1, gpr+(LJ_BE?1:0), r);
lua_assert(rset_test(as->freeset, gpr+1)); /* Already evicted. */
gpr += 2;
+#else
+ emit_tg(as, MIPSI_DMFC1, gpr, r);
+ gpr++; fpr++;
+#endif
} else if (irt_isfloat(ir->t)) {
emit_tg(as, MIPSI_MFC1, gpr, r);
gpr++;
+#if LJ_64
+ fpr++;
+#endif
}
} else
#endif
{
ra_leftov(as, gpr, ref);
gpr++;
+#if LJ_64
+ fpr++;
+#endif
}
} else {
Reg r = ra_alloc1z(as, ref, !LJ_SOFTFP && irt_isfp(ir->t) ? RSET_FPR : RSET_GPR);
+#if LJ_32
if (irt_isnum(ir->t)) ofs = (ofs + 4) & ~4;
emit_spstore(as, ir, r, ofs);
ofs += irt_isnum(ir->t) ? 8 : 4;
+#else
+ emit_spstore(as, ir, r, ofs + ((LJ_BE && (LJ_SOFTFP || r < RID_MAX_GPR) && !irt_is64(ir->t)) ? 4 : 0));
+ ofs += 8;
+#endif
}
}
} else {
#if !LJ_SOFTFP
fpr = REGARG_LASTFPR+1;
#endif
- if (gpr <= REGARG_LASTGPR)
+ if (gpr <= REGARG_LASTGPR) {
gpr++;
- else
- ofs += 4;
+#if LJ_64
+ fpr++;
+#endif
+ } else {
+ ofs += LJ_32 ? 4 : 8;
+ }
}
checkmclim(as);
}
@@ -307,15 +328,19 @@ static void asm_gencall(ASMState *as, const CCallInfo *ci, IRRef *args)
static void asm_setupresult(ASMState *as, IRIns *ir, const CCallInfo *ci)
{
RegSet drop = RSET_SCRATCH;
+#if LJ_32
int hiop = ((ir+1)->o == IR_HIOP && !irt_isnil((ir+1)->t));
+#endif
#if !LJ_SOFTFP
if ((ci->flags & CCI_NOFPRCLOBBER))
drop &= ~RSET_FPR;
#endif
if (ra_hasreg(ir->r))
rset_clear(drop, ir->r); /* Dest reg handled below. */
+#if LJ_32
if (hiop && ra_hasreg((ir+1)->r))
rset_clear(drop, (ir+1)->r); /* Dest reg handled below. */
+#endif
ra_evictset(as, drop); /* Evictions must be performed first. */
if (ra_used(ir)) {
lua_assert(!irt_ispri(ir->t));
@@ -326,18 +351,28 @@ static void asm_setupresult(ASMState *as, IRIns *ir, const CCallInfo *ci)
if (ra_hasreg(dest)) {
ra_free(as, dest);
ra_modified(as, dest);
+#if LJ_32
emit_tg(as, MIPSI_MTC1, RID_RETHI, dest+1);
emit_tg(as, MIPSI_MTC1, RID_RETLO, dest);
+#else
+ emit_tg(as, MIPSI_DMTC1, RID_RET, dest);
+#endif
}
if (ofs) {
+#if LJ_32
emit_tsi(as, MIPSI_SW, RID_RETLO, RID_SP, ofs+(LJ_BE?4:0));
emit_tsi(as, MIPSI_SW, RID_RETHI, RID_SP, ofs+(LJ_BE?0:4));
+#else
+ emit_tsi(as, MIPSI_SD, RID_RET, RID_SP, ofs);
+#endif
}
} else {
ra_destreg(as, ir, RID_FPRET);
}
+#if LJ_32
} else if (hiop) {
ra_destpair(as, ir);
+#endif
} else {
ra_destreg(as, ir, RID_RET);
}
@@ -356,7 +391,7 @@ static void asm_callx(ASMState *as, IRIns *ir)
func = ir->op2; irf = IR(func);
if (irf->o == IR_CARG) { func = irf->op1; irf = IR(func); }
if (irref_isk(func)) { /* Call to constant address. */
- ci.func = (ASMFunction)(void *)(irf->i);
+ ci.func = (ASMFunction)(void *)get_kval(irf);
} else { /* Need specific register for indirect calls. */
Reg r = ra_alloc1(as, func, RID2RSET(RID_CFUNCADDR));
MCode *p = as->mcp;
@@ -399,8 +434,8 @@ static void asm_retf(ASMState *as, IRIns *ir)
emit_setgl(as, base, jit_base);
emit_addptr(as, base, -8*delta);
asm_guard(as, MIPSI_BNE, RID_TMP,
- ra_allock(as, i32ptr(pc), rset_exclude(RSET_GPR, base)));
- emit_tsi(as, MIPSI_LW, RID_TMP, base, -8);
+ ra_allock(as, igcptr(pc), rset_exclude(RSET_GPR, base)));
+ emit_tsi(as, MIPSI_AL, RID_TMP, base, -8);
}
/* -- Type conversions ---------------------------------------------------- */
@@ -435,10 +470,15 @@ static void asm_conv(ASMState *as, IRIns *ir)
#if !LJ_SOFTFP
int stfp = (st == IRT_NUM || st == IRT_FLOAT);
#endif
+#if LJ_64
+ int st64 = (st == IRT_I64 || st == IRT_U64 || st == IRT_P64);
+#endif
IRRef lref = ir->op1;
+#if LJ_32
lua_assert(!(irt_isint64(ir->t) ||
(st == IRT_I64 || st == IRT_U64))); /* Handled by SPLIT. */
-#if LJ_SOFTFP
+#endif
+#if LJ_32 && LJ_SOFTFP
/* FP conversions are handled by SPLIT. */
lua_assert(!irt_isfp(ir->t) && !(st == IRT_NUM || st == IRT_FLOAT));
/* Can't check for same types: SPLIT uses CONV int.int + BXOR for sfp NEG. */
@@ -463,11 +503,40 @@ static void asm_conv(ASMState *as, IRIns *ir)
emit_tg(as, MIPSI_MTC1, RID_TMP, dest);
emit_dst(as, MIPSI_XOR, RID_TMP, RID_TMP, left);
emit_ti(as, MIPSI_LUI, RID_TMP, 0x8000);
+#if LJ_64
+ } else if(st == IRT_U64) { /* U64 to FP conversion. */
+ /* if (x >= 1u<<63) y = (double)(int64_t)(x&(1u<<63)-1) + pow(2.0, 63) */
+ Reg left = ra_alloc1(as, lref, RSET_GPR);
+ Reg tmp = ra_scratch(as, rset_exclude(RSET_FPR, dest));
+ MCLabel l_end = emit_label(as);
+ if (irt_isfloat(ir->t)) {
+ emit_fgh(as, MIPSI_ADD_S, dest, dest, tmp);
+ emit_lsptr(as, MIPSI_LWC1, (tmp & 31), (void *)&as->J->k32[LJ_K32_2P63],
+ rset_exclude(RSET_GPR, left));
+ emit_fg(as, MIPSI_CVT_S_L, dest, dest);
+ } else {
+ emit_fgh(as, MIPSI_ADD_D, dest, dest, tmp);
+ emit_lsptr(as, MIPSI_LDC1, (tmp & 31), (void *)&as->J->k64[LJ_K64_2P63],
+ rset_exclude(RSET_GPR, left));
+ emit_fg(as, MIPSI_CVT_D_L, dest, dest);
+ }
+ emit_branch(as, MIPSI_BGEZ, left, RID_ZERO, l_end);
+ emit_tg(as, MIPSI_DMTC1, RID_TMP, dest);
+ emit_tsml(as, MIPSI_DEXTM, RID_TMP, left, 30, 0);
+#endif
} else { /* Integer to FP conversion. */
Reg left = ra_alloc1(as, lref, RSET_GPR);
+#if LJ_32
emit_fg(as, irt_isfloat(ir->t) ? MIPSI_CVT_S_W : MIPSI_CVT_D_W,
dest, dest);
emit_tg(as, MIPSI_MTC1, left, dest);
+#else
+ MIPSIns mi = irt_isfloat(ir->t) ?
+ (st64 ? MIPSI_CVT_S_L : MIPSI_CVT_S_W) :
+ (st64 ? MIPSI_CVT_D_L : MIPSI_CVT_D_W);
+ emit_fg(as, mi, dest, dest);
+ emit_tg(as, st64 ? MIPSI_DMTC1 : MIPSI_MTC1, left, dest);
+#endif
}
} else if (stfp) { /* FP to integer conversion. */
if (irt_isguard(ir->t)) {
@@ -478,7 +547,7 @@ static void asm_conv(ASMState *as, IRIns *ir)
Reg dest = ra_dest(as, ir, RSET_GPR);
Reg left = ra_alloc1(as, lref, RSET_FPR);
Reg tmp = ra_scratch(as, rset_exclude(RSET_FPR, left));
- if (irt_isu32(ir->t)) {
+ if (irt_isu32(ir->t)) { /* FP to U32 conversion. */
/* y = (int)floor(x - 2147483648.0) ^ 0x80000000 */
emit_dst(as, MIPSI_XOR, dest, dest, RID_TMP);
emit_ti(as, MIPSI_LUI, RID_TMP, 0x8000);
@@ -493,10 +562,50 @@ static void asm_conv(ASMState *as, IRIns *ir)
else
emit_lsptr(as, MIPSI_LDC1, (tmp & 31),
(void *)&as->J->k64[LJ_K64_2P31], RSET_GPR);
+#if LJ_64
+ } else if (irt_isu64(ir->t)) { /* FP to U64 conversion. */
+ MCLabel l_end;
+ emit_tg(as, MIPSI_DMFC1, dest, tmp);
+ l_end = emit_label(as);
+ /* For inputs >= 2^63 add -2^64 and convert again. */
+ if (st == IRT_NUM) {
+ emit_fg(as, MIPSI_TRUNC_L_D, tmp, tmp);
+ emit_fgh(as, MIPSI_ADD_D, tmp, left, tmp);
+ emit_lsptr(as, MIPSI_LDC1, (tmp & 31),
+ (void *)&as->J->k64[LJ_K64_M2P64],
+ rset_exclude(RSET_GPR, dest));
+ emit_fg(as, MIPSI_TRUNC_L_D, tmp, left); /* Delay slot. */
+ emit_branch(as, MIPSI_BC1T, 0, 0, l_end);
+ emit_fgh(as, MIPSI_C_OLT_D, 0, left, tmp);
+ emit_lsptr(as, MIPSI_LDC1, (tmp & 31),
+ (void *)&as->J->k64[LJ_K64_2P63],
+ rset_exclude(RSET_GPR, dest));
+ } else {
+ emit_fg(as, MIPSI_TRUNC_L_S, tmp, tmp);
+ emit_fgh(as, MIPSI_ADD_S, tmp, left, tmp);
+ emit_lsptr(as, MIPSI_LWC1, (tmp & 31),
+ (void *)&as->J->k32[LJ_K32_M2P64],
+ rset_exclude(RSET_GPR, dest));
+ emit_fg(as, MIPSI_TRUNC_L_S, tmp, left); /* Delay slot. */
+ emit_branch(as, MIPSI_BC1T, 0, 0, l_end);
+ emit_fgh(as, MIPSI_C_OLT_S, 0, left, tmp);
+ emit_lsptr(as, MIPSI_LWC1, (tmp & 31),
+ (void *)&as->J->k32[LJ_K32_2P63],
+ rset_exclude(RSET_GPR, dest));
+ }
+#endif
} else {
+#if LJ_32
emit_tg(as, MIPSI_MFC1, dest, tmp);
emit_fg(as, st == IRT_FLOAT ? MIPSI_TRUNC_W_S : MIPSI_TRUNC_W_D,
tmp, left);
+#else
+ MIPSIns mi = irt_is64(ir->t) ?
+ (st == IRT_NUM ? MIPSI_TRUNC_L_D : MIPSI_TRUNC_L_S) :
+ (st == IRT_NUM ? MIPSI_TRUNC_W_D : MIPSI_TRUNC_W_S);
+ emit_tg(as, irt_is64(ir->t) ? MIPSI_DMFC1 : MIPSI_MFC1, dest, left);
+ emit_fg(as, mi, left, left);
+#endif
}
}
} else
@@ -507,7 +616,7 @@ static void asm_conv(ASMState *as, IRIns *ir)
Reg left = ra_alloc1(as, ir->op1, RSET_GPR);
lua_assert(irt_isint(ir->t) || irt_isu32(ir->t));
if ((ir->op2 & IRCONV_SEXT)) {
- if ((as->flags & JIT_F_MIPSXXR2)) {
+ if (LJ_64 || (as->flags & JIT_F_MIPSXXR2)) {
emit_dst(as, st == IRT_I8 ? MIPSI_SEB : MIPSI_SEH, dest, 0, left);
} else {
uint32_t shift = st == IRT_I8 ? 24 : 16;
@@ -519,8 +628,35 @@ static void asm_conv(ASMState *as, IRIns *ir)
(int32_t)(st == IRT_U8 ? 0xff : 0xffff));
}
} else { /* 32/64 bit integer conversions. */
+#if LJ_32
/* Only need to handle 32/32 bit no-op (cast) on 32 bit archs. */
ra_leftov(as, dest, lref); /* Do nothing, but may need to move regs. */
+#else
+ if (irt_is64(ir->t)) {
+ if (st64) {
+ /* 64/64 bit no-op (cast)*/
+ ra_leftov(as, dest, lref);
+ } else {
+ Reg left = ra_alloc1(as, lref, RSET_GPR);
+ if ((ir->op2 & IRCONV_SEXT)) { /* 32 to 64 bit sign extension. */
+ emit_dta(as, MIPSI_SLL, dest, left, 0);
+ } else { /* 32 to 64 bit zero extension. */
+ emit_tsml(as, MIPSI_DEXT, dest, left, 31, 0);
+ }
+ }
+ } else {
+ if (st64) {
+ /* This is either a 32 bit reg/reg mov which zeroes the hiword
+ ** or a load of the loword from a 64 bit address.
+ */
+ Reg left = ra_alloc1(as, lref, RSET_GPR);
+ emit_tsml(as, MIPSI_DEXT, dest, left, 31, 0);
+ } else { /* 32/32 bit no-op (cast). */
+ /* Do nothing, but may need to move regs. */
+ ra_leftov(as, dest, lref);
+ }
+ }
+#endif
}
}
}
@@ -563,23 +699,50 @@ static void asm_strto(ASMState *as, IRIns *ir)
args[1] = ASMREF_TMP1; /* TValue *n */
asm_gencall(as, ci, args);
/* Store the result to the spill slot or temp slots. */
- emit_tsi(as, MIPSI_ADDIU, ra_releasetmp(as, ASMREF_TMP1),
+ emit_tsi(as, MIPSI_AADDIU, ra_releasetmp(as, ASMREF_TMP1),
RID_SP, ofs);
}
/* -- Memory references --------------------------------------------------- */
+#if LJ_64
+/* Store tagged value for ref at base+ofs. */
+static void asm_tvstore64(ASMState *as, Reg base, int32_t ofs, IRRef ref)
+{
+ RegSet allow = rset_exclude(RSET_GPR, base);
+ IRIns *ir = IR(ref);
+ lua_assert(irt_ispri(ir->t) || irt_isaddr(ir->t) || irt_isinteger(ir->t));
+ if (irref_isk(ref)) {
+ TValue k;
+ lj_ir_kvalue(as->J->L, &k, ir);
+ emit_tsi(as, MIPSI_SD, ra_allock(as, (int64_t)k.u64, allow), base, ofs);
+ } else {
+ Reg src = ra_alloc1(as, ref, allow);
+ Reg type = ra_allock(as, (int64_t)irt_toitype(ir->t) << 47,
+ rset_exclude(allow, src));
+ emit_tsi(as, MIPSI_SD, RID_TMP, base, ofs);
+ if (irt_isinteger(ir->t)) {
+ emit_dst(as, MIPSI_DADDU, RID_TMP, RID_TMP, type);
+ emit_tsml(as, MIPSI_DEXT, RID_TMP, src, 31, 0);
+ } else {
+ emit_dst(as, MIPSI_DADDU, RID_TMP, src, type);
+ }
+ }
+}
+#endif
+
/* Get pointer to TValue. */
static void asm_tvptr(ASMState *as, Reg dest, IRRef ref)
{
IRIns *ir = IR(ref);
if (irt_isnum(ir->t)) {
if (irref_isk(ref)) /* Use the number constant itself as a TValue. */
- ra_allockreg(as, i32ptr(ir_knum(ir)), dest);
+ ra_allockreg(as, igcptr(ir_knum(ir)), dest);
else /* Otherwise force a spill and use the spill slot. */
- emit_tsi(as, MIPSI_ADDIU, dest, RID_SP, ra_spill(as, ir));
+ emit_tsi(as, MIPSI_AADDIU, dest, RID_SP, ra_spill(as, ir));
} else {
/* Otherwise use g->tmptv to hold the TValue. */
+#if LJ_32
RegSet allow = rset_exclude(RSET_GPR, dest);
Reg type;
emit_tsi(as, MIPSI_ADDIU, dest, RID_JGL, (int32_t)(offsetof(global_State, tmptv)-32768));
@@ -592,6 +755,11 @@ static void asm_tvptr(ASMState *as, Reg dest, IRRef ref)
else
type = ra_allock(as, (int32_t)irt_toitype(ir->t), allow);
emit_setgl(as, type, tmptv.it);
+#else
+ asm_tvstore64(as, dest, 0, ref);
+ emit_tsi(as, MIPSI_DADDIU, dest, RID_JGL,
+ (int32_t)(offsetof(global_State, tmptv)-32768));
+#endif
}
}
@@ -606,13 +774,13 @@ static void asm_aref(ASMState *as, IRIns *ir)
ofs += 8*IR(ir->op2)->i;
if (checki16(ofs)) {
base = ra_alloc1(as, refa, RSET_GPR);
- emit_tsi(as, MIPSI_ADDIU, dest, base, ofs);
+ emit_tsi(as, MIPSI_AADDIU, dest, base, ofs);
return;
}
}
base = ra_alloc1(as, ir->op1, RSET_GPR);
idx = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, base));
- emit_dst(as, MIPSI_ADDU, dest, RID_TMP, base);
+ emit_dst(as, MIPSI_AADDU, dest, RID_TMP, base);
emit_dta(as, MIPSI_SLL, RID_TMP, idx, 3);
}
@@ -633,13 +801,14 @@ static void asm_href(ASMState *as, IRIns *ir, IROp merge)
Reg key = RID_NONE, type = RID_NONE, tmpnum = RID_NONE, tmp1 = RID_TMP, tmp2;
IRRef refkey = ir->op2;
IRIns *irkey = IR(refkey);
+ int isk = irref_isk(refkey);
IRType1 kt = irkey->t;
uint32_t khash;
MCLabel l_end, l_loop, l_next;
rset_clear(allow, tab);
-#if LJ_SOFTFP
- if (!irref_isk(refkey)) {
+#if LJ_32 && LJ_SOFTFP
+ if (!isk) {
key = ra_alloc1(as, refkey, allow);
rset_clear(allow, key);
if (irkey[1].o == IR_HIOP) {
@@ -664,8 +833,10 @@ static void asm_href(ASMState *as, IRIns *ir, IROp merge)
} else if (!irt_ispri(kt)) {
key = ra_alloc1(as, refkey, allow);
rset_clear(allow, key);
+#if LJ_32
type = ra_allock(as, (int32_t)irt_toitype(irkey->t), allow);
rset_clear(allow, type);
+#endif
}
#endif
tmp2 = ra_scratch(as, allow);
@@ -679,9 +850,9 @@ static void asm_href(ASMState *as, IRIns *ir, IROp merge)
else if (destused)
emit_loada(as, dest, niltvg(J2G(as->J)));
/* Follow hash chain until the end. */
- emit_move(as, dest, tmp2);
+ emit_move(as, dest, tmp1);
l_loop = --as->mcp;
- emit_tsi(as, MIPSI_LW, tmp2, dest, (int32_t)offsetof(Node, next));
+ emit_tsi(as, MIPSI_AL, tmp1, dest, (int32_t)offsetof(Node, next));
l_next = emit_label(as);
/* Type and value comparison. */
@@ -693,38 +864,66 @@ static void asm_href(ASMState *as, IRIns *ir, IROp merge)
emit_branch(as, MIPSI_BC1T, 0, 0, l_end);
emit_fgh(as, MIPSI_C_EQ_D, 0, tmpnum, key);
*--as->mcp = MIPSI_NOP; /* Avoid NaN comparison overhead. */
- emit_branch(as, MIPSI_BEQ, tmp2, RID_ZERO, l_next);
- emit_tsi(as, MIPSI_SLTIU, tmp2, tmp2, (int32_t)LJ_TISNUM);
+ emit_branch(as, MIPSI_BEQ, tmp1, RID_ZERO, l_next);
+ emit_tsi(as, MIPSI_SLTIU, tmp1, tmp1, (int32_t)LJ_TISNUM);
+#if LJ_32
emit_hsi(as, MIPSI_LDC1, tmpnum, dest, (int32_t)offsetof(Node, key.n));
} else {
if (irt_ispri(kt)) {
- emit_branch(as, MIPSI_BEQ, tmp2, type, l_end);
+ emit_branch(as, MIPSI_BEQ, tmp1, type, l_end);
} else {
- emit_branch(as, MIPSI_BEQ, tmp1, key, l_end);
- emit_tsi(as, MIPSI_LW, tmp1, dest, (int32_t)offsetof(Node, key.gcr));
- emit_branch(as, MIPSI_BNE, tmp2, type, l_next);
+ emit_branch(as, MIPSI_BEQ, tmp2, key, l_end);
+ emit_tsi(as, MIPSI_LW, tmp2, dest, (int32_t)offsetof(Node, key.gcr));
+ emit_branch(as, MIPSI_BNE, tmp1, type, l_next);
+ }
+ }
+ emit_tsi(as, MIPSI_LW, tmp1, dest, (int32_t)offsetof(Node, key.it));
+ *l_loop = MIPSI_BNE | MIPSF_S(tmp1) | ((as->mcp-l_loop-1) & 0xffffu);
+#else
+ emit_dta(as, MIPSI_DSRA32, tmp1, tmp1, 15);
+ emit_tg(as, MIPSI_DMTC1, tmp1, tmpnum);
+ emit_tsi(as, MIPSI_LD, tmp1, dest, (int32_t)offsetof(Node, key.u64));
+ } else if (irt_isaddr(kt)) {
+ Reg refk = tmp2;
+ if (isk) {
+ int64_t k = ((int64_t)irt_toitype(irkey->t) << 47) | irkey[1].tv.u64;
+ refk = ra_allock(as, k, allow);
+ rset_clear(allow, refk);
}
+ emit_branch(as, MIPSI_BEQ, tmp1, refk, l_end);
+ emit_tsi(as, MIPSI_LD, tmp1, dest, offsetof(Node, key));
+ } else {
+ Reg pri = ra_allock(as, ~((int64_t)~irt_toitype(ir->t) << 47), allow);
+ rset_clear(allow, pri);
+ lua_assert(irt_ispri(kt) && !irt_isnil(kt));
+ emit_branch(as, MIPSI_BEQ, tmp1, pri, l_end);
+ emit_tsi(as, MIPSI_LD, tmp1, dest, offsetof(Node, key));
+ }
+ *l_loop = MIPSI_BNE | MIPSF_S(tmp1) | ((as->mcp-l_loop-1) & 0xffffu);
+ if (!isk && irt_isaddr(kt)) {
+ type = ra_allock(as, (int64_t)irt_toitype(kt) << 47, allow);
+ emit_dst(as, MIPSI_DADDU, tmp2, key, type);
+ rset_clear(allow, type);
}
- emit_tsi(as, MIPSI_LW, tmp2, dest, (int32_t)offsetof(Node, key.it));
- *l_loop = MIPSI_BNE | MIPSF_S(tmp2) | ((as->mcp-l_loop-1) & 0xffffu);
+#endif
/* Load main position relative to tab->node into dest. */
- khash = irref_isk(refkey) ? ir_khash(irkey) : 1;
+ khash = isk ? ir_khash(irkey) : 1;
if (khash == 0) {
- emit_tsi(as, MIPSI_LW, dest, tab, (int32_t)offsetof(GCtab, node));
+ emit_tsi(as, MIPSI_AL, dest, tab, (int32_t)offsetof(GCtab, node));
} else {
Reg tmphash = tmp1;
- if (irref_isk(refkey))
+ if (isk)
tmphash = ra_allock(as, khash, allow);
- emit_dst(as, MIPSI_ADDU, dest, dest, tmp1);
+ emit_dst(as, MIPSI_AADDU, dest, dest, tmp1);
lua_assert(sizeof(Node) == 24);
emit_dst(as, MIPSI_SUBU, tmp1, tmp2, tmp1);
emit_dta(as, MIPSI_SLL, tmp1, tmp1, 3);
emit_dta(as, MIPSI_SLL, tmp2, tmp1, 5);
emit_dst(as, MIPSI_AND, tmp1, tmp2, tmphash);
- emit_tsi(as, MIPSI_LW, dest, tab, (int32_t)offsetof(GCtab, node));
+ emit_tsi(as, MIPSI_AL, dest, tab, (int32_t)offsetof(GCtab, node));
emit_tsi(as, MIPSI_LW, tmp2, tab, (int32_t)offsetof(GCtab, hmask));
- if (irref_isk(refkey)) {
+ if (isk) {
/* Nothing to do. */
} else if (irt_isstr(kt)) {
emit_tsi(as, MIPSI_LW, tmp1, key, (int32_t)offsetof(GCstr, hash));
@@ -734,6 +933,7 @@ static void asm_href(ASMState *as, IRIns *ir, IROp merge)
emit_dst(as, MIPSI_XOR, tmp1, tmp1, tmp2);
emit_rotr(as, tmp1, tmp1, dest, (-HASH_ROT2-HASH_ROT1)&31);
emit_dst(as, MIPSI_SUBU, tmp2, tmp2, dest);
+#if LJ_32
if (LJ_SOFTFP ? (irkey[1].o == IR_HIOP) : irt_isnum(kt)) {
emit_dst(as, MIPSI_XOR, tmp2, tmp2, tmp1);
if ((as->flags & JIT_F_MIPSXXR2)) {
@@ -756,6 +956,23 @@ static void asm_href(ASMState *as, IRIns *ir, IROp merge)
emit_rotr(as, dest, tmp1, tmp2, (-HASH_ROT1)&31);
emit_dst(as, MIPSI_ADDU, tmp1, key, ra_allock(as, HASH_BIAS, allow));
}
+#else
+ emit_dst(as, MIPSI_XOR, tmp2, tmp2, tmp1);
+ emit_dta(as, MIPSI_ROTR, dest, tmp1, (-HASH_ROT1)&31);
+ if (irt_isnum(kt)) {
+ emit_dst(as, MIPSI_ADDU, tmp1, tmp1, tmp1);
+ emit_dta(as, MIPSI_DSRA32, tmp1, tmp1, 0);
+ emit_dta(as, MIPSI_SLL, tmp2, LJ_SOFTFP ? key : tmp1, 0);
+#if !LJ_SOFTFP
+ emit_tg(as, MIPSI_DMFC1, tmp1, key);
+#endif
+ } else {
+ checkmclim(as);
+ emit_dta(as, MIPSI_DSRA32, tmp1, tmp1, 0);
+ emit_dta(as, MIPSI_SLL, tmp2, key, 0);
+ emit_dst(as, MIPSI_DADDU, tmp1, key, type);
+ }
+#endif
}
}
}
@@ -768,17 +985,24 @@ static void asm_hrefk(ASMState *as, IRIns *ir)
int32_t kofs = ofs + (int32_t)offsetof(Node, key);
Reg dest = (ra_used(ir)||ofs > 32736) ? ra_dest(as, ir, RSET_GPR) : RID_NONE;
Reg node = ra_alloc1(as, ir->op1, RSET_GPR);
- Reg key = RID_NONE, type = RID_TMP, idx = node;
RegSet allow = rset_exclude(RSET_GPR, node);
+ Reg idx = node;
+#if LJ_32
+ Reg key = RID_NONE, type = RID_TMP;
int32_t lo, hi;
+#else
+ Reg key = ra_scratch(as, allow);
+ int64_t k;
+#endif
lua_assert(ofs % sizeof(Node) == 0);
if (ofs > 32736) {
idx = dest;
rset_clear(allow, dest);
kofs = (int32_t)offsetof(Node, key);
} else if (ra_hasreg(dest)) {
- emit_tsi(as, MIPSI_ADDIU, dest, node, ofs);
+ emit_tsi(as, MIPSI_AADDIU, dest, node, ofs);
}
+#if LJ_32
if (!irt_ispri(irkey->t)) {
key = ra_scratch(as, allow);
rset_clear(allow, key);
@@ -797,8 +1021,20 @@ nolo:
asm_guard(as, MIPSI_BNE, type, hi ? ra_allock(as, hi, allow) : RID_ZERO);
if (ra_hasreg(key)) emit_tsi(as, MIPSI_LW, key, idx, kofs+(LJ_BE?4:0));
emit_tsi(as, MIPSI_LW, type, idx, kofs+(LJ_BE?0:4));
+#else
+ if (irt_ispri(irkey->t)) {
+ lua_assert(!irt_isnil(irkey->t));
+ k = ~((int64_t)~irt_toitype(irkey->t) << 47);
+ } else if (irt_isnum(irkey->t)) {
+ k = (int64_t)ir_knum(irkey)->u64;
+ } else {
+ k = ((int64_t)irt_toitype(irkey->t) << 47) | (int64_t)ir_kgc(irkey);
+ }
+ asm_guard(as, MIPSI_BNE, key, ra_allock(as, k, allow));
+ emit_tsi(as, MIPSI_LD, key, idx, kofs);
+#endif
if (ofs > 32736)
- emit_tsi(as, MIPSI_ADDU, dest, node, ra_allock(as, ofs, allow));
+ emit_tsi(as, MIPSI_AADDU, dest, node, ra_allock(as, ofs, allow));
}
static void asm_uref(ASMState *as, IRIns *ir)
@@ -807,19 +1043,19 @@ static void asm_uref(ASMState *as, IRIns *ir)
if (irref_isk(ir->op1)) {
GCfunc *fn = ir_kfunc(IR(ir->op1));
MRef *v = &gcref(fn->l.uvptr[(ir->op2 >> 8)])->uv.v;
- emit_lsptr(as, MIPSI_LW, dest, v, RSET_GPR);
+ emit_lsptr(as, MIPSI_AL, dest, v, RSET_GPR);
} else {
Reg uv = ra_scratch(as, RSET_GPR);
Reg func = ra_alloc1(as, ir->op1, RSET_GPR);
if (ir->o == IR_UREFC) {
asm_guard(as, MIPSI_BEQ, RID_TMP, RID_ZERO);
- emit_tsi(as, MIPSI_ADDIU, dest, uv, (int32_t)offsetof(GCupval, tv));
+ emit_tsi(as, MIPSI_AADDIU, dest, uv, (int32_t)offsetof(GCupval, tv));
emit_tsi(as, MIPSI_LBU, RID_TMP, uv, (int32_t)offsetof(GCupval, closed));
} else {
- emit_tsi(as, MIPSI_LW, dest, uv, (int32_t)offsetof(GCupval, v));
+ emit_tsi(as, MIPSI_AL, dest, uv, (int32_t)offsetof(GCupval, v));
}
- emit_tsi(as, MIPSI_LW, uv, func,
- (int32_t)offsetof(GCfuncL, uvptr) + 4*(int32_t)(ir->op2 >> 8));
+ emit_tsi(as, MIPSI_AL, uv, func, (int32_t)offsetof(GCfuncL, uvptr) +
+ (int32_t)sizeof(MRef) * (int32_t)(ir->op2 >> 8));
}
}
@@ -831,6 +1067,7 @@ static void asm_fref(ASMState *as, IRIns *ir)
static void asm_strref(ASMState *as, IRIns *ir)
{
+#if LJ_32
Reg dest = ra_dest(as, ir, RSET_GPR);
IRRef ref = ir->op2, refk = ir->op1;
int32_t ofs = (int32_t)sizeof(GCstr);
@@ -862,6 +1099,20 @@ static void asm_strref(ASMState *as, IRIns *ir)
else
emit_dst(as, MIPSI_ADDU, dest, r,
ra_allock(as, ofs, rset_exclude(RSET_GPR, r)));
+#else
+ RegSet allow = RSET_GPR;
+ Reg dest = ra_dest(as, ir, allow);
+ Reg base = ra_alloc1(as, ir->op1, allow);
+ IRIns *irr = IR(ir->op2);
+ int32_t ofs = sizeof(GCstr);
+ rset_clear(allow, base);
+ if (irref_isk(ir->op2) && checki16(ofs + irr->i)) {
+ emit_tsi(as, MIPSI_DADDIU, dest, base, ofs + irr->i);
+ } else {
+ emit_tsi(as, MIPSI_DADDIU, dest, dest, ofs);
+ emit_dst(as, MIPSI_DADDU, dest, base, ra_alloc1(as, ir->op2, allow));
+ }
+#endif
}
/* -- Loads and stores ---------------------------------------------------- */
@@ -875,7 +1126,7 @@ static MIPSIns asm_fxloadins(IRIns *ir)
case IRT_U16: return MIPSI_LHU;
case IRT_NUM: lua_assert(!LJ_SOFTFP); return MIPSI_LDC1;
case IRT_FLOAT: if (!LJ_SOFTFP) return MIPSI_LWC1;
- default: return MIPSI_LW;
+ default: return (LJ_64 && irt_is64(ir->t)) ? MIPSI_LD : MIPSI_LW;
}
}
@@ -886,7 +1137,7 @@ static MIPSIns asm_fxstoreins(IRIns *ir)
case IRT_I16: case IRT_U16: return MIPSI_SH;
case IRT_NUM: lua_assert(!LJ_SOFTFP); return MIPSI_SDC1;
case IRT_FLOAT: if (!LJ_SOFTFP) return MIPSI_SWC1;
- default: return MIPSI_SW;
+ default: return (LJ_64 && irt_is64(ir->t)) ? MIPSI_SD : MIPSI_SW;
}
}
@@ -898,13 +1149,13 @@ static void asm_fload(ASMState *as, IRIns *ir)
int32_t ofs;
if (ir->op1 == REF_NIL) {
idx = RID_JGL;
- ofs = (ir->op2 << 2) - 32768;
+ ofs = (ir->op2 << 2) - 32768 - GG_OFS(g);
} else {
idx = ra_alloc1(as, ir->op1, RSET_GPR);
if (ir->op2 == IRFL_TAB_ARRAY) {
ofs = asm_fuseabase(as, ir->op1);
if (ofs) { /* Turn the t->array load into an add for colocated arrays. */
- emit_tsi(as, MIPSI_ADDIU, dest, idx, ofs);
+ emit_tsi(as, MIPSI_AADDIU, dest, idx, ofs);
return;
}
}
@@ -949,36 +1200,59 @@ static void asm_xstore_(ASMState *as, IRIns *ir, int32_t ofs)
static void asm_ahuvload(ASMState *as, IRIns *ir)
{
- int hiop = (LJ_SOFTFP && (ir+1)->o == IR_HIOP);
- IRType t = hiop ? IRT_NUM : irt_type(ir->t);
+ int hiop = (LJ_32 && LJ_SOFTFP && (ir+1)->o == IR_HIOP);
Reg dest = RID_NONE, type = RID_TMP, idx;
RegSet allow = RSET_GPR;
int32_t ofs = 0;
- if (hiop && ra_used(ir+1)) {
- type = ra_dest(as, ir+1, allow);
- rset_clear(allow, type);
+ IRType1 t = ir->t;
+ if (hiop) {
+ t.irt = IRT_NUM;
+ if (ra_used(ir+1)) {
+ type = ra_dest(as, ir+1, allow);
+ rset_clear(allow, type);
+ }
}
if (ra_used(ir)) {
lua_assert((LJ_SOFTFP ? 0 : irt_isnum(ir->t)) ||
irt_isint(ir->t) || irt_isaddr(ir->t));
- dest = ra_dest(as, ir, (!LJ_SOFTFP && t == IRT_NUM) ? RSET_FPR : allow);
+ dest = ra_dest(as, ir, (!LJ_SOFTFP && irt_isnum(t)) ? RSET_FPR : allow);
rset_clear(allow, dest);
+#if LJ_64
+ if (irt_isaddr(t))
+ emit_tsml(as, MIPSI_DEXTM, dest, dest, 14, 0);
+ else if (irt_isint(t))
+ emit_dta(as, MIPSI_SLL, dest, dest, 0);
+#endif
}
idx = asm_fuseahuref(as, ir->op1, &ofs, allow);
rset_clear(allow, idx);
- if (t == IRT_NUM) {
+ if (irt_isnum(t)) {
asm_guard(as, MIPSI_BEQ, RID_TMP, RID_ZERO);
emit_tsi(as, MIPSI_SLTIU, RID_TMP, type, (int32_t)LJ_TISNUM);
} else {
- asm_guard(as, MIPSI_BNE, type, ra_allock(as, irt_toitype_(t), allow));
+ asm_guard(as, MIPSI_BNE, type,
+ ra_allock(as, (int32_t)irt_toitype(t), allow));
}
+#if LJ_32
if (ra_hasreg(dest)) {
- if (!LJ_SOFTFP && t == IRT_NUM)
+ if (!LJ_SOFTFP && irt_isnum(t))
emit_hsi(as, MIPSI_LDC1, dest, idx, ofs);
else
emit_tsi(as, MIPSI_LW, dest, idx, ofs+(LJ_BE?4:0));
}
emit_tsi(as, MIPSI_LW, type, idx, ofs+(LJ_BE?0:4));
+#else
+ if (ra_hasreg(dest)) {
+ if (!LJ_SOFTFP && irt_isnum(t)) {
+ emit_hsi(as, MIPSI_LDC1, dest, idx, ofs);
+ dest = type;
+ }
+ } else {
+ dest = type;
+ }
+ emit_dta(as, MIPSI_DSRA32, type, dest, 15);
+ emit_tsi(as, MIPSI_LD, dest, idx, ofs);
+#endif
}
static void asm_ahustore(ASMState *as, IRIns *ir)
@@ -990,103 +1264,159 @@ static void asm_ahustore(ASMState *as, IRIns *ir)
return;
if (!LJ_SOFTFP && irt_isnum(ir->t)) {
src = ra_alloc1(as, ir->op2, RSET_FPR);
+ idx = asm_fuseahuref(as, ir->op1, &ofs, allow);
+ emit_hsi(as, MIPSI_SDC1, src, idx, ofs);
} else {
- int hiop = (LJ_SOFTFP && (ir+1)->o == IR_HIOP);
+#if LJ_32
if (!irt_ispri(ir->t)) {
src = ra_alloc1(as, ir->op2, allow);
rset_clear(allow, src);
}
- if (hiop)
+ if (LJ_SOFTFP && (ir+1)->o == IR_HIOP)
type = ra_alloc1(as, (ir+1)->op2, allow);
else
type = ra_allock(as, (int32_t)irt_toitype(ir->t), allow);
rset_clear(allow, type);
- }
- idx = asm_fuseahuref(as, ir->op1, &ofs, allow);
- if (!LJ_SOFTFP && irt_isnum(ir->t)) {
- emit_hsi(as, MIPSI_SDC1, src, idx, ofs);
- } else {
+ idx = asm_fuseahuref(as, ir->op1, &ofs, allow);
if (ra_hasreg(src))
emit_tsi(as, MIPSI_SW, src, idx, ofs+(LJ_BE?4:0));
emit_tsi(as, MIPSI_SW, type, idx, ofs+(LJ_BE?0:4));
+#else
+ Reg tmp = RID_TMP;
+ if (irt_ispri(ir->t)) {
+ tmp = ra_allock(as, ~((int64_t)~irt_toitype(ir->t) << 47), allow);
+ rset_clear(allow, tmp);
+ } else {
+ src = ra_alloc1(as, ir->op2, allow);
+ rset_clear(allow, src);
+ type = ra_allock(as, (int64_t)irt_toitype(ir->t) << 47, allow);
+ rset_clear(allow, type);
+ }
+ idx = asm_fuseahuref(as, ir->op1, &ofs, allow);
+ emit_tsi(as, MIPSI_SD, tmp, idx, ofs);
+ if (ra_hasreg(src)) {
+ if (irt_isinteger(ir->t)) {
+ emit_dst(as, MIPSI_DADDU, tmp, tmp, type);
+ emit_tsml(as, MIPSI_DEXT, tmp, src, 31, 0);
+ } else {
+ emit_dst(as, MIPSI_DADDU, tmp, src, type);
+ }
+ }
+#endif
}
}
static void asm_sload(ASMState *as, IRIns *ir)
{
- int32_t ofs = 8*((int32_t)ir->op1-1) + ((ir->op2 & IRSLOAD_FRAME) ? 4 : 0);
- int hiop = (LJ_SOFTFP && (ir+1)->o == IR_HIOP);
- IRType t = hiop ? IRT_NUM : irt_type(ir->t);
Reg dest = RID_NONE, type = RID_NONE, base;
RegSet allow = RSET_GPR;
+ IRType1 t = ir->t;
+#if LJ_32
+ int32_t ofs = 8*((int32_t)ir->op1-1) + ((ir->op2 & IRSLOAD_FRAME) ? 4 : 0);
+ int hiop = (LJ_32 && LJ_SOFTFP && (ir+1)->o == IR_HIOP);
+ if (hiop)
+ t.irt = IRT_NUM;
+#else
+ int32_t ofs = 8*((int32_t)ir->op1-2);
+#endif
lua_assert(!(ir->op2 & IRSLOAD_PARENT)); /* Handled by asm_head_side(). */
lua_assert(irt_isguard(ir->t) || !(ir->op2 & IRSLOAD_TYPECHECK));
-#if LJ_SOFTFP
+#if LJ_32 && LJ_SOFTFP
lua_assert(!(ir->op2 & IRSLOAD_CONVERT)); /* Handled by LJ_SOFTFP SPLIT. */
if (hiop && ra_used(ir+1)) {
type = ra_dest(as, ir+1, allow);
rset_clear(allow, type);
}
#else
- if ((ir->op2 & IRSLOAD_CONVERT) && irt_isguard(ir->t) && t == IRT_INT) {
+ if ((ir->op2 & IRSLOAD_CONVERT) && irt_isguard(t) && irt_isint(t)) {
dest = ra_scratch(as, RSET_FPR);
asm_tointg(as, ir, dest);
- t = IRT_NUM; /* Continue with a regular number type check. */
+ t.irt = IRT_NUM; /* Continue with a regular number type check. */
} else
#endif
if (ra_used(ir)) {
lua_assert((LJ_SOFTFP ? 0 : irt_isnum(ir->t)) ||
irt_isint(ir->t) || irt_isaddr(ir->t));
- dest = ra_dest(as, ir, (!LJ_SOFTFP && t == IRT_NUM) ? RSET_FPR : allow);
+ dest = ra_dest(as, ir, (!LJ_SOFTFP && irt_isnum(t)) ? RSET_FPR : allow);
rset_clear(allow, dest);
base = ra_alloc1(as, REF_BASE, allow);
rset_clear(allow, base);
if (!LJ_SOFTFP && (ir->op2 & IRSLOAD_CONVERT)) {
- if (t == IRT_INT) {
+ if (irt_isint(t)) {
Reg tmp = ra_scratch(as, RSET_FPR);
emit_tg(as, MIPSI_MFC1, dest, tmp);
emit_fg(as, MIPSI_TRUNC_W_D, tmp, tmp);
dest = tmp;
- t = IRT_NUM; /* Check for original type. */
+ t.irt = IRT_NUM; /* Check for original type. */
} else {
Reg tmp = ra_scratch(as, RSET_GPR);
emit_fg(as, MIPSI_CVT_D_W, dest, dest);
emit_tg(as, MIPSI_MTC1, tmp, dest);
dest = tmp;
- t = IRT_INT; /* Check for original type. */
+ t.irt = IRT_INT; /* Check for original type. */
}
}
+#if LJ_64
+ else if (irt_isaddr(t)) {
+ /* Clear type from pointers. */
+ emit_tsml(as, MIPSI_DEXTM, dest, dest, 14, 0);
+ } else if (irt_isint(t) && (ir->op2 & IRSLOAD_TYPECHECK)) {
+ /* Sign-extend integers. */
+ emit_dta(as, MIPSI_SLL, dest, dest, 0);
+ }
+#endif
goto dotypecheck;
}
base = ra_alloc1(as, REF_BASE, allow);
rset_clear(allow, base);
dotypecheck:
+#if LJ_32
if ((ir->op2 & IRSLOAD_TYPECHECK)) {
- if (ra_noreg(type)) {
- if (ofs < 256 && ra_hasreg(dest) && (dest & 1) == 0 &&
- rset_test((as->freeset & allow), dest+1)) {
- type = dest+1;
- ra_modified(as, type);
- } else {
- type = RID_TMP;
- }
- }
- if (t == IRT_NUM) {
+ if (ra_noreg(type))
+ type = RID_TMP;
+ if (irt_isnum(t)) {
asm_guard(as, MIPSI_BEQ, RID_TMP, RID_ZERO);
emit_tsi(as, MIPSI_SLTIU, RID_TMP, type, (int32_t)LJ_TISNUM);
} else {
- Reg ktype = ra_allock(as, irt_toitype_(t), allow);
+ Reg ktype = ra_allock(as, irt_toitype(t), allow);
asm_guard(as, MIPSI_BNE, type, ktype);
}
}
if (ra_hasreg(dest)) {
- if (!LJ_SOFTFP && t == IRT_NUM)
+ if (!LJ_SOFTFP && irt_isnum(t))
emit_hsi(as, MIPSI_LDC1, dest, base, ofs);
else
emit_tsi(as, MIPSI_LW, dest, base, ofs ^ (LJ_BE?4:0));
}
if (ra_hasreg(type))
emit_tsi(as, MIPSI_LW, type, base, ofs ^ (LJ_BE?0:4));
+#else
+ if ((ir->op2 & IRSLOAD_TYPECHECK)) {
+ type = dest < RID_MAX_GPR ? dest : RID_TMP;
+ if (irt_ispri(t)) {
+ asm_guard(as, MIPSI_BNE, type,
+ ra_allock(as, ~((int64_t)~irt_toitype(t) << 47) , allow));
+ } else {
+ if (irt_isnum(t)) {
+ asm_guard(as, MIPSI_BEQ, RID_TMP, RID_ZERO);
+ emit_tsi(as, MIPSI_SLTIU, RID_TMP, RID_TMP, (int32_t)LJ_TISNUM);
+ if (ra_hasreg(dest))
+ emit_hsi(as, MIPSI_LDC1, dest, base, ofs);
+ } else {
+ asm_guard(as, MIPSI_BNE, RID_TMP,
+ ra_allock(as, (int32_t)irt_toitype(t), allow));
+ }
+ emit_dta(as, MIPSI_DSRA32, RID_TMP, type, 15);
+ }
+ emit_tsi(as, MIPSI_LD, type, base, ofs);
+ } else if (ra_hasreg(dest)) {
+ if (irt_isnum(t))
+ emit_hsi(as, MIPSI_LDC1, dest, base, ofs);
+ else
+ emit_tsi(as, irt_isint(t) ? MIPSI_LW : MIPSI_LD, dest, base,
+ ofs ^ ((LJ_BE && irt_isint(t)) ? 4 : 0));
+ }
+#endif
}
/* -- Allocations --------------------------------------------------------- */
@@ -1113,8 +1443,8 @@ static void asm_cnew(ASMState *as, IRIns *ir)
/* Initialize immutable cdata object. */
if (ir->o == IR_CNEWI) {
RegSet allow = (RSET_GPR & ~RSET_SCRATCH);
+#if LJ_32
int32_t ofs = sizeof(GCcdata);
- lua_assert(sz == 4 || sz == 8);
if (sz == 8) {
ofs += 4;
lua_assert((ir+1)->o == IR_HIOP);
@@ -1127,6 +1457,11 @@ static void asm_cnew(ASMState *as, IRIns *ir)
if (ofs == sizeof(GCcdata)) break;
ofs -= 4; if (LJ_BE) ir++; else ir--;
}
+#else
+ emit_tsi(as, MIPSI_SD, ra_alloc1(as, ir->op2, allow),
+ RID_RET, sizeof(GCcdata));
+#endif
+ lua_assert(sz == 4 || sz == 8);
} else if (ir->op2 != REF_NIL) { /* Create VLA/VLS/aligned cdata. */
ci = &lj_ir_callinfo[IRCALL_lj_cdata_newv];
args[0] = ASMREF_L; /* lua_State *L */
@@ -1161,7 +1496,7 @@ static void asm_tbar(ASMState *as, IRIns *ir)
Reg mark = ra_scratch(as, rset_exclude(RSET_GPR, tab));
Reg link = RID_TMP;
MCLabel l_end = emit_label(as);
- emit_tsi(as, MIPSI_SW, link, tab, (int32_t)offsetof(GCtab, gclist));
+ emit_tsi(as, MIPSI_AS, link, tab, (int32_t)offsetof(GCtab, gclist));
emit_tsi(as, MIPSI_SB, mark, tab, (int32_t)offsetof(GCtab, marked));
emit_setgl(as, tab, gc.grayagain);
emit_getgl(as, link, gc.grayagain);
@@ -1184,7 +1519,7 @@ static void asm_obar(ASMState *as, IRIns *ir)
args[0] = ASMREF_TMP1; /* global_State *g */
args[1] = ir->op1; /* TValue *tv */
asm_gencall(as, ci, args);
- emit_tsi(as, MIPSI_ADDIU, ra_releasetmp(as, ASMREF_TMP1), RID_JGL, -32768);
+ emit_tsi(as, MIPSI_AADDIU, ra_releasetmp(as, ASMREF_TMP1), RID_JGL, -32768);
obj = IR(ir->op1)->r;
tmp = ra_scratch(as, rset_exclude(RSET_GPR, obj));
emit_branch(as, MIPSI_BEQ, RID_TMP, RID_ZERO, l_end);
@@ -1230,8 +1565,9 @@ static void asm_fpmath(ASMState *as, IRIns *ir)
static void asm_add(ASMState *as, IRIns *ir)
{
+ IRType1 t = ir->t;
#if !LJ_SOFTFP
- if (irt_isnum(ir->t)) {
+ if (irt_isnum(t)) {
asm_fparith(as, ir, MIPSI_ADD_D);
} else
#endif
@@ -1239,14 +1575,16 @@ static void asm_add(ASMState *as, IRIns *ir)
Reg dest = ra_dest(as, ir, RSET_GPR);
Reg right, left = ra_hintalloc(as, ir->op1, dest, RSET_GPR);
if (irref_isk(ir->op2)) {
- int32_t k = IR(ir->op2)->i;
+ intptr_t k = get_kval(IR(ir->op2));
if (checki16(k)) {
- emit_tsi(as, MIPSI_ADDIU, dest, left, k);
+ emit_tsi(as, (LJ_64 && irt_is64(t)) ? MIPSI_DADDIU : MIPSI_ADDIU, dest,
+ left, k);
return;
}
}
right = ra_alloc1(as, ir->op2, rset_exclude(RSET_GPR, left));
- emit_dst(as, MIPSI_ADDU, dest, left, right);
+ emit_dst(as, (LJ_64 && irt_is64(t)) ? MIPSI_DADDU : MIPSI_ADDU, dest,
+ left, right);
}
}
@@ -1261,7 +1599,8 @@ static void asm_sub(ASMState *as, IRIns *ir)
Reg dest = ra_dest(as, ir, RSET_GPR);
Reg right, left = ra_alloc2(as, ir, RSET_GPR);
right = (left >> 8); left &= 255;
- emit_dst(as, MIPSI_SUBU, dest, left, right);
+ emit_dst(as, (LJ_64 && irt_is64(ir->t)) ? MIPSI_DSUBU : MIPSI_SUBU, dest,
+ left, right);
}
}
@@ -1276,13 +1615,49 @@ static void asm_mul(ASMState *as, IRIns *ir)
Reg dest = ra_dest(as, ir, RSET_GPR);
Reg right, left = ra_alloc2(as, ir, RSET_GPR);
right = (left >> 8); left &= 255;
- emit_dst(as, MIPSI_MUL, dest, left, right);
+ if (LJ_64 && irt_is64(ir->t)) {
+ emit_dst(as, MIPSI_MFLO, dest, 0, 0);
+ emit_dst(as, MIPSI_DMULT, 0, left, right);
+ } else {
+ emit_dst(as, MIPSI_MUL, dest, left, right);
+ }
}
}
-#define asm_div(as, ir) asm_fparith(as, ir, MIPSI_DIV_D)
-#define asm_mod(as, ir) asm_callid(as, ir, IRCALL_lj_vm_modi)
-#define asm_pow(as, ir) asm_callid(as, ir, IRCALL_lj_vm_powi)
+static void asm_mod(ASMState *as, IRIns *ir)
+{
+#if LJ_64 && LJ_HASFFI
+ if (!irt_isint(ir->t))
+ asm_callid(as, ir, irt_isi64(ir->t) ? IRCALL_lj_carith_modi64 :
+ IRCALL_lj_carith_modu64);
+ else
+#endif
+ asm_callid(as, ir, IRCALL_lj_vm_modi);
+}
+
+#if !LJ_SOFTFP
+static void asm_pow(ASMState *as, IRIns *ir)
+{
+#if LJ_64 && LJ_HASFFI
+ if (!irt_isnum(ir->t))
+ asm_callid(as, ir, irt_isi64(ir->t) ? IRCALL_lj_carith_powi64 :
+ IRCALL_lj_carith_powu64);
+ else
+#endif
+ asm_callid(as, ir, IRCALL_lj_vm_powi);
+}
+
+static void asm_div(ASMState *as, IRIns *ir)
+{
+#if LJ_64 && LJ_HASFFI
+ if (!irt_isnum(ir->t))
+ asm_callid(as, ir, irt_isi64(ir->t) ? IRCALL_lj_carith_divi64 :
+ IRCALL_lj_carith_divu64);
+ else
+#endif
+ asm_fparith(as, ir, MIPSI_DIV_D);
+}
+#endif
static void asm_neg(ASMState *as, IRIns *ir)
{
@@ -1294,7 +1669,8 @@ static void asm_neg(ASMState *as, IRIns *ir)
{
Reg dest = ra_dest(as, ir, RSET_GPR);
Reg left = ra_hintalloc(as, ir->op1, dest, RSET_GPR);
- emit_dst(as, MIPSI_SUBU, dest, RID_ZERO, left);
+ emit_dst(as, (LJ_64 && irt_is64(ir->t)) ? MIPSI_DSUBU : MIPSI_SUBU, dest,
+ RID_ZERO, left);
}
}
@@ -1305,6 +1681,7 @@ static void asm_neg(ASMState *as, IRIns *ir)
static void asm_arithov(ASMState *as, IRIns *ir)
{
Reg right, left, tmp, dest = ra_dest(as, ir, RSET_GPR);
+ lua_assert(!irt_is64(ir->t));
if (irref_isk(ir->op2)) {
int k = IR(ir->op2)->i;
if (ir->o == IR_SUBOV) k = -k;
@@ -1352,7 +1729,7 @@ static void asm_mulov(ASMState *as, IRIns *ir)
emit_dst(as, MIPSI_MULT, 0, left, right);
}
-#if LJ_HASFFI
+#if LJ_32 && LJ_HASFFI
static void asm_add64(ASMState *as, IRIns *ir)
{
Reg dest = ra_dest(as, ir, RSET_GPR);
@@ -1454,6 +1831,7 @@ static void asm_bswap(ASMState *as, IRIns *ir)
{
Reg dest = ra_dest(as, ir, RSET_GPR);
Reg left = ra_alloc1(as, ir->op1, RSET_GPR);
+#if LJ_32
if ((as->flags & JIT_F_MIPSXXR2)) {
emit_dta(as, MIPSI_ROTR, dest, RID_TMP, 16);
emit_dst(as, MIPSI_WSBH, RID_TMP, 0, left);
@@ -1469,6 +1847,15 @@ static void asm_bswap(ASMState *as, IRIns *ir)
emit_dta(as, MIPSI_SRL, tmp, left, 24);
emit_dta(as, MIPSI_SLL, RID_TMP, left, 24);
}
+#else
+ if (irt_is64(ir->t)) {
+ emit_dst(as, MIPSI_DSHD, dest, 0, RID_TMP);
+ emit_dst(as, MIPSI_DSBH, RID_TMP, 0, left);
+ } else {
+ emit_dta(as, MIPSI_ROTR, dest, RID_TMP, 16);
+ emit_dst(as, MIPSI_WSBH, RID_TMP, 0, left);
+ }
+#endif
}
static void asm_bitop(ASMState *as, IRIns *ir, MIPSIns mi, MIPSIns mik)
@@ -1476,7 +1863,7 @@ static void asm_bitop(ASMState *as, IRIns *ir, MIPSIns mi, MIPSIns mik)
Reg dest = ra_dest(as, ir, RSET_GPR);
Reg right, left = ra_hintalloc(as, ir->op1, dest, RSET_GPR);
if (irref_isk(ir->op2)) {
- int32_t k = IR(ir->op2)->i;
+ intptr_t k = get_kval(IR(ir->op2));
if (checku16(k)) {
emit_tsi(as, mik, dest, left, k);
return;
@@ -1494,11 +1881,14 @@ static void asm_bitshift(ASMState *as, IRIns *ir, MIPSIns mi, MIPSIns mik)
{
Reg dest = ra_dest(as, ir, RSET_GPR);
if (irref_isk(ir->op2)) { /* Constant shifts. */
- uint32_t shift = (uint32_t)(IR(ir->op2)->i & 31);
- emit_dta(as, mik, dest, ra_hintalloc(as, ir->op1, dest, RSET_GPR), shift);
+ uint32_t shift = (uint32_t)IR(ir->op2)->i;
+ if (LJ_64 && irt_is64(ir->t)) mik |= (shift & 32) ? MIPSI_D32 : MIPSI_D;
+ emit_dta(as, mik, dest, ra_hintalloc(as, ir->op1, dest, RSET_GPR),
+ (shift & 31));
} else {
Reg right, left = ra_alloc2(as, ir, RSET_GPR);
right = (left >> 8); left &= 255;
+ if (LJ_64 && irt_is64(ir->t)) mi |= MIPSI_DV;
emit_dst(as, mi, dest, right, left); /* Shift amount is in rs. */
}
}
@@ -1510,7 +1900,7 @@ static void asm_bitshift(ASMState *as, IRIns *ir, MIPSIns mi, MIPSIns mik)
static void asm_bror(ASMState *as, IRIns *ir)
{
- if ((as->flags & JIT_F_MIPSXXR2)) {
+ if (LJ_64 || (as->flags & JIT_F_MIPSXXR2)) {
asm_bitshift(as, ir, MIPSI_ROTRV, MIPSI_ROTR);
} else {
Reg dest = ra_dest(as, ir, RSET_GPR);
@@ -1529,7 +1919,7 @@ static void asm_bror(ASMState *as, IRIns *ir)
}
}
-#if LJ_SOFTFP
+#if LJ_32 && LJ_SOFTFP
static void asm_sfpmin_max(ASMState *as, IRIns *ir)
{
CCallInfo ci = lj_ir_callinfo[(IROp)ir->o == IR_MIN ? IRCALL_lj_vm_sfmin : IRCALL_lj_vm_sfmax];
@@ -1578,7 +1968,7 @@ static void asm_min_max(ASMState *as, IRIns *ir, int ismax)
/* -- Comparisons --------------------------------------------------------- */
-#if LJ_SOFTFP
+#if LJ_32 && LJ_SOFTFP
/* SFP comparisons. */
static void asm_sfpcomp(ASMState *as, IRIns *ir)
{
@@ -1651,13 +2041,13 @@ static void asm_comp(ASMState *as, IRIns *ir)
} else {
Reg right, left = ra_alloc1(as, ir->op1, RSET_GPR);
if (op == IR_ABC) op = IR_UGT;
- if ((op&4) == 0 && irref_isk(ir->op2) && IR(ir->op2)->i == 0) {
+ if ((op&4) == 0 && irref_isk(ir->op2) && get_kval(IR(ir->op2)) == 0) {
MIPSIns mi = (op&2) ? ((op&1) ? MIPSI_BLEZ : MIPSI_BGTZ) :
((op&1) ? MIPSI_BLTZ : MIPSI_BGEZ);
asm_guard(as, mi, left, 0);
} else {
if (irref_isk(ir->op2)) {
- int32_t k = IR(ir->op2)->i;
+ intptr_t k = get_kval(IR(ir->op2));
if ((op&2)) k++;
if (checki16(k)) {
asm_guard(as, (op&1) ? MIPSI_BNE : MIPSI_BEQ, RID_TMP, RID_ZERO);
@@ -1676,7 +2066,8 @@ static void asm_comp(ASMState *as, IRIns *ir)
static void asm_equal(ASMState *as, IRIns *ir)
{
- Reg right, left = ra_alloc2(as, ir, (!LJ_SOFTFP && irt_isnum(ir->t)) ? RSET_FPR : RSET_GPR);
+ Reg right, left = ra_alloc2(as, ir, (!LJ_SOFTFP && irt_isnum(ir->t)) ?
+ RSET_FPR : RSET_GPR);
right = (left >> 8); left &= 255;
if (!LJ_SOFTFP && irt_isnum(ir->t)) {
asm_guard(as, (ir->o & 1) ? MIPSI_BC1T : MIPSI_BC1F, 0, 0);
@@ -1686,7 +2077,7 @@ static void asm_equal(ASMState *as, IRIns *ir)
}
}
-#if LJ_HASFFI
+#if LJ_32 && LJ_HASFFI
/* 64 bit integer comparisons. */
static void asm_comp64(ASMState *as, IRIns *ir)
{
@@ -1728,7 +2119,7 @@ static void asm_comp64eq(ASMState *as, IRIns *ir)
/* Hiword op of a split 64 bit op. Previous op must be the loword op. */
static void asm_hiop(ASMState *as, IRIns *ir)
{
-#if LJ_HASFFI || LJ_SOFTFP
+#if LJ_32 && (LJ_HASFFI || LJ_SOFTFP)
/* HIOP is marked as a store because it needs its own DCE logic. */
int uselo = ra_used(ir-1), usehi = ra_used(ir); /* Loword/hiword used? */
if (LJ_UNLIKELY(!(as->flags & JIT_F_OPT_DCE))) uselo = usehi = 1;
@@ -1832,36 +2223,42 @@ static void asm_stack_check(ASMState *as, BCReg topslot,
Reg tmp, pbase = irp ? (ra_hasreg(irp->r) ? irp->r : RID_TMP) : RID_BASE;
ExitNo oldsnap = as->snapno;
rset_clear(allow, pbase);
+#if LJ_32
tmp = allow ? rset_pickbot(allow) :
(pbase == RID_RETHI ? RID_RETLO : RID_RETHI);
+#else
+ tmp = allow ? rset_pickbot(allow) : RID_RET;
+#endif
as->snapno = exitno;
asm_guard(as, MIPSI_BNE, RID_TMP, RID_ZERO);
as->snapno = oldsnap;
if (allow == RSET_EMPTY) /* Restore temp. register. */
- emit_tsi(as, MIPSI_LW, tmp, RID_SP, 0);
+ emit_tsi(as, MIPSI_AL, tmp, RID_SP, 0);
else
ra_modified(as, tmp);
emit_tsi(as, MIPSI_SLTIU, RID_TMP, RID_TMP, (int32_t)(8*topslot));
- emit_dst(as, MIPSI_SUBU, RID_TMP, tmp, pbase);
- emit_tsi(as, MIPSI_LW, tmp, tmp, offsetof(lua_State, maxstack));
+ emit_dst(as, MIPSI_ASUBU, RID_TMP, tmp, pbase);
+ emit_tsi(as, MIPSI_AL, tmp, tmp, offsetof(lua_State, maxstack));
if (pbase == RID_TMP)
emit_getgl(as, RID_TMP, jit_base);
emit_getgl(as, tmp, cur_L);
if (allow == RSET_EMPTY) /* Spill temp. register. */
- emit_tsi(as, MIPSI_SW, tmp, RID_SP, 0);
+ emit_tsi(as, MIPSI_AS, tmp, RID_SP, 0);
}
/* Restore Lua stack from on-trace state. */
static void asm_stack_restore(ASMState *as, SnapShot *snap)
{
SnapEntry *map = &as->T->snapmap[snap->mapofs];
- SnapEntry *flinks = &as->T->snapmap[snap_nextofs(as->T, snap)-1];
+#if LJ_32 || defined(LUA_USE_ASSERT)
+ SnapEntry *flinks = &as->T->snapmap[snap_nextofs(as->T, snap)-1-LJ_FR2];
+#endif
MSize n, nent = snap->nent;
/* Store the value of all modified slots to the Lua stack. */
for (n = 0; n < nent; n++) {
SnapEntry sn = map[n];
BCReg s = snap_slot(sn);
- int32_t ofs = 8*((int32_t)s-1);
+ int32_t ofs = 8*((int32_t)s-1-LJ_FR2);
IRRef ref = snap_ref(sn);
IRIns *ir = IR(ref);
if ((sn & SNAP_NORESTORE))
@@ -1881,8 +2278,9 @@ static void asm_stack_restore(ASMState *as, SnapShot *snap)
emit_hsi(as, MIPSI_SDC1, src, RID_BASE, ofs);
#endif
} else {
- Reg type;
+#if LJ_32
RegSet allow = rset_exclude(RSET_GPR, RID_BASE);
+ Reg type;
lua_assert(irt_ispri(ir->t) || irt_isaddr(ir->t) || irt_isinteger(ir->t));
if (!irt_ispri(ir->t)) {
Reg src = ra_alloc1(as, ref, allow);
@@ -1900,6 +2298,9 @@ static void asm_stack_restore(ASMState *as, SnapShot *snap)
type = ra_allock(as, (int32_t)irt_toitype(ir->t), allow);
}
emit_tsi(as, MIPSI_SW, type, RID_BASE, ofs+(LJ_BE?0:4));
+#else
+ asm_tvstore64(as, RID_BASE, ofs, ref);
+#endif
}
checkmclim(as);
}
@@ -1923,7 +2324,7 @@ static void asm_gc_check(ASMState *as)
args[0] = ASMREF_TMP1; /* global_State *g */
args[1] = ASMREF_TMP2; /* MSize steps */
asm_gencall(as, ci, args);
- emit_tsi(as, MIPSI_ADDIU, ra_releasetmp(as, ASMREF_TMP1), RID_JGL, -32768);
+ emit_tsi(as, MIPSI_AADDIU, ra_releasetmp(as, ASMREF_TMP1), RID_JGL, -32768);
tmp = ra_releasetmp(as, ASMREF_TMP2);
emit_loadi(as, tmp, as->gcsteps);
/* Jump around GC step if GC total < GC threshold. */
@@ -1998,7 +2399,7 @@ static void asm_tail_fixup(ASMState *as, TraceNo lnk)
MCode *target = lnk ? traceref(as->J,lnk)->mcode : (MCode *)lj_vm_exit_interp;
int32_t spadj = as->T->spadjust;
MCode *p = as->mctop-1;
- *p = spadj ? (MIPSI_ADDIU|MIPSF_T(RID_SP)|MIPSF_S(RID_SP)|spadj) : MIPSI_NOP;
+ *p = spadj ? (MIPSI_AADDIU|MIPSF_T(RID_SP)|MIPSF_S(RID_SP)|spadj) : MIPSI_NOP;
p[-1] = MIPSI_J|(((uintptr_t)target>>2)&0x03ffffffu);
}
@@ -2016,9 +2417,14 @@ static Reg asm_setup_call_slots(ASMState *as, IRIns *ir, const CCallInfo *ci)
{
IRRef args[CCI_NARGS_MAX*2];
uint32_t i, nargs = CCI_XNARGS(ci);
+#if LJ_32
int nslots = 4, ngpr = REGARG_NUMGPR, nfpr = REGARG_NUMFPR;
+#else
+ int nslots = 0, ngpr = REGARG_NUMGPR;
+#endif
asm_collectargs(as, ir, ci, args);
for (i = 0; i < nargs; i++) {
+#if LJ_32
if (!LJ_SOFTFP && args[i] && irt_isfp(IR(args[i])->t) &&
nfpr > 0 && !(ci->flags & CCI_VARARG)) {
nfpr--;
@@ -2031,6 +2437,9 @@ static Reg asm_setup_call_slots(ASMState *as, IRIns *ir, const CCallInfo *ci)
nfpr = 0;
if (ngpr > 0) ngpr--; else nslots++;
}
+#else
+ if (ngpr > 0) ngpr--; else nslots += 2;
+#endif
}
if (nslots > as->evenspill) /* Leave room for args in stack slots. */
as->evenspill = nslots;
diff --git a/src/lj_emit_mips.h b/src/lj_emit_mips.h
index 20ecb7a2..8a9ee24d 100644
--- a/src/lj_emit_mips.h
+++ b/src/lj_emit_mips.h
@@ -3,6 +3,28 @@
** Copyright (C) 2005-2017 Mike Pall. See Copyright Notice in luajit.h
*/
+#if LJ_64
+static intptr_t get_k64val(IRIns *ir)
+{
+ if (ir->o == IR_KINT64) {
+ return (intptr_t)ir_kint64(ir)->u64;
+ } else if (ir->o == IR_KGC) {
+ return (intptr_t)ir_kgc(ir);
+ } else if (ir->o == IR_KPTR || ir->o == IR_KKPTR) {
+ return (intptr_t)ir_kptr(ir);
+ } else {
+ lua_assert(ir->o == IR_KINT || ir->o == IR_KNULL);
+ return ir->i; /* Sign-extended. */
+ }
+}
+#endif
+
+#if LJ_64
+#define get_kval(ir) get_k64val(ir)
+#else
+#define get_kval(ir) ((ir)->i)
+#endif
+
/* -- Emit basic instructions --------------------------------------------- */
static void emit_dst(ASMState *as, MIPSIns mi, Reg rd, Reg rs, Reg rt)
@@ -35,7 +57,7 @@ static void emit_fgh(ASMState *as, MIPSIns mi, Reg rf, Reg rg, Reg rh)
static void emit_rotr(ASMState *as, Reg dest, Reg src, Reg tmp, uint32_t shift)
{
- if ((as->flags & JIT_F_MIPSXXR2)) {
+ if (LJ_64 || (as->flags & JIT_F_MIPSXXR2)) {
emit_dta(as, MIPSI_ROTR, dest, src, shift);
} else {
emit_dst(as, MIPSI_OR, dest, dest, tmp);
@@ -44,13 +66,21 @@ static void emit_rotr(ASMState *as, Reg dest, Reg src, Reg tmp, uint32_t shift)
}
}
+#if LJ_64
+static void emit_tsml(ASMState *as, MIPSIns mi, Reg rt, Reg rs, uint32_t msb,
+ uint32_t lsb)
+{
+ *--as->mcp = mi | MIPSF_T(rt) | MIPSF_S(rs) | MIPSF_M(msb) | MIPSF_L(lsb);
+}
+#endif
+
/* -- Emit loads/stores --------------------------------------------------- */
/* Prefer rematerialization of BASE/L from global_State over spills. */
#define emit_canremat(ref) ((ref) <= REF_BASE)
/* Try to find a one step delta relative to another constant. */
-static int emit_kdelta1(ASMState *as, Reg t, int32_t i)
+static int emit_kdelta1(ASMState *as, Reg t, intptr_t i)
{
RegSet work = ~as->freeset & RSET_GPR;
while (work) {
@@ -58,9 +88,10 @@ static int emit_kdelta1(ASMState *as, Reg t, int32_t i)
IRRef ref = regcost_ref(as->cost[r]);
lua_assert(r != t);
if (ref < ASMREF_L) {
- int32_t delta = i - (ra_iskref(ref) ? ra_krefk(as, ref) : IR(ref)->i);
+ intptr_t delta = (intptr_t)((uintptr_t)i -
+ (uintptr_t)(ra_iskref(ref) ? ra_krefk(as, ref) : get_kval(IR(ref))));
if (checki16(delta)) {
- emit_tsi(as, MIPSI_ADDIU, t, r, delta);
+ emit_tsi(as, MIPSI_AADDIU, t, r, delta);
return 1;
}
}
@@ -76,8 +107,8 @@ static void emit_loadi(ASMState *as, Reg r, int32_t i)
emit_ti(as, MIPSI_LI, r, i);
} else {
if ((i & 0xffff)) {
- int32_t jgl = i32ptr(J2G(as->J));
- if ((uint32_t)(i-jgl) < 65536) {
+ intptr_t jgl = (intptr_t)(void *)J2G(as->J);
+ if ((uintptr_t)(i-jgl) < 65536) {
emit_tsi(as, MIPSI_ADDIU, r, RID_JGL, i-jgl-32768);
return;
} else if (emit_kdelta1(as, r, i)) {
@@ -92,7 +123,39 @@ static void emit_loadi(ASMState *as, Reg r, int32_t i)
}
}
+#if LJ_64
+/* Load a 64 bit constant into a GPR. */
+static void emit_loadu64(ASMState *as, Reg r, uint64_t u64)
+{
+ if (checki32((int64_t)u64)) {
+ emit_loadi(as, r, (int32_t)u64);
+ } else {
+ uint64_t delta = u64 - (uint64_t)(void *)J2G(as->J);
+ if (delta < 65536) {
+ emit_tsi(as, MIPSI_DADDIU, r, RID_JGL, (int32_t)(delta-32768));
+ } else if (emit_kdelta1(as, r, (intptr_t)u64)) {
+ return;
+ } else {
+ if ((u64 & 0xffff)) {
+ emit_tsi(as, MIPSI_ORI, r, r, u64 & 0xffff);
+ }
+ if (((u64 >> 16) & 0xffff)) {
+ emit_dta(as, MIPSI_DSLL, r, r, 16);
+ emit_tsi(as, MIPSI_ORI, r, r, (u64 >> 16) & 0xffff);
+ emit_dta(as, MIPSI_DSLL, r, r, 16);
+ } else {
+ emit_dta(as, MIPSI_DSLL32, r, r, 0);
+ }
+ emit_loadi(as, r, (int32_t)(u64 >> 32));
+ }
+ /* TODO: There are probably more optimization opportunities. */
+ }
+}
+
+#define emit_loada(as, r, addr) emit_loadu64(as, (r), u64ptr((addr)))
+#else
#define emit_loada(as, r, addr) emit_loadi(as, (r), i32ptr((addr)))
+#endif
static Reg ra_allock(ASMState *as, intptr_t k, RegSet allow);
static void ra_allockreg(ASMState *as, intptr_t k, Reg r);
@@ -100,8 +163,8 @@ static void ra_allockreg(ASMState *as, intptr_t k, Reg r);
/* Get/set from constant pointer. */
static void emit_lsptr(ASMState *as, MIPSIns mi, Reg r, void *p, RegSet allow)
{
- int32_t jgl = i32ptr(J2G(as->J));
- int32_t i = i32ptr(p);
+ intptr_t jgl = (intptr_t)(J2G(as->J));
+ intptr_t i = (intptr_t)(p);
Reg base;
if ((uint32_t)(i-jgl) < 65536) {
i = i-jgl-32768;
@@ -112,8 +175,24 @@ static void emit_lsptr(ASMState *as, MIPSIns mi, Reg r, void *p, RegSet allow)
emit_tsi(as, mi, r, base, i);
}
+#if LJ_64
+static void emit_loadk64(ASMState *as, Reg r, IRIns *ir)
+{
+ const uint64_t *k = &ir_k64(ir)->u64;
+ Reg r64 = r;
+ if (rset_test(RSET_FPR, r)) {
+ r64 = RID_TMP;
+ emit_tg(as, MIPSI_DMTC1, r64, r);
+ }
+ if ((uint32_t)((intptr_t)k-(intptr_t)J2G(as->J)) < 65536)
+ emit_lsptr(as, MIPSI_LD, r64, (void *)k, 0);
+ else
+ emit_loadu64(as, r64, *k);
+}
+#else
#define emit_loadk64(as, r, ir) \
emit_lsptr(as, MIPSI_LDC1, ((r) & 31), (void *)&ir_knum((ir))->u64, RSET_GPR)
+#endif
/* Get/set global_State fields. */
static void emit_lsglptr(ASMState *as, MIPSIns mi, Reg r, int32_t ofs)
@@ -122,9 +201,9 @@ static void emit_lsglptr(ASMState *as, MIPSIns mi, Reg r, int32_t ofs)
}
#define emit_getgl(as, r, field) \
- emit_lsglptr(as, MIPSI_LW, (r), (int32_t)offsetof(global_State, field))
+ emit_lsglptr(as, MIPSI_AL, (r), (int32_t)offsetof(global_State, field))
#define emit_setgl(as, r, field) \
- emit_lsglptr(as, MIPSI_SW, (r), (int32_t)offsetof(global_State, field))
+ emit_lsglptr(as, MIPSI_AS, (r), (int32_t)offsetof(global_State, field))
/* Trace number is determined from per-trace exit stubs. */
#define emit_setvmstate(as, i) UNUSED(i)
@@ -164,7 +243,7 @@ static void emit_call(ASMState *as, void *target, int needcfa)
needcfa = 1;
}
as->mcp = p;
- if (needcfa) ra_allockreg(as, i32ptr(target), RID_CFUNCADDR);
+ if (needcfa) ra_allockreg(as, (intptr_t)target, RID_CFUNCADDR);
}
/* -- Emit generic operations --------------------------------------------- */
@@ -185,7 +264,7 @@ static void emit_movrr(ASMState *as, IRIns *ir, Reg dst, Reg src)
static void emit_loadofs(ASMState *as, IRIns *ir, Reg r, Reg base, int32_t ofs)
{
if (r < RID_MAX_GPR)
- emit_tsi(as, MIPSI_LW, r, base, ofs);
+ emit_tsi(as, irt_is64(ir->t) ? MIPSI_LD : MIPSI_LW, r, base, ofs);
else
emit_tsi(as, irt_isnum(ir->t) ? MIPSI_LDC1 : MIPSI_LWC1,
(r & 31), base, ofs);
@@ -195,7 +274,7 @@ static void emit_loadofs(ASMState *as, IRIns *ir, Reg r, Reg base, int32_t ofs)
static void emit_storeofs(ASMState *as, IRIns *ir, Reg r, Reg base, int32_t ofs)
{
if (r < RID_MAX_GPR)
- emit_tsi(as, MIPSI_SW, r, base, ofs);
+ emit_tsi(as, irt_is64(ir->t) ? MIPSI_SD : MIPSI_SW, r, base, ofs);
else
emit_tsi(as, irt_isnum(ir->t) ? MIPSI_SDC1 : MIPSI_SWC1,
(r&31), base, ofs);
@@ -206,7 +285,7 @@ static void emit_addptr(ASMState *as, Reg r, int32_t ofs)
{
if (ofs) {
lua_assert(checki16(ofs));
- emit_tsi(as, MIPSI_ADDIU, r, r, ofs);
+ emit_tsi(as, MIPSI_AADDIU, r, r, ofs);
}
}
diff --git a/src/lj_jit.h b/src/lj_jit.h
index ddcb576c..92054e3d 100644
--- a/src/lj_jit.h
+++ b/src/lj_jit.h
@@ -337,6 +337,10 @@ enum {
#endif
#if LJ_TARGET_MIPS
LJ_K64_2P31, /* 2^31 */
+#if LJ_64
+ LJ_K64_2P63, /* 2^63 */
+ LJ_K64_M2P64, /* -2^64 */
+#endif
#endif
LJ_K64__MAX,
};
@@ -352,6 +356,10 @@ enum {
#if LJ_TARGET_PPC || LJ_TARGET_MIPS
LJ_K32_2P31, /* 2^31 */
#endif
+#if LJ_TARGET_MIPS64
+ LJ_K32_2P63, /* 2^63 */
+ LJ_K32_M2P64, /* -2^64 */
+#endif
LJ_K32__MAX
};
diff --git a/src/lj_mcode.c b/src/lj_mcode.c
index a33a4c5d..0f29a3ce 100644
--- a/src/lj_mcode.c
+++ b/src/lj_mcode.c
@@ -206,7 +206,7 @@ static void mcode_protect(jit_State *J, int prot)
#if LJ_TARGET_X64
#define mcode_validptr(p) ((p) && (uintptr_t)(p) < (uintptr_t)1<<47)
-#elif LJ_TARGET_ARM64
+#elif LJ_TARGET_ARM64 || LJ_TARGET_MIPS64
/* We have no clue about the valid VA range. It could be 39 - 52 bits. */
#define mcode_validptr(p) (p)
#else
@@ -224,8 +224,8 @@ static void *mcode_alloc(jit_State *J, size_t sz)
*/
#if LJ_TARGET_MIPS
/* Use the middle of the 256MB-aligned region. */
- uintptr_t target = ((uintptr_t)(void *)lj_vm_exit_handler & 0xf0000000u) +
- 0x08000000u;
+ uintptr_t target = ((uintptr_t)(void *)lj_vm_exit_handler &
+ ~(uintptr_t)0x0fffffffu) + 0x08000000u;
#else
uintptr_t target = (uintptr_t)(void *)lj_vm_exit_handler & ~(uintptr_t)0xffff;
#endif
diff --git a/src/lj_snap.c b/src/lj_snap.c
index 2eb40a8d..bb063c2b 100644
--- a/src/lj_snap.c
+++ b/src/lj_snap.c
@@ -723,8 +723,9 @@ static void snap_restoredata(GCtrace *T, ExitState *ex,
#else
if (LJ_BE && sz == 4) src++;
#endif
- }
+ } else
#endif
+ if (LJ_64 && LJ_BE && sz == 4) src++;
}
}
lua_assert(sz == 1 || sz == 2 || sz == 4 || sz == 8);
diff --git a/src/lj_target_mips.h b/src/lj_target_mips.h
index 1b061943..740687b3 100644
--- a/src/lj_target_mips.h
+++ b/src/lj_target_mips.h
@@ -81,7 +81,7 @@ enum {
RID2RSET(RID_SYS1)|RID2RSET(RID_SYS2)|RID2RSET(RID_JGL)|RID2RSET(RID_GP))
#define RSET_GPR (RSET_RANGE(RID_MIN_GPR, RID_MAX_GPR) - RSET_FIXED)
#if LJ_SOFTFP
-#define RSET_FPR 0
+#define RSET_FPR 0
#else
#if LJ_32
#define RSET_FPR \
@@ -90,11 +90,11 @@ enum {
RID2RSET(RID_F16)|RID2RSET(RID_F18)|RID2RSET(RID_F20)|RID2RSET(RID_F22)|\
RID2RSET(RID_F24)|RID2RSET(RID_F26)|RID2RSET(RID_F28)|RID2RSET(RID_F30))
#else
-#define RSET_FPR RSET_RANGE(RID_MIN_FPR, RID_MAX_FPR)
+#define RSET_FPR RSET_RANGE(RID_MIN_FPR, RID_MAX_FPR)
#endif
#endif
-#define RSET_ALL (RSET_GPR|RSET_FPR)
-#define RSET_INIT RSET_ALL
+#define RSET_ALL (RSET_GPR|RSET_FPR)
+#define RSET_INIT RSET_ALL
#define RSET_SCRATCH_GPR \
(RSET_RANGE(RID_R1, RID_R15+1)|\
@@ -192,8 +192,12 @@ static LJ_AINLINE uint32_t *exitstub_trace_addr_(uint32_t *p)
#define MIPSF_F(r) ((r) << 6)
#define MIPSF_A(n) ((n) << 6)
#define MIPSF_M(n) ((n) << 11)
+#define MIPSF_L(n) ((n) << 6)
typedef enum MIPSIns {
+ MIPSI_D = 0x38,
+ MIPSI_DV = 0x10,
+ MIPSI_D32 = 0x3c,
/* Integer instructions. */
MIPSI_MOVE = 0x00000025,
MIPSI_NOP = 0x00000000,
@@ -202,22 +206,27 @@ typedef enum MIPSIns {
MIPSI_LU = 0x34000000,
MIPSI_LUI = 0x3c000000,
- MIPSI_ADDIU = 0x24000000,
+ MIPSI_AND = 0x00000024,
MIPSI_ANDI = 0x30000000,
+ MIPSI_OR = 0x00000025,
MIPSI_ORI = 0x34000000,
+ MIPSI_XOR = 0x00000026,
MIPSI_XORI = 0x38000000,
+ MIPSI_NOR = 0x00000027,
+
+ MIPSI_SLT = 0x0000002a,
+ MIPSI_SLTU = 0x0000002b,
MIPSI_SLTI = 0x28000000,
MIPSI_SLTIU = 0x2c000000,
MIPSI_ADDU = 0x00000021,
+ MIPSI_ADDIU = 0x24000000,
+ MIPSI_SUB = 0x00000022,
MIPSI_SUBU = 0x00000023,
MIPSI_MUL = 0x70000002,
- MIPSI_AND = 0x00000024,
- MIPSI_OR = 0x00000025,
- MIPSI_XOR = 0x00000026,
- MIPSI_NOR = 0x00000027,
- MIPSI_SLT = 0x0000002a,
- MIPSI_SLTU = 0x0000002b,
+ MIPSI_DIV = 0x0000001a,
+ MIPSI_DIVU = 0x0000001b,
+
MIPSI_MOVZ = 0x0000000a,
MIPSI_MOVN = 0x0000000b,
MIPSI_MFHI = 0x00000010,
@@ -228,14 +237,18 @@ typedef enum MIPSIns {
MIPSI_SRL = 0x00000002,
MIPSI_SRA = 0x00000003,
MIPSI_ROTR = 0x00200002, /* MIPSXXR2 */
+ MIPSI_DROTR = 0x0020003a,
+ MIPSI_DROTR32 = 0x0020003e,
MIPSI_SLLV = 0x00000004,
MIPSI_SRLV = 0x00000006,
MIPSI_SRAV = 0x00000007,
MIPSI_ROTRV = 0x00000046, /* MIPSXXR2 */
+ MIPSI_DROTRV = 0x00000056,
MIPSI_SEB = 0x7c000420, /* MIPSXXR2 */
MIPSI_SEH = 0x7c000620, /* MIPSXXR2 */
MIPSI_WSBH = 0x7c0000a0, /* MIPSXXR2 */
+ MIPSI_DSBH = 0x7c0000a4,
MIPSI_B = 0x10000000,
MIPSI_J = 0x08000000,
@@ -253,7 +266,9 @@ typedef enum MIPSIns {
/* Load/store instructions. */
MIPSI_LW = 0x8c000000,
+ MIPSI_LD = 0xdc000000,
MIPSI_SW = 0xac000000,
+ MIPSI_SD = 0xfc000000,
MIPSI_LB = 0x80000000,
MIPSI_SB = 0xa0000000,
MIPSI_LH = 0x84000000,
@@ -266,13 +281,48 @@ typedef enum MIPSIns {
MIPSI_SDC1 = 0xf4000000,
/* MIPS64 instructions. */
- MIPSI_DSLL = 0x00000038,
- MIPSI_LD = 0xdc000000,
+ MIPSI_DADD = 0x0000002c,
+ MIPSI_DADDI = 0x60000000,
+ MIPSI_DADDU = 0x0000002d,
MIPSI_DADDIU = 0x64000000,
- MIPSI_SD = 0xfc000000,
- MIPSI_DMFC1 = 0x44200000,
+ MIPSI_DSUB = 0x0000002e,
+ MIPSI_DSUBU = 0x0000002f,
+ MIPSI_DDIV = 0x0000001e,
+ MIPSI_DDIVU = 0x0000001f,
+ MIPSI_DMULT = 0x0000001c,
+ MIPSI_DMULTU = 0x0000001d,
+
+ MIPSI_DSLL = 0x00000038,
+ MIPSI_DSRL = 0x0000003a,
+ MIPSI_DSLLV = 0x00000014,
+ MIPSI_DSRLV = 0x00000016,
+ MIPSI_DSRA = 0x0000003b,
+ MIPSI_DSRAV = 0x00000017,
MIPSI_DSRA32 = 0x0000003f,
- MIPSI_MFHC1 = 0x44600000,
+ MIPSI_DSLL32 = 0x0000003c,
+ MIPSI_DSRL32 = 0x0000003e,
+ MIPSI_DSHD = 0x7c000164,
+
+ MIPSI_AADDU = LJ_32 ? MIPSI_ADDU : MIPSI_DADDU,
+ MIPSI_AADDIU = LJ_32 ? MIPSI_ADDIU : MIPSI_DADDIU,
+ MIPSI_ASUBU = LJ_32 ? MIPSI_SUBU : MIPSI_DSUBU,
+ MIPSI_AL = LJ_32 ? MIPSI_LW : MIPSI_LD,
+ MIPSI_AS = LJ_32 ? MIPSI_SW : MIPSI_SD,
+
+ /* Extract/insert instructions. */
+ MIPSI_DEXTM = 0x7c000001,
+ MIPSI_DEXTU = 0x7c000002,
+ MIPSI_DEXT = 0x7c000003,
+ MIPSI_DINSM = 0x7c000005,
+ MIPSI_DINSU = 0x7c000006,
+ MIPSI_DINS = 0x7c000007,
+
+ MIPSI_RINT_D = 0x4620001a,
+ MIPSI_RINT_S = 0x4600001a,
+ MIPSI_RINT = 0x4400001a,
+ MIPSI_FLOOR_D = 0x4620000b,
+ MIPSI_CEIL_D = 0x4620000a,
+ MIPSI_ROUND_D = 0x46200008,
/* FP instructions. */
MIPSI_MOV_S = 0x46000006,
@@ -298,24 +348,30 @@ typedef enum MIPSIns {
MIPSI_CVT_W_D = 0x46200024,
MIPSI_CVT_S_W = 0x46800020,
MIPSI_CVT_D_W = 0x46800021,
+ MIPSI_CVT_S_L = 0x46a00020,
+ MIPSI_CVT_D_L = 0x46a00021,
MIPSI_TRUNC_W_S = 0x4600000d,
MIPSI_TRUNC_W_D = 0x4620000d,
+ MIPSI_TRUNC_L_S = 0x46000009,
+ MIPSI_TRUNC_L_D = 0x46200009,
MIPSI_FLOOR_W_S = 0x4600000f,
MIPSI_FLOOR_W_D = 0x4620000f,
MIPSI_MFC1 = 0x44000000,
MIPSI_MTC1 = 0x44800000,
+ MIPSI_DMTC1 = 0x44a00000,
+ MIPSI_DMFC1 = 0x44200000,
MIPSI_BC1F = 0x45000000,
MIPSI_BC1T = 0x45010000,
MIPSI_C_EQ_D = 0x46200032,
+ MIPSI_C_OLT_S = 0x46000034,
MIPSI_C_OLT_D = 0x46200034,
MIPSI_C_ULT_D = 0x46200035,
MIPSI_C_OLE_D = 0x46200036,
MIPSI_C_ULE_D = 0x46200037,
-
} MIPSIns;
#endif
diff --git a/src/lj_trace.c b/src/lj_trace.c
index 4cd925ed..80a7f024 100644
--- a/src/lj_trace.c
+++ b/src/lj_trace.c
@@ -319,13 +319,15 @@ void lj_trace_initstate(global_State *g)
/* Initialize 32/64 bit constants. */
#if LJ_TARGET_X86ORX64
J->k64[LJ_K64_TOBIT].u64 = U64x(43380000,00000000);
- J->k64[LJ_K64_2P64].u64 = U64x(43f00000,00000000);
- J->k64[LJ_K64_M2P64].u64 = U64x(c3f00000,00000000);
#if LJ_32
J->k64[LJ_K64_M2P64_31].u64 = U64x(c1e00000,00000000);
#endif
+ J->k64[LJ_K64_2P64].u64 = U64x(43f00000,00000000);
J->k32[LJ_K32_M2P64_31] = LJ_64 ? 0xdf800000 : 0xcf000000;
#endif
+#if LJ_TARGET_X86ORX64 || LJ_TARGET_MIPS64
+ J->k64[LJ_K64_M2P64].u64 = U64x(c3f00000,00000000);
+#endif
#if LJ_TARGET_PPC
J->k32[LJ_K32_2P52_2P31] = 0x59800004;
J->k32[LJ_K32_2P52] = 0x59800000;
@@ -335,6 +337,11 @@ void lj_trace_initstate(global_State *g)
#endif
#if LJ_TARGET_MIPS
J->k64[LJ_K64_2P31].u64 = U64x(41e00000,00000000);
+#if LJ_64
+ J->k64[LJ_K64_2P63].u64 = U64x(43e00000,00000000);
+ J->k32[LJ_K32_2P63] = 0x5f000000;
+ J->k32[LJ_K32_M2P64] = 0xdf800000;
+#endif
#endif
}
diff --git a/src/vm_mips64.dasc b/src/vm_mips64.dasc
index c518c306..f0c22a74 100644
--- a/src/vm_mips64.dasc
+++ b/src/vm_mips64.dasc
@@ -327,7 +327,13 @@
|.macro jmp_extern; jr CFUNCADDR; .endmacro
|
|.macro hotcheck, delta, target
-| NYI
+| dsrl TMP1, PC, 1
+| andi TMP1, TMP1, 126
+| daddu TMP1, TMP1, DISPATCH
+| lhu TMP2, GG_DISP2HOT(TMP1)
+| addiu TMP2, TMP2, -delta
+| bltz TMP2, target
+|. sh TMP2, GG_DISP2HOT(TMP1)
|.endmacro
|
|.macro hotloop
@@ -2150,7 +2156,21 @@ static void build_subroutines(BuildCtx *ctx)
|//-----------------------------------------------------------------------
|
|->vm_record: // Dispatch target for recording phase.
- | NYI
+ |.if JIT
+ | lbu TMP3, DISPATCH_GL(hookmask)(DISPATCH)
+ | andi AT, TMP3, HOOK_VMEVENT // No recording while in vmevent.
+ | bnez AT, >5
+ | // Decrement the hookcount for consistency, but always do the call.
+ |. lw TMP2, DISPATCH_GL(hookcount)(DISPATCH)
+ | andi AT, TMP3, HOOK_ACTIVE
+ | bnez AT, >1
+ |. addiu TMP2, TMP2, -1
+ | andi AT, TMP3, LUA_MASKLINE|LUA_MASKCOUNT
+ | beqz AT, >1
+ |. nop
+ | b >1
+ |. sw TMP2, DISPATCH_GL(hookcount)(DISPATCH)
+ |.endif
|
|->vm_rethook: // Dispatch target for return hooks.
| lbu TMP3, DISPATCH_GL(hookmask)(DISPATCH)
@@ -2201,7 +2221,25 @@ static void build_subroutines(BuildCtx *ctx)
|. lw MULTRES, -24+LO(RB) // Restore MULTRES for *M ins.
|
|->vm_hotloop: // Hot loop counter underflow.
- | NYI
+ |.if JIT
+ | ld LFUNC:TMP1, FRAME_FUNC(BASE)
+ | daddiu CARG1, DISPATCH, GG_DISP2J
+ | cleartp LFUNC:TMP1
+ | sd PC, SAVE_PC
+ | ld TMP1, LFUNC:TMP1->pc
+ | move CARG2, PC
+ | sd L, DISPATCH_J(L)(DISPATCH)
+ | lbu TMP1, PC2PROTO(framesize)(TMP1)
+ | load_got lj_trace_hot
+ | sd BASE, L->base
+ | dsll TMP1, TMP1, 3
+ | daddu TMP1, BASE, TMP1
+ | call_intern lj_trace_hot // (jit_State *J, const BCIns *pc)
+ |. sd TMP1, L->top
+ | b <3
+ |. nop
+ |.endif
+ |
|
|->vm_callhook: // Dispatch target for call hooks.
|.if JIT
@@ -2235,21 +2273,69 @@ static void build_subroutines(BuildCtx *ctx)
|
|->cont_stitch: // Trace stitching.
|.if JIT
- | NYI
+ | // RA = resultptr, RB = meta base
+ | lw INS, -4(PC)
+ | ld TRACE:TMP2, -40(RB) // Save previous trace.
+ | decode_RA8a RC, INS
+ | daddiu AT, MULTRES, -8
+ | cleartp TRACE:TMP2
+ | decode_RA8b RC
+ | beqz AT, >2
+ |. daddu RC, BASE, RC // Call base.
+ |1: // Move results down.
+ | ld CARG1, 0(RA)
+ | daddiu AT, AT, -8
+ | daddiu RA, RA, 8
+ | sd CARG1, 0(RC)
+ | bnez AT, <1
+ |. daddiu RC, RC, 8
+ |2:
+ | decode_RA8a RA, INS
+ | decode_RB8a RB, INS
+ | decode_RA8b RA
+ | decode_RB8b RB
+ | daddu RA, RA, RB
+ | daddu RA, BASE, RA
+ |3:
+ | sltu AT, RC, RA
+ | bnez AT, >9 // More results wanted?
+ |. nop
+ |
+ | lhu TMP3, TRACE:TMP2->traceno
+ | lhu RD, TRACE:TMP2->link
+ | beq RD, TMP3, ->cont_nop // Blacklisted.
+ |. load_got lj_dispatch_stitch
+ | bnez RD, =>BC_JLOOP // Jump to stitched trace.
+ |. sll RD, RD, 3
+ |
+ | // Stitch a new trace to the previous trace.
+ | sw TMP3, DISPATCH_J(exitno)(DISPATCH)
+ | sd L, DISPATCH_J(L)(DISPATCH)
+ | sd BASE, L->base
+ | daddiu CARG1, DISPATCH, GG_DISP2J
+ | call_intern lj_dispatch_stitch // (jit_State *J, const BCIns *pc)
+ |. move CARG2, PC
+ | b ->cont_nop
+ |. ld BASE, L->base
+ |
+ |9:
+ | sd TISNIL, 0(RC)
+ | b <3
+ |. daddiu RC, RC, 8
|.endif
|
|->vm_profhook: // Dispatch target for profiler hook.
#if LJ_HASPROFILE
| load_got lj_dispatch_profile
- | sw MULTRES, SAVE_MULTRES
+ | sd MULTRES, SAVE_MULTRES
| move CARG2, PC
- | sw BASE, L->base
+ | sd BASE, L->base
| call_intern lj_dispatch_profile // (lua_State *L, const BCIns *pc)
|. move CARG1, L
| // HOOK_PROFILE is off again, so re-dispatch to dynamic instruction.
| daddiu PC, PC, -4
| b ->cont_nop
- |. lw BASE, L->base
+ |. ld BASE, L->base
#endif
|
|//-----------------------------------------------------------------------
@@ -2259,6 +2345,7 @@ static void build_subroutines(BuildCtx *ctx)
|.macro savex_, a, b
|.if FPU
| sdc1 f..a, a*8(sp)
+ | sdc1 f..b, b*8(sp)
| sd r..a, 32*8+a*8(sp)
| sd r..b, 32*8+b*8(sp)
|.else
@@ -2269,11 +2356,124 @@ static void build_subroutines(BuildCtx *ctx)
|
|->vm_exit_handler:
|.if JIT
- | NYI
+ |.if FPU
+ | daddiu sp, sp, -(32*8+32*8)
+ |.else
+ | daddiu sp, sp, -(32*8)
+ |.endif
+ | savex_ 0, 1
+ | savex_ 2, 3
+ | savex_ 4, 5
+ | savex_ 6, 7
+ | savex_ 8, 9
+ | savex_ 10, 11
+ | savex_ 12, 13
+ | savex_ 14, 15
+ | savex_ 16, 17
+ | savex_ 18, 19
+ | savex_ 20, 21
+ | savex_ 22, 23
+ | savex_ 24, 25
+ | savex_ 26, 27
+ | savex_ 28, 30
+ |.if FPU
+ | sdc1 f29, 29*8(sp)
+ | sdc1 f31, 31*8(sp)
+ | sd r0, 32*8+31*8(sp) // Clear RID_TMP.
+ | daddiu TMP2, sp, 32*8+32*8 // Recompute original value of sp.
+ | sd TMP2, 32*8+29*8(sp) // Store sp in RID_SP
+ |.else
+ | sd r0, 31*8(sp) // Clear RID_TMP.
+ | daddiu TMP2, sp, 32*8 // Recompute original value of sp.
+ | sd TMP2, 29*8(sp) // Store sp in RID_SP
+ |.endif
+ | li_vmstate EXIT
+ | daddiu DISPATCH, JGL, -GG_DISP2G-32768
+ | lw TMP1, 0(TMP2) // Load exit number.
+ | st_vmstate
+ | ld L, DISPATCH_GL(cur_L)(DISPATCH)
+ | ld BASE, DISPATCH_GL(jit_base)(DISPATCH)
+ | load_got lj_trace_exit
+ | sd L, DISPATCH_J(L)(DISPATCH)
+ | sw ra, DISPATCH_J(parent)(DISPATCH) // Store trace number.
+ | sd BASE, L->base
+ | sw TMP1, DISPATCH_J(exitno)(DISPATCH) // Store exit number.
+ | daddiu CARG1, DISPATCH, GG_DISP2J
+ | sd r0, DISPATCH_GL(jit_base)(DISPATCH)
+ | call_intern lj_trace_exit // (jit_State *J, ExitState *ex)
+ |. move CARG2, sp
+ | // Returns MULTRES (unscaled) or negated error code.
+ | ld TMP1, L->cframe
+ | li AT, -4
+ | ld BASE, L->base
+ | and sp, TMP1, AT
+ | ld PC, SAVE_PC // Get SAVE_PC.
+ | b >1
+ |. sd L, SAVE_L // Set SAVE_L (on-trace resume/yield).
|.endif
|->vm_exit_interp:
|.if JIT
- | NYI
+ | // CRET1 = MULTRES or negated error code, BASE, PC and JGL set.
+ | ld L, SAVE_L
+ | daddiu DISPATCH, JGL, -GG_DISP2G-32768
+ | sd BASE, L->base
+ |1:
+ | bltz CRET1, >9 // Check for error from exit.
+ |. ld LFUNC:RB, FRAME_FUNC(BASE)
+ | .FPU lui TMP3, 0x59c0 // TOBIT = 2^52 + 2^51 (float).
+ | dsll MULTRES, CRET1, 3
+ | cleartp LFUNC:RB
+ | sd MULTRES, SAVE_MULTRES
+ | li TISNIL, LJ_TNIL
+ | li TISNUM, LJ_TISNUM // Setup type comparison constants.
+ | .FPU mtc1 TMP3, TOBIT
+ | ld TMP1, LFUNC:RB->pc
+ | sd r0, DISPATCH_GL(jit_base)(DISPATCH)
+ | ld KBASE, PC2PROTO(k)(TMP1)
+ | .FPU cvt.d.s TOBIT, TOBIT
+ | // Modified copy of ins_next which handles function header dispatch, too.
+ | lw INS, 0(PC)
+ | daddiu PC, PC, 4
+ | // Assumes TISNIL == ~LJ_VMST_INTERP == -1
+ | sw TISNIL, DISPATCH_GL(vmstate)(DISPATCH)
+ | decode_OP8a TMP1, INS
+ | decode_OP8b TMP1
+ | sltiu TMP2, TMP1, BC_FUNCF*8
+ | daddu TMP0, DISPATCH, TMP1
+ | decode_RD8a RD, INS
+ | ld AT, 0(TMP0)
+ | decode_RA8a RA, INS
+ | beqz TMP2, >2
+ |. decode_RA8b RA
+ | jr AT
+ |. decode_RD8b RD
+ |2:
+ | sltiu TMP2, TMP1, (BC_FUNCC+2)*8 // Fast function?
+ | bnez TMP2, >3
+ |. ld TMP1, FRAME_PC(BASE)
+ | // Check frame below fast function.
+ | andi TMP0, TMP1, FRAME_TYPE
+ | bnez TMP0, >3 // Trace stitching continuation?
+ |. nop
+ | // Otherwise set KBASE for Lua function below fast function.
+ | lw TMP2, -4(TMP1)
+ | decode_RA8a TMP0, TMP2
+ | decode_RA8b TMP0
+ | dsubu TMP1, BASE, TMP0
+ | ld LFUNC:TMP2, -32(TMP1)
+ | cleartp LFUNC:TMP2
+ | ld TMP1, LFUNC:TMP2->pc
+ | ld KBASE, PC2PROTO(k)(TMP1)
+ |3:
+ | daddiu RC, MULTRES, -8
+ | jr AT
+ |. daddu RA, RA, BASE
+ |
+ |9: // Rethrow error from the right C frame.
+ | load_got lj_err_throw
+ | negu CARG2, CRET1
+ | call_intern lj_err_throw // (lua_State *L, int errcode)
+ |. move CARG1, L
|.endif
|
|//-----------------------------------------------------------------------
@@ -4013,7 +4213,7 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
| ins_next2
|
|7: // Possible table write barrier for the value. Skip valiswhite check.
- | barrierback TAB:RB, TMP3, TMP0, <2
+ | barrierback TAB:CARG2, TMP3, TMP0, <2
break;
case BC_TSETM:
@@ -4632,7 +4832,18 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
case BC_JLOOP:
|.if JIT
- | NYI
+ | // RA = base*8 (ignored), RD = traceno*8
+ | ld TMP1, DISPATCH_J(trace)(DISPATCH)
+ | li AT, 0
+ | daddu TMP1, TMP1, RD
+ | // Traces on MIPS don't store the trace number, so use 0.
+ | sd AT, DISPATCH_GL(vmstate)(DISPATCH)
+ | ld TRACE:TMP2, 0(TMP1)
+ | sd BASE, DISPATCH_GL(jit_base)(DISPATCH)
+ | ld TMP2, TRACE:TMP2->mcode
+ | sd L, DISPATCH_GL(tmpbuf.L)(DISPATCH)
+ | jr TMP2
+ |. daddiu JGL, DISPATCH, GG_DISP2G+32768
|.endif
break;
@@ -4694,10 +4905,12 @@ static void build_ins(BuildCtx *ctx, BCOp op, int defop)
case BC_IFUNCV:
| // BASE = new base, RA = BASE+framesize*8, RB = LFUNC, RC = nargs*8
+ | li TMP0, LJ_TFUNC
| daddu TMP1, BASE, RC
| ld TMP2, L->maxstack
+ | settp LFUNC:RB, TMP0
| daddu TMP0, RA, RC
- | sd LFUNC:RB, 0(TMP1) // Store (untagged) copy of LFUNC.
+ | sd LFUNC:RB, 0(TMP1) // Store (tagged) copy of LFUNC.
| daddiu TMP3, RC, 16+FRAME_VARG
| sltu AT, TMP0, TMP2
| ld KBASE, -4+PC2PROTO(k)(PC)
View Lorry Controller Status