Merge branch 'jit-jumptable-x64'

* jit-jumptable-x64:
  Use RIP-relative addressing
  Align jumptables
  Fix jumtable index calculation
  Support JIT jumptables on x64

4 files changed, 48 insertions, 33 deletions

diff --git a/ext/opcache/jit/dynasm/dasm_x86.h b/ext/opcache/jit/dynasm/dasm_x86.h
index 77945ac655..39449c15af 100644
--- a/ext/opcache/jit/dynasm/dasm_x86.h
+++ b/ext/opcache/jit/dynasm/dasm_x86.h
@@ -17,11 +17,11 @@
 
 /* Action definitions. DASM_STOP must be 255. */
 enum {
-  DASM_DISP = 233,
+  DASM_DISP = 232,
   DASM_IMM_S, DASM_IMM_B, DASM_IMM_W, DASM_IMM_D, DASM_IMM_WB, DASM_IMM_DB,
   DASM_VREG, DASM_SPACE, DASM_SETLABEL, DASM_REL_A, DASM_REL_LG, DASM_REL_PC,
-  DASM_IMM_LG, DASM_IMM_PC, DASM_LABEL_LG, DASM_LABEL_PC, DASM_ALIGN,
-  DASM_EXTERN, DASM_ESC, DASM_MARK, DASM_SECTION, DASM_STOP
+  DASM_IMM_LG, DASM_IMM_PC, DASM_IMM_PC64, DASM_LABEL_LG, DASM_LABEL_PC,
+  DASM_ALIGN, DASM_EXTERN, DASM_ESC, DASM_MARK, DASM_SECTION, DASM_STOP
 };
 
 /* Maximum number of section buffer positions for a single dasm_put() call. */
@@ -228,6 +228,7 @@ void dasm_put(Dst_DECL, int start, ...)
 	pl -= 246; n = *pl;
 	if (n < 0) n = 0;  /* Start new chain for fwd rel if label exists. */
 	goto linkrel;
+	  case DASM_IMM_PC64: ofs += 4;
       case DASM_REL_PC:
       case DASM_IMM_PC: pl = D->pclabels + va_arg(ap, int); CKPL(pc, PC);
       putrel:
@@ -335,7 +336,8 @@ int dasm_link(Dst_DECL, size_t *szp)
 	case DASM_SPACE: case DASM_IMM_LG: case DASM_VREG: p++;
 	case DASM_DISP: case DASM_IMM_S: case DASM_IMM_B: case DASM_IMM_W:
 	case DASM_IMM_D: case DASM_IMM_WB: case DASM_IMM_DB:
-	case DASM_SETLABEL: case DASM_REL_A: case DASM_IMM_PC: pos++; break;
+	case DASM_SETLABEL: case DASM_REL_A: case DASM_IMM_PC: case DASM_IMM_PC64:
+	  pos++; break;
 	case DASM_LABEL_LG: p++;
 	case DASM_LABEL_PC: b[pos++] += ofs; break; /* Fix label offset. */
 	case DASM_ALIGN: ofs -= (b[pos++]+ofs)&*p++; break; /* Adjust ofs. */
@@ -359,10 +361,13 @@ int dasm_link(Dst_DECL, size_t *szp)
 #ifndef DASM_ALIGNED_WRITES
 typedef ZEND_SET_ALIGNED(1, unsigned short unaligned_short);
 typedef ZEND_SET_ALIGNED(1, unsigned int unaligned_int);
+typedef ZEND_SET_ALIGNED(1, uint64_t unaligned_uint64_t);
 #define dasmw(x) \
   do { *((unaligned_short *)cp) = (unsigned short)(x); cp+=2; } while (0)
 #define dasmd(x) \
   do { *((unaligned_int *)cp) = (unsigned int)(x); cp+=4; } while (0)
+#define dasmq(x) \
+  do { *((unaligned_uint64_t *)cp) = (uint64_t)(x); cp+=8; } while (0)
 #else
 #define dasmw(x)	do { dasmb(x); dasmb((x)>>8); } while (0)
 #define dasmd(x)	do { dasmw(x); dasmw((x)>>16); } while (0)
@@ -442,6 +447,11 @@ int dasm_encode(Dst_DECL, void *buffer)
 	  n = *pb < 0 ? pb[1] : (*pb + (int)(ptrdiff_t)base);
 	  goto wd;
 	}
+	case DASM_IMM_PC64: {
+	  int *pb = DASM_POS2PTR(D, n);
+	  dasmq(*pb < 0 ? pb[1] : (*pb + (ptrdiff_t)base));
+	  break;
+	}
 	case DASM_LABEL_LG: {
 	  int idx = *p++;
 	  if (idx >= 10)
diff --git a/ext/opcache/jit/dynasm/dasm_x86.lua b/ext/opcache/jit/dynasm/dasm_x86.lua
index a5efd98fb5..f7d81d838f 100644
--- a/ext/opcache/jit/dynasm/dasm_x86.lua
+++ b/ext/opcache/jit/dynasm/dasm_x86.lua
@@ -47,7 +47,7 @@ local action_names = {
   -- action arg (1 byte) or int arg, 2 buffer pos (link, offset):
   "REL_LG", "REL_PC",
   -- action arg (1 byte) or int arg, 1 buffer pos (link):
-  "IMM_LG", "IMM_PC",
+  "IMM_LG", "IMM_PC", "IMM_PC64",
   -- action arg (1 byte) or int arg, 1 buffer pos (offset):
   "LABEL_LG", "LABEL_PC",
   -- action arg (1 byte), 1 buffer pos (offset):
@@ -434,7 +434,11 @@ local function wputlabel(aprefix, imm, num)
     end
     wputxb(imm)
   else
-    waction(aprefix.."PC", imm, num)
+    if aprefix == "IMM_" and x64 then
+      waction("IMM_PC64", imm, num)
+    else
+      waction(aprefix.."PC", imm, num)
+    end
   end
 end
 
diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c
index 478244b6d7..0031dc6731 100644
--- a/ext/opcache/jit/zend_jit.c
+++ b/ext/opcache/jit/zend_jit.c
@@ -309,6 +309,9 @@ static void *dasm_link_and_encode(dasm_State             **dasm_state,
 
 	if (ret != DASM_S_OK) {
 		// TODO: dasm_encode() failed ???
+#if ZEND_DEBUG
+		ZEND_UNREACHABLE();
+#endif
 		return NULL;
 	}
 
diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc
index 12dccc00ff..7ba23e3d10 100644
--- a/ext/opcache/jit/zend_jit_x86.dasc
+++ b/ext/opcache/jit/zend_jit_x86.dasc
@@ -11377,10 +11377,6 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o
 {
 	HashTable *jumptable = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2));
 
-	if (sizeof(void*) == 8 && !IS_32BIT(dasm_end)) {
-		// TODO: DynASM stores .aword as 4-bytes even in 64-bit mode ???
-		return 1;
-	}
 	if (opline->op1_type == IS_CONST) {
 		zval *zv = RT_CONSTANT(opline, opline->op1);
 		zval *jump_zv;
@@ -11444,23 +11440,25 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o
 					|	cmp FCARG2a, jumptable->nNumUsed
 					|	jae >3
 					|.if X64
-					|	// TODO: DynASM stores .aword as 4-bytes even in 64-bit mode ???
-					|	movsxd r0, dword [FCARG2a * 4 + >4]
-					|	jmp r0
+						if (!IS_32BIT(dasm_end)) {
+							|	lea r0, aword [>4]
+							|	jmp aword [r0 + FCARG2a * 8]
+						} else {
+							|	jmp aword [FCARG2a * 8 + >4]
+						}
 					|.else
 					|	jmp aword [FCARG2a * 4 + >4]
 					|.endif
 					|3:
 					|.cold_code
+					|.align aword
 					|4:
 					p = jumptable->arData;
 					do {
 						if (Z_TYPE(p->val) == IS_UNDEF) {
-							|	// TODO: DynASM stores .aword as 4-bytes even in 64-bit mode ???
 							|	.aword =>b
 						} else {
 							int b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL(p->val)) - op_array->opcodes];
-							|	// TODO: DynASM stores .aword as 4-bytes even in 64-bit mode ???
 							|	.aword =>b
 						}
 						p++;
@@ -11474,8 +11472,7 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o
 					|	jz =>b
 					|	LOAD_ADDR FCARG1a, jumptable
 					|	sub r0, aword [FCARG1a + offsetof(HashTable, arData)]
-					|	// TODO: DynASM stores .aword as 4-bytes even in 64-bit mode ???
-					|	mov FCARG1a, (sizeof(Bucket) / sizeof(uint32_t))
+					|	mov FCARG1a, (sizeof(Bucket) / sizeof(void*))
 					|.if X64
 					|	cqo
 					|.else
@@ -11483,18 +11480,21 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o
 					|.endif
 					|	idiv FCARG1a
 					|.if X64
-					|	// TODO: DynASM stores .aword as 4-bytes even in 64-bit mode ???
-					|	movsxd r0, dword [r0 + >4]
-					|	jmp r0
+					if (!IS_32BIT(dasm_end)) {
+						|	lea FCARG1a, aword [>4]
+						|	jmp aword [FCARG1a + r0]
+					} else {
+						|	jmp aword [r0 + >4]
+					}
 					|.else
-					|	jmp dword [r0 + >4]
+					|	jmp aword [r0 + >4]
 					|.endif
 					|3:
 					|.cold_code
+					|.align aword
 					|4:
 					ZEND_HASH_FOREACH_VAL(jumptable, val) {
 						b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(val)) - op_array->opcodes];
-						|	// TODO: DynASM stores .aword as 4-bytes even in 64-bit mode ???
 						|	.aword =>b
 					} ZEND_HASH_FOREACH_END();
 					|.code
@@ -11527,8 +11527,7 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o
 				|	jz =>b
 				|	LOAD_ADDR FCARG1a, jumptable
 				|	sub r0, aword [FCARG1a + offsetof(HashTable, arData)]
-				|	// TODO: DynASM stores .aword as 4-bytes even in 64-bit mode ???
-				|	mov FCARG1a, (sizeof(Bucket) / sizeof(uint32_t))
+				|	mov FCARG1a, (sizeof(Bucket) / sizeof(void*))
 				|.if X64
 				|	cqo
 				|.else
@@ -11536,23 +11535,22 @@ static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_o
 				|.endif
 				|	idiv FCARG1a
 				|.if X64
-				|	// TODO: DynASM stores .aword as 4-bytes even in 64-bit mode ???
-				|	movsxd r0, dword [r0 + >4]
-				|	jmp r0
+				if (!IS_32BIT(dasm_end)) {
+					|	lea FCARG1a, aword [>4]
+					|	jmp aword [FCARG1a + r0]
+				} else {
+					|	jmp aword [r0 + >4]
+				}
 				|.else
-				|	jmp dword [r0 + >4]
+				|	jmp aword [r0 + >4]
 				|.endif
 				|3:
 				|.cold_code
+				|.align aword
 				|4:
 				ZEND_HASH_FOREACH_VAL(jumptable, val) {
 					b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(val)) - op_array->opcodes];
-					|	// TODO: DynASM stores .aword as 4-bytes even in 64-bit mode ???
-					|.if X64
 					|	.aword =>b
-					|.else
-					|	.aword =>b
-					|.endif
 				} ZEND_HASH_FOREACH_END();
 				|.code
 			}