Preallocate stack space for JIT in execute_ex() to eliminate JIT prologue/epilogue.

4 files changed, 69 insertions, 14 deletions

diff --git a/Zend/zend_execute.c b/Zend/zend_execute.c
index c58e1fa228..fb35b8a116 100644
--- a/Zend/zend_execute.c
+++ b/Zend/zend_execute.c
@@ -4757,6 +4757,11 @@ static zend_always_inline zend_execute_data *_zend_vm_stack_push_call_frame(uint
 		} \
 	} while (0)
 
+#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE
+/* This callback disables optimization of "vm_stack_data" variable in VM */
+void (*zend_touch_vm_stack_data)(void *vm_stack_data) = NULL;
+#endif
+
 #include "zend_vm_execute.h"
 
 ZEND_API zend_result zend_set_user_opcode_handler(zend_uchar opcode, user_opcode_handler_t handler)
diff --git a/Zend/zend_vm_execute.h b/Zend/zend_vm_execute.h
index 0d14ff3bab..60d725b365 100644
--- a/Zend/zend_vm_execute.h
+++ b/Zend/zend_vm_execute.h
@@ -50826,15 +50826,34 @@ ZEND_API void execute_ex(zend_execute_data *ex)
 {
 	DCL_OPLINE
 
+#if defined(ZEND_VM_IP_GLOBAL_REG) || defined(ZEND_VM_IP_GLOBAL_REG)
+	struct {
 #ifdef ZEND_VM_IP_GLOBAL_REG
-	const zend_op *orig_opline = opline;
+		const zend_op *orig_opline;
 #endif
 #ifdef ZEND_VM_FP_GLOBAL_REG
-	zend_execute_data *orig_execute_data = execute_data;
+		zend_execute_data *orig_execute_data;
+#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE
+		char hybrid_jit_red_zone[ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE];
+#endif
+#endif
+	} vm_stack_data;
+#endif
+#ifdef ZEND_VM_IP_GLOBAL_REG
+	vm_stack_data.orig_opline = opline;
+#endif
+#ifdef ZEND_VM_FP_GLOBAL_REG
+	vm_stack_data.orig_execute_data = execute_data;
 	execute_data = ex;
 #else
 	zend_execute_data *execute_data = ex;
 #endif
+#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE
+	memset(vm_stack_data.hybrid_jit_red_zone, 0, ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE);
+	if (zend_touch_vm_stack_data) {
+		zend_touch_vm_stack_data(&vm_stack_data);
+	}
+#endif
 
 #if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID)
 	if (UNEXPECTED(execute_data == NULL)) {
@@ -58788,10 +58807,10 @@ zend_leave_helper_SPEC_LABEL:
 				HYBRID_BREAK();
 			HYBRID_CASE(HYBRID_HALT):
 #ifdef ZEND_VM_FP_GLOBAL_REG
-				execute_data = orig_execute_data;
+				execute_data = vm_stack_data.orig_execute_data;
 #endif
 #ifdef ZEND_VM_IP_GLOBAL_REG
-				opline = orig_opline;
+				opline = vm_stack_data.orig_opline;
 #endif
 				return;
 			HYBRID_DEFAULT:
@@ -58800,9 +58819,9 @@ zend_leave_helper_SPEC_LABEL:
 				HYBRID_BREAK(); /* Never reached */
 #else
 #ifdef ZEND_VM_FP_GLOBAL_REG
-			execute_data = orig_execute_data;
+			execute_data = vm_stack_data.orig_execute_data;
 # ifdef ZEND_VM_IP_GLOBAL_REG
-			opline = orig_opline;
+			opline = vm_stack_data.orig_opline;
 # endif
 			return;
 #else
@@ -58811,7 +58830,7 @@ zend_leave_helper_SPEC_LABEL:
 				ZEND_VM_LOOP_INTERRUPT_CHECK();
 			} else {
 # ifdef ZEND_VM_IP_GLOBAL_REG
-				opline = orig_opline;
+				opline = vm_stack_data.orig_opline;
 # endif
 				return;
 			}
diff --git a/Zend/zend_vm_gen.php b/Zend/zend_vm_gen.php
index 4958189ce4..46030cad09 100755
--- a/Zend/zend_vm_gen.php
+++ b/Zend/zend_vm_gen.php
@@ -1780,10 +1780,10 @@ function gen_executor_code($f, $spec, $kind, $prolog, &$switch_labels = array())
         case ZEND_VM_KIND_HYBRID:
             out($f,"\t\t\tHYBRID_CASE(HYBRID_HALT):\n");
             out($f,"#ifdef ZEND_VM_FP_GLOBAL_REG\n");
-            out($f,"\t\t\t\texecute_data = orig_execute_data;\n");
+            out($f,"\t\t\t\texecute_data = vm_stack_data.orig_execute_data;\n");
             out($f,"#endif\n");
             out($f,"#ifdef ZEND_VM_IP_GLOBAL_REG\n");
-            out($f,"\t\t\t\topline = orig_opline;\n");
+            out($f,"\t\t\t\topline = vm_stack_data.orig_opline;\n");
             out($f,"#endif\n");
             out($f,"\t\t\t\treturn;\n");
             out($f,"\t\t\tHYBRID_DEFAULT:\n");
@@ -2065,15 +2065,34 @@ function gen_executor($f, $skl, $spec, $kind, $executor_name, $initializer_name)
                         out($f,$m[1]."zend_execute_data *execute_data = ex;\n");
                         out($f,"#endif\n");
                     } else {
+                        out($f,"#if defined(ZEND_VM_IP_GLOBAL_REG) || defined(ZEND_VM_IP_GLOBAL_REG)\n");
+                        out($f,$m[1]."struct {\n");
                         out($f,"#ifdef ZEND_VM_IP_GLOBAL_REG\n");
-                        out($f,$m[1]."const zend_op *orig_opline = opline;\n");
+                        out($f,$m[1]."\tconst zend_op *orig_opline;\n");
                         out($f,"#endif\n");
                         out($f,"#ifdef ZEND_VM_FP_GLOBAL_REG\n");
-                        out($f,$m[1]."zend_execute_data *orig_execute_data = execute_data;\n");
+                        out($f,$m[1]."\tzend_execute_data *orig_execute_data;\n");
+                        out($f,"#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE\n");
+                        out($f,$m[1]."\tchar hybrid_jit_red_zone[ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE];\n");
+                        out($f,"#endif\n");
+                        out($f,"#endif\n");
+                        out($f,$m[1]."} vm_stack_data;\n");
+                        out($f,"#endif\n");
+                        out($f,"#ifdef ZEND_VM_IP_GLOBAL_REG\n");
+                        out($f,$m[1]."vm_stack_data.orig_opline = opline;\n");
+                        out($f,"#endif\n");
+                        out($f,"#ifdef ZEND_VM_FP_GLOBAL_REG\n");
+                        out($f,$m[1]."vm_stack_data.orig_execute_data = execute_data;\n");
                         out($f,$m[1]."execute_data = ex;\n");
                         out($f,"#else\n");
                         out($f,$m[1]."zend_execute_data *execute_data = ex;\n");
                         out($f,"#endif\n");
+                        out($f,"#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE\n");
+                        out($f,$m[1]."memset(vm_stack_data.hybrid_jit_red_zone, 0, ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE);\n");
+                        out($f,$m[1]."if (zend_touch_vm_stack_data) {\n");
+                        out($f,$m[1]."\tzend_touch_vm_stack_data(&vm_stack_data);\n");
+                        out($f,$m[1]."}\n");
+                        out($f,"#endif\n");
                     }
                     break;
                 case "INTERNAL_LABELS":
@@ -2159,9 +2178,9 @@ function gen_executor($f, $skl, $spec, $kind, $executor_name, $initializer_name)
                         }
                         out($f,
                                 "#ifdef ZEND_VM_FP_GLOBAL_REG\n" .
-                                $m[1]."execute_data = orig_execute_data;\n" .
+                                $m[1]."execute_data = vm_stack_data.orig_execute_data;\n" .
                                 "# ifdef ZEND_VM_IP_GLOBAL_REG\n" .
-                                $m[1]."opline = orig_opline;\n" .
+                                $m[1]."opline = vm_stack_data.orig_opline;\n" .
                                 "# endif\n" .
                                 $m[1]."return;\n" .
                                 "#else\n" .
@@ -2170,7 +2189,7 @@ function gen_executor($f, $skl, $spec, $kind, $executor_name, $initializer_name)
                                 $m[1]."\tZEND_VM_LOOP_INTERRUPT_CHECK();\n".
                                 $m[1]."} else {\n" .
                                 "# ifdef ZEND_VM_IP_GLOBAL_REG\n" .
-                                $m[1]."\topline = orig_opline;\n" .
+                                $m[1]."\topline = vm_stack_data.orig_opline;\n" .
                                 "# endif\n".
                                 $m[1]."\treturn;\n".
                                 $m[1]."}\n".
@@ -2578,6 +2597,12 @@ function gen_vm($def, $skel) {
         fputs($f, "#define ZEND_VM_KIND\t\t" . $GLOBALS["vm_kind_name"][ZEND_VM_KIND] . "\n");
     }
     fputs($f, "\n");
+    fputs($f, "#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) && !defined(__SANITIZE_ADDRESS__)\n");
+    fputs($f, "# if (defined(i386) || defined(__x86_64__) || defined(_M_X64))\n");
+    fputs($f, "#  define ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 16\n");
+    fputs($f, "# endif\n");
+    fputs($f, "#endif\n");
+    fputs($f, "\n");
     foreach($vm_op_flags as $name => $val) {
         fprintf($f, "#define %-24s 0x%08x\n", $name, $val);
     }
diff --git a/Zend/zend_vm_opcodes.h b/Zend/zend_vm_opcodes.h
index df5a147999..584a44f5ae 100644
--- a/Zend/zend_vm_opcodes.h
+++ b/Zend/zend_vm_opcodes.h
@@ -34,6 +34,12 @@
 # define ZEND_VM_KIND		ZEND_VM_KIND_CALL
 #endif
 
+#if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) && !defined(__SANITIZE_ADDRESS__)
+# if (defined(i386) || defined(__x86_64__) || defined(_M_X64))
+#  define ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE 16
+# endif
+#endif
+
 #define ZEND_VM_OP_SPEC          0x00000001
 #define ZEND_VM_OP_CONST         0x00000002
 #define ZEND_VM_OP_TMPVAR        0x00000004