summaryrefslogtreecommitdiff
path: root/gcc/config/i386
diff options
context:
space:
mode:
Diffstat (limited to 'gcc/config/i386')
-rw-r--r--gcc/config/i386/bdver3.md18
-rw-r--r--gcc/config/i386/constraints.md27
-rw-r--r--gcc/config/i386/i386-builtin-types.def1
-rw-r--r--gcc/config/i386/i386-c.c17
-rw-r--r--gcc/config/i386/i386-modes.def3
-rw-r--r--gcc/config/i386/i386-protos.h8
-rw-r--r--gcc/config/i386/i386.c3477
-rw-r--r--gcc/config/i386/i386.h195
-rw-r--r--gcc/config/i386/i386.md453
-rw-r--r--gcc/config/i386/i386.opt10
-rw-r--r--gcc/config/i386/mmx.md11
-rw-r--r--gcc/config/i386/predicates.md131
-rw-r--r--gcc/config/i386/rtemself.h12
-rw-r--r--gcc/config/i386/sse.md3293
-rw-r--r--gcc/config/i386/sync.md15
-rw-r--r--gcc/config/i386/t-i38619
-rw-r--r--gcc/config/i386/t-rtems7
-rw-r--r--gcc/config/i386/winnt.c5
-rw-r--r--gcc/config/i386/x-darwin7
-rw-r--r--gcc/config/i386/x-i3867
-rw-r--r--gcc/config/i386/x86-tune.def613
21 files changed, 6301 insertions, 2028 deletions
diff --git a/gcc/config/i386/bdver3.md b/gcc/config/i386/bdver3.md
index 52418b5e5c9..421a3d1b30e 100644
--- a/gcc/config/i386/bdver3.md
+++ b/gcc/config/i386/bdver3.md
@@ -34,20 +34,22 @@
(define_cpu_unit "bdver3-decode0" "bdver3")
(define_cpu_unit "bdver3-decode1" "bdver3")
-(define_cpu_unit "bdver3-decodev" "bdver3")
+(define_cpu_unit "bdver3-decode2" "bdver3")
+(define_cpu_unit "bdver3-decode3" "bdver3")
;; Double decoded instructions take two cycles whereas
;; direct instructions take one cycle.
-;; Therefore four direct instructions can be decoded by
-;; two decoders in two cycles.
;; Vectorpath instructions are single issue instructions.
-;; So, we have separate unit for vector instructions.
-(exclusion_set "bdver3-decodev" "bdver3-decode0,bdver3-decode1")
+;; So, we engage all units vector instructions.
+(define_reservation "bdver3-vector" "bdver3-decode0+bdver3-decode1+bdver3-decode2+bdver3-decode3")
+
+;; Direct instructions can be issued to any of the four decoders
+(define_reservation "bdver3-direct" "(bdver3-decode0|bdver3-decode1|bdver3-decode2|bdver3-decode3)")
-(define_reservation "bdver3-vector" "bdver3-decodev")
-(define_reservation "bdver3-direct" "(bdver3-decode0|bdver3-decode1)")
;; Double instructions take two cycles to decode.
-(define_reservation "bdver3-double" "(bdver3-decode0|bdver3-decode1)*2")
+(define_reservation "bdver3-double" "(bdver3-decode0,bdver3-decode0)|
+ (bdver3-decode1,bdver3-decode1)| (bdver3-decode2,bdver3-decode2)|
+ (bdver3-decode3,bdver3-decode3)")
(define_cpu_unit "bdver3-ieu0" "bdver3_ieu")
(define_cpu_unit "bdver3-ieu1" "bdver3_ieu")
diff --git a/gcc/config/i386/constraints.md b/gcc/config/i386/constraints.md
index 92e0c053fac..7289ae493b0 100644
--- a/gcc/config/i386/constraints.md
+++ b/gcc/config/i386/constraints.md
@@ -18,7 +18,7 @@
;; <http://www.gnu.org/licenses/>.
;;; Unused letters:
-;;; B H T
+;;; H
;;; h j
;; Integer register constraints.
@@ -91,6 +91,9 @@
(define_register_constraint "x" "TARGET_SSE ? SSE_REGS : NO_REGS"
"Any SSE register.")
+(define_register_constraint "B" "TARGET_MPX ? BND_REGS : NO_REGS"
+ "@internal Any bound register.")
+
;; We use the Y prefix to denote any number of conditional register sets:
;; z First SSE register.
;; i SSE2 inter-unit moves to SSE register enabled
@@ -232,3 +235,25 @@
to fit that range (for immediate operands in zero-extending x86-64
instructions)."
(match_operand 0 "x86_64_zext_immediate_operand"))
+
+;; T prefix is used for different address constraints
+;; v - VSIB address
+;; s - address with no segment register
+;; i - address with no index and no rip
+;; b - address with no base and no rip
+
+(define_address_constraint "Tv"
+ "VSIB address operand"
+ (match_operand 0 "vsib_address_operand"))
+
+(define_address_constraint "Ts"
+ "Address operand without segment register"
+ (match_operand 0 "address_no_seg_operand"))
+
+(define_address_constraint "Ti"
+ "MPX address operand without index"
+ (match_operand 0 "address_mpx_no_index_operand"))
+
+(define_address_constraint "Tb"
+ "MPX address operand without base"
+ (match_operand 0 "address_mpx_no_base_operand"))
diff --git a/gcc/config/i386/i386-builtin-types.def b/gcc/config/i386/i386-builtin-types.def
index 314f3e888d8..c866170bde8 100644
--- a/gcc/config/i386/i386-builtin-types.def
+++ b/gcc/config/i386/i386-builtin-types.def
@@ -227,6 +227,7 @@ DEF_FUNCTION_TYPE (VOID, PCVOID)
DEF_FUNCTION_TYPE (VOID, PVOID)
DEF_FUNCTION_TYPE (VOID, UINT64)
DEF_FUNCTION_TYPE (VOID, UNSIGNED)
+DEF_FUNCTION_TYPE (VOID, PUSHORT)
DEF_FUNCTION_TYPE (INT, PUSHORT)
DEF_FUNCTION_TYPE (INT, PUNSIGNED)
DEF_FUNCTION_TYPE (INT, PULONGLONG)
diff --git a/gcc/config/i386/i386-c.c b/gcc/config/i386/i386-c.c
index 2e764e79987..1c053b1e51e 100644
--- a/gcc/config/i386/i386-c.c
+++ b/gcc/config/i386/i386-c.c
@@ -141,6 +141,10 @@ ix86_target_macros_internal (HOST_WIDE_INT isa_flag,
def_or_undef (parse_in, "__corei7");
def_or_undef (parse_in, "__corei7__");
break;
+ case PROCESSOR_COREI7_AVX:
+ def_or_undef (parse_in, "__corei7_avx");
+ def_or_undef (parse_in, "__corei7_avx__");
+ break;
case PROCESSOR_HASWELL:
def_or_undef (parse_in, "__core_avx2");
def_or_undef (parse_in, "__core_avx2__");
@@ -238,6 +242,9 @@ ix86_target_macros_internal (HOST_WIDE_INT isa_flag,
case PROCESSOR_COREI7:
def_or_undef (parse_in, "__tune_corei7__");
break;
+ case PROCESSOR_COREI7_AVX:
+ def_or_undef (parse_in, "__tune_corei7_avx__");
+ break;
case PROCESSOR_HASWELL:
def_or_undef (parse_in, "__tune_core_avx2__");
break;
@@ -358,6 +365,8 @@ ix86_target_macros_internal (HOST_WIDE_INT isa_flag,
def_or_undef (parse_in, "__SSE_MATH__");
if ((fpmath & FPMATH_SSE) && (isa_flag & OPTION_MASK_ISA_SSE2))
def_or_undef (parse_in, "__SSE2_MATH__");
+ if (isa_flag & OPTION_MASK_ISA_MPX)
+ def_or_undef (parse_in, "__MPX__");
}
@@ -368,7 +377,7 @@ ix86_target_macros_internal (HOST_WIDE_INT isa_flag,
static bool
ix86_pragma_target_parse (tree args, tree pop_target)
{
- tree prev_tree = build_target_option_node ();
+ tree prev_tree = build_target_option_node (&global_options);
tree cur_tree;
struct cl_target_option *prev_opt;
struct cl_target_option *cur_opt;
@@ -388,7 +397,8 @@ ix86_pragma_target_parse (tree args, tree pop_target)
}
else
{
- cur_tree = ix86_valid_target_attribute_tree (args);
+ cur_tree = ix86_valid_target_attribute_tree (args, &global_options,
+ &global_options_set);
if (!cur_tree || cur_tree == error_mark_node)
{
cl_target_option_restore (&global_options,
@@ -464,6 +474,9 @@ ix86_target_macros (void)
builtin_define_std ("i386");
}
+ if (!TARGET_80387)
+ cpp_define (parse_in, "_SOFT_FLOAT");
+
if (TARGET_LONG_DOUBLE_64)
cpp_define (parse_in, "__LONG_DOUBLE_64__");
diff --git a/gcc/config/i386/i386-modes.def b/gcc/config/i386/i386-modes.def
index e0b8fc826ab..a73730e08e2 100644
--- a/gcc/config/i386/i386-modes.def
+++ b/gcc/config/i386/i386-modes.def
@@ -87,6 +87,9 @@ VECTOR_MODE (INT, DI, 1); /* V1DI */
VECTOR_MODE (INT, SI, 1); /* V1SI */
VECTOR_MODE (INT, QI, 2); /* V2QI */
+POINTER_BOUNDS_MODE (BND32, 8);
+POINTER_BOUNDS_MODE (BND64, 16);
+
INT_MODE (OI, 32);
INT_MODE (XI, 64);
diff --git a/gcc/config/i386/i386-protos.h b/gcc/config/i386/i386-protos.h
index 3ab2f3a2ac8..fdf9d5804df 100644
--- a/gcc/config/i386/i386-protos.h
+++ b/gcc/config/i386/i386-protos.h
@@ -143,6 +143,7 @@ extern void ix86_split_lshr (rtx *, rtx, enum machine_mode);
extern rtx ix86_find_base_term (rtx);
extern bool ix86_check_movabs (rtx, int);
extern void ix86_split_idivmod (enum machine_mode, rtx[], bool);
+extern bool ix86_emit_cfi ();
extern rtx assign_386_stack_local (enum machine_mode, enum ix86_stack_slot);
extern int ix86_attr_length_immediate_default (rtx, bool);
@@ -220,7 +221,9 @@ extern int ix86_constant_alignment (tree, int);
extern tree ix86_handle_shared_attribute (tree *, tree, tree, int, bool *);
extern tree ix86_handle_selectany_attribute (tree *, tree, tree, int, bool *);
extern int x86_field_alignment (tree, int);
-extern tree ix86_valid_target_attribute_tree (tree);
+extern tree ix86_valid_target_attribute_tree (tree,
+ struct gcc_options *,
+ struct gcc_options *);
extern unsigned int ix86_get_callcvt (const_tree);
#endif
@@ -238,6 +241,9 @@ extern void ix86_expand_mul_widen_evenodd (rtx, rtx, rtx, bool, bool);
extern void ix86_expand_mul_widen_hilo (rtx, rtx, rtx, bool, bool);
extern void ix86_expand_sse2_mulv4si3 (rtx, rtx, rtx);
extern void ix86_expand_sse2_mulvxdi3 (rtx, rtx, rtx);
+extern void ix86_expand_sse2_abs (rtx, rtx);
+
+extern bool ix86_bnd_prefixed_insn_p (rtx);
/* In i386-c.c */
extern void ix86_target_macros (void);
diff --git a/gcc/config/i386/i386.c b/gcc/config/i386/i386.c
index 46c37d800d2..8b8cdfae681 100644
--- a/gcc/config/i386/i386.c
+++ b/gcc/config/i386/i386.c
@@ -61,7 +61,6 @@ along with GCC; see the file COPYING3. If not see
#include "diagnostic.h"
#include "dumpfile.h"
#include "tree-pass.h"
-#include "tree-flow.h"
#include "context.h"
#include "pass_manager.h"
@@ -1834,8 +1833,9 @@ const struct processor_costs *ix86_cost = &pentium_cost;
#define m_P4_NOCONA (m_PENT4 | m_NOCONA)
#define m_CORE2 (1<<PROCESSOR_CORE2)
#define m_COREI7 (1<<PROCESSOR_COREI7)
+#define m_COREI7_AVX (1<<PROCESSOR_COREI7_AVX)
#define m_HASWELL (1<<PROCESSOR_HASWELL)
-#define m_CORE_ALL (m_CORE2 | m_COREI7 | m_HASWELL)
+#define m_CORE_ALL (m_CORE2 | m_COREI7 | m_COREI7_AVX | m_HASWELL)
#define m_ATOM (1<<PROCESSOR_ATOM)
#define m_SLM (1<<PROCESSOR_SLM)
@@ -1898,18 +1898,6 @@ static unsigned int initial_ix86_arch_features[X86_ARCH_LAST] = {
~m_386,
};
-static const unsigned int x86_accumulate_outgoing_args
- = m_PPRO | m_P4_NOCONA | m_ATOM | m_SLM | m_CORE_ALL | m_AMD_MULTIPLE | m_GENERIC;
-
-static const unsigned int x86_arch_always_fancy_math_387
- = m_PENT | m_PPRO | m_P4_NOCONA | m_CORE_ALL | m_ATOM | m_SLM | m_AMD_MULTIPLE | m_GENERIC;
-
-static const unsigned int x86_avx256_split_unaligned_load
- = m_COREI7 | m_GENERIC;
-
-static const unsigned int x86_avx256_split_unaligned_store
- = m_COREI7 | m_BDVER | m_GENERIC;
-
/* In case the average insn count for single function invocation is
lower than this constant, emit fast (but longer) prologue and
epilogue code. */
@@ -1956,6 +1944,8 @@ enum reg_class const regclass_map[FIRST_PSEUDO_REGISTER] =
/* Mask registers. */
MASK_REGS, MASK_EVEX_REGS, MASK_EVEX_REGS, MASK_EVEX_REGS,
MASK_EVEX_REGS, MASK_EVEX_REGS, MASK_EVEX_REGS, MASK_EVEX_REGS,
+ /* MPX bound registers */
+ BND_REGS, BND_REGS, BND_REGS, BND_REGS,
};
/* The "default" register map used in 32bit mode. */
@@ -1972,6 +1962,7 @@ int const dbx_register_map[FIRST_PSEUDO_REGISTER] =
-1, -1, -1, -1, -1, -1, -1, -1, /* AVX-512 registers 16-23*/
-1, -1, -1, -1, -1, -1, -1, -1, /* AVX-512 registers 24-31*/
93, 94, 95, 96, 97, 98, 99, 100, /* Mask registers */
+ 101, 102, 103, 104, /* bound registers */
};
/* The "default" register map used in 64bit mode. */
@@ -1988,6 +1979,7 @@ int const dbx64_register_map[FIRST_PSEUDO_REGISTER] =
67, 68, 69, 70, 71, 72, 73, 74, /* AVX-512 registers 16-23 */
75, 76, 77, 78, 79, 80, 81, 82, /* AVX-512 registers 24-31 */
118, 119, 120, 121, 122, 123, 124, 125, /* Mask registers */
+ 126, 127, 128, 129, /* bound registers */
};
/* Define the register numbers to be used in Dwarf debugging information.
@@ -2056,6 +2048,7 @@ int const svr4_dbx_register_map[FIRST_PSEUDO_REGISTER] =
-1, -1, -1, -1, -1, -1, -1, -1, /* AVX-512 registers 16-23*/
-1, -1, -1, -1, -1, -1, -1, -1, /* AVX-512 registers 24-31*/
93, 94, 95, 96, 97, 98, 99, 100, /* Mask registers */
+ -1, -1, -1, -1, /* bound registers */
};
/* Define parameter passing and return registers. */
@@ -2248,12 +2241,16 @@ enum ix86_function_specific_strings
static char *ix86_target_string (HOST_WIDE_INT, int, const char *,
const char *, enum fpmath_unit, bool);
-static void ix86_function_specific_save (struct cl_target_option *);
-static void ix86_function_specific_restore (struct cl_target_option *);
+static void ix86_function_specific_save (struct cl_target_option *,
+ struct gcc_options *opts);
+static void ix86_function_specific_restore (struct gcc_options *opts,
+ struct cl_target_option *);
static void ix86_function_specific_print (FILE *, int,
struct cl_target_option *);
static bool ix86_valid_target_attribute_p (tree, tree, tree, int);
static bool ix86_valid_target_attribute_inner_p (tree, char *[],
+ struct gcc_options *,
+ struct gcc_options *,
struct gcc_options *);
static bool ix86_can_inline_p (tree, tree);
static void ix86_set_current_function (tree);
@@ -2303,6 +2300,8 @@ static const struct ptt processor_target_table[PROCESSOR_max] =
{&core_cost, 16, 10, 16, 10, 16},
/* Core i7 */
{&core_cost, 16, 10, 16, 10, 16},
+ /* Core i7 avx */
+ {&core_cost, 16, 10, 16, 10, 16},
/* Core avx2 */
{&core_cost, 16, 10, 16, 10, 16},
{&generic_cost, 16, 10, 16, 10, 16},
@@ -2332,6 +2331,7 @@ static const char *const cpu_names[TARGET_CPU_DEFAULT_max] =
"nocona",
"core2",
"corei7",
+ "corei7-avx",
"core-avx2",
"atom",
"slm",
@@ -2478,6 +2478,7 @@ ix86_target_string (HOST_WIDE_INT isa, int flags, const char *arch,
{ "-mrtm", OPTION_MASK_ISA_RTM },
{ "-mxsave", OPTION_MASK_ISA_XSAVE },
{ "-mxsaveopt", OPTION_MASK_ISA_XSAVEOPT },
+ { "-mmpx", OPTION_MASK_ISA_MPX },
};
/* Flag options. */
@@ -2917,11 +2918,13 @@ set_ix86_tune_features (enum processor_type ix86_tune, bool dump)
attributes. */
static void
-ix86_option_override_internal (bool main_args_p)
+ix86_option_override_internal (bool main_args_p,
+ struct gcc_options *opts,
+ struct gcc_options *opts_set)
{
int i;
- unsigned int ix86_arch_mask, ix86_tune_mask;
- const bool ix86_tune_specified = (ix86_tune_string != NULL);
+ unsigned int ix86_arch_mask;
+ const bool ix86_tune_specified = (opts->x_ix86_tune_string != NULL);
const char *prefix;
const char *suffix;
const char *sw;
@@ -2970,6 +2973,7 @@ ix86_option_override_internal (bool main_args_p)
#define PTA_AVX512ER (HOST_WIDE_INT_1 << 41)
#define PTA_AVX512PF (HOST_WIDE_INT_1 << 42)
#define PTA_AVX512CD (HOST_WIDE_INT_1 << 43)
+#define PTA_MPX (HOST_WIDE_INT_1 << 44)
/* if this reaches 64, need to widen struct pta flags below */
@@ -3016,12 +3020,12 @@ ix86_option_override_internal (bool main_args_p)
{"corei7", PROCESSOR_COREI7, CPU_COREI7,
PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3 | PTA_SSSE3
| PTA_SSE4_1 | PTA_SSE4_2 | PTA_CX16 | PTA_POPCNT | PTA_FXSR},
- {"corei7-avx", PROCESSOR_COREI7, CPU_COREI7,
+ {"corei7-avx", PROCESSOR_COREI7_AVX, CPU_COREI7,
PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3
| PTA_SSSE3 | PTA_SSE4_1 | PTA_SSE4_2 | PTA_AVX
| PTA_CX16 | PTA_POPCNT | PTA_AES | PTA_PCLMUL
| PTA_FXSR | PTA_XSAVE | PTA_XSAVEOPT},
- {"core-avx-i", PROCESSOR_COREI7, CPU_COREI7,
+ {"core-avx-i", PROCESSOR_COREI7_AVX, CPU_COREI7,
PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3
| PTA_SSSE3 | PTA_SSE4_1 | PTA_SSE4_2 | PTA_AVX
| PTA_CX16 | PTA_POPCNT | PTA_AES | PTA_PCLMUL | PTA_FSGSBASE
@@ -3098,7 +3102,7 @@ ix86_option_override_internal (bool main_args_p)
{"bdver3", PROCESSOR_BDVER3, CPU_BDVER3,
PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3
| PTA_SSE4A | PTA_CX16 | PTA_ABM | PTA_SSSE3 | PTA_SSE4_1
- | PTA_SSE4_2 | PTA_AES | PTA_PCLMUL | PTA_AVX
+ | PTA_SSE4_2 | PTA_AES | PTA_PCLMUL | PTA_AVX | PTA_FMA4
| PTA_XOP | PTA_LWP | PTA_BMI | PTA_TBM | PTA_F16C
| PTA_FMA | PTA_PRFCHW | PTA_FXSR | PTA_XSAVE
| PTA_XSAVEOPT | PTA_FSGSBASE},
@@ -3153,8 +3157,8 @@ ix86_option_override_internal (bool main_args_p)
/* Turn off both OPTION_MASK_ABI_64 and OPTION_MASK_ABI_X32 if
TARGET_64BIT_DEFAULT is true and TARGET_64BIT is false. */
- if (TARGET_64BIT_DEFAULT && !TARGET_64BIT)
- ix86_isa_flags &= ~(OPTION_MASK_ABI_64 | OPTION_MASK_ABI_X32);
+ if (TARGET_64BIT_DEFAULT && !TARGET_64BIT_P (opts->x_ix86_isa_flags))
+ opts->x_ix86_isa_flags &= ~(OPTION_MASK_ABI_64 | OPTION_MASK_ABI_X32);
#ifdef TARGET_BI_ARCH
else
{
@@ -3163,32 +3167,32 @@ ix86_option_override_internal (bool main_args_p)
is on and OPTION_MASK_ABI_X32 is off. We turn off
OPTION_MASK_ABI_64 if OPTION_MASK_ABI_X32 is turned on by
-mx32. */
- if (TARGET_X32)
- ix86_isa_flags &= ~OPTION_MASK_ABI_64;
+ if (TARGET_X32_P (opts->x_ix86_isa_flags))
+ opts->x_ix86_isa_flags &= ~OPTION_MASK_ABI_64;
#else
/* When TARGET_BI_ARCH == 2, by default, OPTION_MASK_ABI_X32 is
on and OPTION_MASK_ABI_64 is off. We turn off
OPTION_MASK_ABI_X32 if OPTION_MASK_ABI_64 is turned on by
-m64. */
- if (TARGET_LP64)
- ix86_isa_flags &= ~OPTION_MASK_ABI_X32;
+ if (TARGET_LP64_P (opts->x_ix86_isa_flags))
+ opts->x_ix86_isa_flags &= ~OPTION_MASK_ABI_X32;
#endif
}
#endif
- if (TARGET_X32)
+ if (TARGET_X32_P (opts->x_ix86_isa_flags))
{
/* Always turn on OPTION_MASK_ISA_64BIT and turn off
OPTION_MASK_ABI_64 for TARGET_X32. */
- ix86_isa_flags |= OPTION_MASK_ISA_64BIT;
- ix86_isa_flags &= ~OPTION_MASK_ABI_64;
+ opts->x_ix86_isa_flags |= OPTION_MASK_ISA_64BIT;
+ opts->x_ix86_isa_flags &= ~OPTION_MASK_ABI_64;
}
- else if (TARGET_LP64)
+ else if (TARGET_LP64_P (opts->x_ix86_isa_flags))
{
/* Always turn on OPTION_MASK_ISA_64BIT and turn off
OPTION_MASK_ABI_X32 for TARGET_LP64. */
- ix86_isa_flags |= OPTION_MASK_ISA_64BIT;
- ix86_isa_flags &= ~OPTION_MASK_ABI_X32;
+ opts->x_ix86_isa_flags |= OPTION_MASK_ISA_64BIT;
+ opts->x_ix86_isa_flags &= ~OPTION_MASK_ABI_X32;
}
#ifdef SUBTARGET_OVERRIDE_OPTIONS
@@ -3200,138 +3204,144 @@ ix86_option_override_internal (bool main_args_p)
#endif
/* -fPIC is the default for x86_64. */
- if (TARGET_MACHO && TARGET_64BIT)
- flag_pic = 2;
+ if (TARGET_MACHO && TARGET_64BIT_P (opts->x_ix86_isa_flags))
+ opts->x_flag_pic = 2;
/* Need to check -mtune=generic first. */
- if (ix86_tune_string)
+ if (opts->x_ix86_tune_string)
{
- if (!strcmp (ix86_tune_string, "generic")
- || !strcmp (ix86_tune_string, "i686")
+ if (!strcmp (opts->x_ix86_tune_string, "generic")
+ || !strcmp (opts->x_ix86_tune_string, "i686")
/* As special support for cross compilers we read -mtune=native
as -mtune=generic. With native compilers we won't see the
-mtune=native, as it was changed by the driver. */
- || !strcmp (ix86_tune_string, "native"))
+ || !strcmp (opts->x_ix86_tune_string, "native"))
{
- ix86_tune_string = "generic";
+ opts->x_ix86_tune_string = "generic";
}
/* If this call is for setting the option attribute, allow the
generic that was previously set. */
else if (!main_args_p
- && !strcmp (ix86_tune_string, "generic"))
+ && !strcmp (opts->x_ix86_tune_string, "generic"))
;
- else if (!strncmp (ix86_tune_string, "generic", 7))
+ else if (!strncmp (opts->x_ix86_tune_string, "generic", 7))
error ("bad value (%s) for %stune=%s %s",
- ix86_tune_string, prefix, suffix, sw);
- else if (!strcmp (ix86_tune_string, "x86-64"))
+ opts->x_ix86_tune_string, prefix, suffix, sw);
+ else if (!strcmp (opts->x_ix86_tune_string, "x86-64"))
warning (OPT_Wdeprecated, "%stune=x86-64%s is deprecated; use "
"%stune=k8%s or %stune=generic%s instead as appropriate",
prefix, suffix, prefix, suffix, prefix, suffix);
}
else
{
- if (ix86_arch_string)
- ix86_tune_string = ix86_arch_string;
- if (!ix86_tune_string)
+ if (opts->x_ix86_arch_string)
+ opts->x_ix86_tune_string = opts->x_ix86_arch_string;
+ if (!opts->x_ix86_tune_string)
{
- ix86_tune_string = cpu_names[TARGET_CPU_DEFAULT];
+ opts->x_ix86_tune_string = cpu_names[TARGET_CPU_DEFAULT];
ix86_tune_defaulted = 1;
}
- /* ix86_tune_string is set to ix86_arch_string or defaulted. We
- need to use a sensible tune option. */
- if (!strcmp (ix86_tune_string, "generic")
- || !strcmp (ix86_tune_string, "x86-64")
- || !strcmp (ix86_tune_string, "i686"))
+ /* opts->x_ix86_tune_string is set to opts->x_ix86_arch_string
+ or defaulted. We need to use a sensible tune option. */
+ if (!strcmp (opts->x_ix86_tune_string, "generic")
+ || !strcmp (opts->x_ix86_tune_string, "x86-64")
+ || !strcmp (opts->x_ix86_tune_string, "i686"))
{
- ix86_tune_string = "generic";
+ opts->x_ix86_tune_string = "generic";
}
}
- if (ix86_stringop_alg == rep_prefix_8_byte && !TARGET_64BIT)
+ if (opts->x_ix86_stringop_alg == rep_prefix_8_byte
+ && !TARGET_64BIT_P (opts->x_ix86_isa_flags))
{
/* rep; movq isn't available in 32-bit code. */
error ("-mstringop-strategy=rep_8byte not supported for 32-bit code");
- ix86_stringop_alg = no_stringop;
+ opts->x_ix86_stringop_alg = no_stringop;
}
- if (!ix86_arch_string)
- ix86_arch_string = TARGET_64BIT ? "x86-64" : SUBTARGET32_DEFAULT_CPU;
+ if (!opts->x_ix86_arch_string)
+ opts->x_ix86_arch_string
+ = TARGET_64BIT_P (opts->x_ix86_isa_flags)
+ ? "x86-64" : SUBTARGET32_DEFAULT_CPU;
else
ix86_arch_specified = 1;
- if (global_options_set.x_ix86_pmode)
+ if (opts_set->x_ix86_pmode)
{
- if ((TARGET_LP64 && ix86_pmode == PMODE_SI)
- || (!TARGET_64BIT && ix86_pmode == PMODE_DI))
+ if ((TARGET_LP64_P (opts->x_ix86_isa_flags)
+ && opts->x_ix86_pmode == PMODE_SI)
+ || (!TARGET_64BIT_P (opts->x_ix86_isa_flags)
+ && opts->x_ix86_pmode == PMODE_DI))
error ("address mode %qs not supported in the %s bit mode",
- TARGET_64BIT ? "short" : "long",
- TARGET_64BIT ? "64" : "32");
+ TARGET_64BIT_P (opts->x_ix86_isa_flags) ? "short" : "long",
+ TARGET_64BIT_P (opts->x_ix86_isa_flags) ? "64" : "32");
}
else
- ix86_pmode = TARGET_LP64 ? PMODE_DI : PMODE_SI;
+ opts->x_ix86_pmode = TARGET_LP64_P (opts->x_ix86_isa_flags)
+ ? PMODE_DI : PMODE_SI;
- if (!global_options_set.x_ix86_abi)
- ix86_abi = DEFAULT_ABI;
+ if (!opts_set->x_ix86_abi)
+ opts->x_ix86_abi = DEFAULT_ABI;
/* For targets using ms ABI enable ms-extensions, if not
explicit turned off. For non-ms ABI we turn off this
option. */
- if (!global_options_set.x_flag_ms_extensions)
- flag_ms_extensions = (MS_ABI == DEFAULT_ABI);
+ if (!opts_set->x_flag_ms_extensions)
+ opts->x_flag_ms_extensions = (MS_ABI == DEFAULT_ABI);
- if (global_options_set.x_ix86_cmodel)
+ if (opts_set->x_ix86_cmodel)
{
- switch (ix86_cmodel)
+ switch (opts->x_ix86_cmodel)
{
case CM_SMALL:
case CM_SMALL_PIC:
- if (flag_pic)
- ix86_cmodel = CM_SMALL_PIC;
- if (!TARGET_64BIT)
+ if (opts->x_flag_pic)
+ opts->x_ix86_cmodel = CM_SMALL_PIC;
+ if (!TARGET_64BIT_P (opts->x_ix86_isa_flags))
error ("code model %qs not supported in the %s bit mode",
"small", "32");
break;
case CM_MEDIUM:
case CM_MEDIUM_PIC:
- if (flag_pic)
- ix86_cmodel = CM_MEDIUM_PIC;
- if (!TARGET_64BIT)
+ if (opts->x_flag_pic)
+ opts->x_ix86_cmodel = CM_MEDIUM_PIC;
+ if (!TARGET_64BIT_P (opts->x_ix86_isa_flags))
error ("code model %qs not supported in the %s bit mode",
"medium", "32");
- else if (TARGET_X32)
+ else if (TARGET_X32_P (opts->x_ix86_isa_flags))
error ("code model %qs not supported in x32 mode",
"medium");
break;
case CM_LARGE:
case CM_LARGE_PIC:
- if (flag_pic)
- ix86_cmodel = CM_LARGE_PIC;
- if (!TARGET_64BIT)
+ if (opts->x_flag_pic)
+ opts->x_ix86_cmodel = CM_LARGE_PIC;
+ if (!TARGET_64BIT_P (opts->x_ix86_isa_flags))
error ("code model %qs not supported in the %s bit mode",
"large", "32");
- else if (TARGET_X32)
+ else if (TARGET_X32_P (opts->x_ix86_isa_flags))
error ("code model %qs not supported in x32 mode",
"large");
break;
case CM_32:
- if (flag_pic)
+ if (opts->x_flag_pic)
error ("code model %s does not support PIC mode", "32");
- if (TARGET_64BIT)
+ if (TARGET_64BIT_P (opts->x_ix86_isa_flags))
error ("code model %qs not supported in the %s bit mode",
"32", "64");
break;
case CM_KERNEL:
- if (flag_pic)
+ if (opts->x_flag_pic)
{
error ("code model %s does not support PIC mode", "kernel");
- ix86_cmodel = CM_32;
+ opts->x_ix86_cmodel = CM_32;
}
- if (!TARGET_64BIT)
+ if (!TARGET_64BIT_P (opts->x_ix86_isa_flags))
error ("code model %qs not supported in the %s bit mode",
"kernel", "32");
break;
@@ -3346,191 +3356,195 @@ ix86_option_override_internal (bool main_args_p)
use of rip-relative addressing. This eliminates fixups that
would otherwise be needed if this object is to be placed in a
DLL, and is essentially just as efficient as direct addressing. */
- if (TARGET_64BIT && (TARGET_RDOS || TARGET_PECOFF))
- ix86_cmodel = CM_MEDIUM_PIC, flag_pic = 1;
- else if (TARGET_64BIT)
- ix86_cmodel = flag_pic ? CM_SMALL_PIC : CM_SMALL;
+ if (TARGET_64BIT_P (opts->x_ix86_isa_flags)
+ && (TARGET_RDOS || TARGET_PECOFF))
+ opts->x_ix86_cmodel = CM_MEDIUM_PIC, opts->x_flag_pic = 1;
+ else if (TARGET_64BIT_P (opts->x_ix86_isa_flags))
+ opts->x_ix86_cmodel = opts->x_flag_pic ? CM_SMALL_PIC : CM_SMALL;
else
- ix86_cmodel = CM_32;
+ opts->x_ix86_cmodel = CM_32;
}
- if (TARGET_MACHO && ix86_asm_dialect == ASM_INTEL)
+ if (TARGET_MACHO && opts->x_ix86_asm_dialect == ASM_INTEL)
{
error ("-masm=intel not supported in this configuration");
- ix86_asm_dialect = ASM_ATT;
+ opts->x_ix86_asm_dialect = ASM_ATT;
}
- if ((TARGET_64BIT != 0) != ((ix86_isa_flags & OPTION_MASK_ISA_64BIT) != 0))
+ if ((TARGET_64BIT_P (opts->x_ix86_isa_flags) != 0)
+ != ((opts->x_ix86_isa_flags & OPTION_MASK_ISA_64BIT) != 0))
sorry ("%i-bit mode not compiled in",
- (ix86_isa_flags & OPTION_MASK_ISA_64BIT) ? 64 : 32);
+ (opts->x_ix86_isa_flags & OPTION_MASK_ISA_64BIT) ? 64 : 32);
for (i = 0; i < pta_size; i++)
- if (! strcmp (ix86_arch_string, processor_alias_table[i].name))
+ if (! strcmp (opts->x_ix86_arch_string, processor_alias_table[i].name))
{
ix86_schedule = processor_alias_table[i].schedule;
ix86_arch = processor_alias_table[i].processor;
/* Default cpu tuning to the architecture. */
ix86_tune = ix86_arch;
- if (TARGET_64BIT && !(processor_alias_table[i].flags & PTA_64BIT))
+ if (TARGET_64BIT_P (opts->x_ix86_isa_flags)
+ && !(processor_alias_table[i].flags & PTA_64BIT))
error ("CPU you selected does not support x86-64 "
"instruction set");
if (processor_alias_table[i].flags & PTA_MMX
- && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_MMX))
- ix86_isa_flags |= OPTION_MASK_ISA_MMX;
+ && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_MMX))
+ opts->x_ix86_isa_flags |= OPTION_MASK_ISA_MMX;
if (processor_alias_table[i].flags & PTA_3DNOW
- && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_3DNOW))
- ix86_isa_flags |= OPTION_MASK_ISA_3DNOW;
+ && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_3DNOW))
+ opts->x_ix86_isa_flags |= OPTION_MASK_ISA_3DNOW;
if (processor_alias_table[i].flags & PTA_3DNOW_A
- && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_3DNOW_A))
- ix86_isa_flags |= OPTION_MASK_ISA_3DNOW_A;
+ && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_3DNOW_A))
+ opts->x_ix86_isa_flags |= OPTION_MASK_ISA_3DNOW_A;
if (processor_alias_table[i].flags & PTA_SSE
- && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_SSE))
- ix86_isa_flags |= OPTION_MASK_ISA_SSE;
+ && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_SSE))
+ opts->x_ix86_isa_flags |= OPTION_MASK_ISA_SSE;
if (processor_alias_table[i].flags & PTA_SSE2
- && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_SSE2))
- ix86_isa_flags |= OPTION_MASK_ISA_SSE2;
+ && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_SSE2))
+ opts->x_ix86_isa_flags |= OPTION_MASK_ISA_SSE2;
if (processor_alias_table[i].flags & PTA_SSE3
- && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_SSE3))
- ix86_isa_flags |= OPTION_MASK_ISA_SSE3;
+ && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_SSE3))
+ opts->x_ix86_isa_flags |= OPTION_MASK_ISA_SSE3;
if (processor_alias_table[i].flags & PTA_SSSE3
- && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_SSSE3))
- ix86_isa_flags |= OPTION_MASK_ISA_SSSE3;
+ && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_SSSE3))
+ opts->x_ix86_isa_flags |= OPTION_MASK_ISA_SSSE3;
if (processor_alias_table[i].flags & PTA_SSE4_1
- && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_SSE4_1))
- ix86_isa_flags |= OPTION_MASK_ISA_SSE4_1;
+ && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_SSE4_1))
+ opts->x_ix86_isa_flags |= OPTION_MASK_ISA_SSE4_1;
if (processor_alias_table[i].flags & PTA_SSE4_2
- && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_SSE4_2))
- ix86_isa_flags |= OPTION_MASK_ISA_SSE4_2;
+ && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_SSE4_2))
+ opts->x_ix86_isa_flags |= OPTION_MASK_ISA_SSE4_2;
if (processor_alias_table[i].flags & PTA_AVX
- && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX))
- ix86_isa_flags |= OPTION_MASK_ISA_AVX;
+ && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX))
+ opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX;
if (processor_alias_table[i].flags & PTA_AVX2
- && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX2))
- ix86_isa_flags |= OPTION_MASK_ISA_AVX2;
+ && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX2))
+ opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX2;
if (processor_alias_table[i].flags & PTA_FMA
- && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_FMA))
- ix86_isa_flags |= OPTION_MASK_ISA_FMA;
+ && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_FMA))
+ opts->x_ix86_isa_flags |= OPTION_MASK_ISA_FMA;
if (processor_alias_table[i].flags & PTA_SSE4A
- && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_SSE4A))
- ix86_isa_flags |= OPTION_MASK_ISA_SSE4A;
+ && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_SSE4A))
+ opts->x_ix86_isa_flags |= OPTION_MASK_ISA_SSE4A;
if (processor_alias_table[i].flags & PTA_FMA4
- && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_FMA4))
- ix86_isa_flags |= OPTION_MASK_ISA_FMA4;
+ && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_FMA4))
+ opts->x_ix86_isa_flags |= OPTION_MASK_ISA_FMA4;
if (processor_alias_table[i].flags & PTA_XOP
- && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_XOP))
- ix86_isa_flags |= OPTION_MASK_ISA_XOP;
+ && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_XOP))
+ opts->x_ix86_isa_flags |= OPTION_MASK_ISA_XOP;
if (processor_alias_table[i].flags & PTA_LWP
- && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_LWP))
- ix86_isa_flags |= OPTION_MASK_ISA_LWP;
+ && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_LWP))
+ opts->x_ix86_isa_flags |= OPTION_MASK_ISA_LWP;
if (processor_alias_table[i].flags & PTA_ABM
- && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_ABM))
- ix86_isa_flags |= OPTION_MASK_ISA_ABM;
+ && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_ABM))
+ opts->x_ix86_isa_flags |= OPTION_MASK_ISA_ABM;
if (processor_alias_table[i].flags & PTA_BMI
- && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_BMI))
- ix86_isa_flags |= OPTION_MASK_ISA_BMI;
+ && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_BMI))
+ opts->x_ix86_isa_flags |= OPTION_MASK_ISA_BMI;
if (processor_alias_table[i].flags & (PTA_LZCNT | PTA_ABM)
- && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_LZCNT))
- ix86_isa_flags |= OPTION_MASK_ISA_LZCNT;
+ && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_LZCNT))
+ opts->x_ix86_isa_flags |= OPTION_MASK_ISA_LZCNT;
if (processor_alias_table[i].flags & PTA_TBM
- && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_TBM))
- ix86_isa_flags |= OPTION_MASK_ISA_TBM;
+ && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_TBM))
+ opts->x_ix86_isa_flags |= OPTION_MASK_ISA_TBM;
if (processor_alias_table[i].flags & PTA_BMI2
- && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_BMI2))
- ix86_isa_flags |= OPTION_MASK_ISA_BMI2;
+ && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_BMI2))
+ opts->x_ix86_isa_flags |= OPTION_MASK_ISA_BMI2;
if (processor_alias_table[i].flags & PTA_CX16
- && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_CX16))
- ix86_isa_flags |= OPTION_MASK_ISA_CX16;
+ && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_CX16))
+ opts->x_ix86_isa_flags |= OPTION_MASK_ISA_CX16;
if (processor_alias_table[i].flags & (PTA_POPCNT | PTA_ABM)
- && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_POPCNT))
- ix86_isa_flags |= OPTION_MASK_ISA_POPCNT;
- if (!(TARGET_64BIT && (processor_alias_table[i].flags & PTA_NO_SAHF))
- && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_SAHF))
- ix86_isa_flags |= OPTION_MASK_ISA_SAHF;
+ && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_POPCNT))
+ opts->x_ix86_isa_flags |= OPTION_MASK_ISA_POPCNT;
+ if (!(TARGET_64BIT_P (opts->x_ix86_isa_flags)
+ && (processor_alias_table[i].flags & PTA_NO_SAHF))
+ && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_SAHF))
+ opts->x_ix86_isa_flags |= OPTION_MASK_ISA_SAHF;
if (processor_alias_table[i].flags & PTA_MOVBE
- && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_MOVBE))
- ix86_isa_flags |= OPTION_MASK_ISA_MOVBE;
+ && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_MOVBE))
+ opts->x_ix86_isa_flags |= OPTION_MASK_ISA_MOVBE;
if (processor_alias_table[i].flags & PTA_AES
- && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_AES))
- ix86_isa_flags |= OPTION_MASK_ISA_AES;
+ && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_AES))
+ opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AES;
if (processor_alias_table[i].flags & PTA_PCLMUL
- && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_PCLMUL))
- ix86_isa_flags |= OPTION_MASK_ISA_PCLMUL;
+ && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_PCLMUL))
+ opts->x_ix86_isa_flags |= OPTION_MASK_ISA_PCLMUL;
if (processor_alias_table[i].flags & PTA_FSGSBASE
- && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_FSGSBASE))
- ix86_isa_flags |= OPTION_MASK_ISA_FSGSBASE;
+ && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_FSGSBASE))
+ opts->x_ix86_isa_flags |= OPTION_MASK_ISA_FSGSBASE;
if (processor_alias_table[i].flags & PTA_RDRND
- && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_RDRND))
- ix86_isa_flags |= OPTION_MASK_ISA_RDRND;
+ && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_RDRND))
+ opts->x_ix86_isa_flags |= OPTION_MASK_ISA_RDRND;
if (processor_alias_table[i].flags & PTA_F16C
- && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_F16C))
- ix86_isa_flags |= OPTION_MASK_ISA_F16C;
+ && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_F16C))
+ opts->x_ix86_isa_flags |= OPTION_MASK_ISA_F16C;
if (processor_alias_table[i].flags & PTA_RTM
- && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_RTM))
- ix86_isa_flags |= OPTION_MASK_ISA_RTM;
+ && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_RTM))
+ opts->x_ix86_isa_flags |= OPTION_MASK_ISA_RTM;
if (processor_alias_table[i].flags & PTA_HLE
- && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_HLE))
- ix86_isa_flags |= OPTION_MASK_ISA_HLE;
+ && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_HLE))
+ opts->x_ix86_isa_flags |= OPTION_MASK_ISA_HLE;
if (processor_alias_table[i].flags & PTA_PRFCHW
- && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_PRFCHW))
- ix86_isa_flags |= OPTION_MASK_ISA_PRFCHW;
+ && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_PRFCHW))
+ opts->x_ix86_isa_flags |= OPTION_MASK_ISA_PRFCHW;
if (processor_alias_table[i].flags & PTA_RDSEED
- && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_RDSEED))
- ix86_isa_flags |= OPTION_MASK_ISA_RDSEED;
+ && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_RDSEED))
+ opts->x_ix86_isa_flags |= OPTION_MASK_ISA_RDSEED;
if (processor_alias_table[i].flags & PTA_ADX
- && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_ADX))
- ix86_isa_flags |= OPTION_MASK_ISA_ADX;
+ && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_ADX))
+ opts->x_ix86_isa_flags |= OPTION_MASK_ISA_ADX;
if (processor_alias_table[i].flags & PTA_FXSR
- && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_FXSR))
- ix86_isa_flags |= OPTION_MASK_ISA_FXSR;
+ && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_FXSR))
+ opts->x_ix86_isa_flags |= OPTION_MASK_ISA_FXSR;
if (processor_alias_table[i].flags & PTA_XSAVE
- && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_XSAVE))
- ix86_isa_flags |= OPTION_MASK_ISA_XSAVE;
+ && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_XSAVE))
+ opts->x_ix86_isa_flags |= OPTION_MASK_ISA_XSAVE;
if (processor_alias_table[i].flags & PTA_XSAVEOPT
- && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_XSAVEOPT))
- ix86_isa_flags |= OPTION_MASK_ISA_XSAVEOPT;
+ && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_XSAVEOPT))
+ opts->x_ix86_isa_flags |= OPTION_MASK_ISA_XSAVEOPT;
if (processor_alias_table[i].flags & PTA_AVX512F
- && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX512F))
- ix86_isa_flags |= OPTION_MASK_ISA_AVX512F;
+ && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX512F))
+ opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX512F;
if (processor_alias_table[i].flags & PTA_AVX512ER
- && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX512ER))
- ix86_isa_flags |= OPTION_MASK_ISA_AVX512ER;
+ && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX512ER))
+ opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX512ER;
if (processor_alias_table[i].flags & PTA_AVX512PF
- && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX512PF))
- ix86_isa_flags |= OPTION_MASK_ISA_AVX512PF;
+ && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX512PF))
+ opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX512PF;
if (processor_alias_table[i].flags & PTA_AVX512CD
- && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX512CD))
- ix86_isa_flags |= OPTION_MASK_ISA_AVX512CD;
+ && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX512CD))
+ opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX512CD;
if (processor_alias_table[i].flags & (PTA_PREFETCH_SSE | PTA_SSE))
x86_prefetch_sse = true;
break;
}
- if (!strcmp (ix86_arch_string, "generic"))
+ if (!strcmp (opts->x_ix86_arch_string, "generic"))
error ("generic CPU can be used only for %stune=%s %s",
prefix, suffix, sw);
- else if (!strncmp (ix86_arch_string, "generic", 7) || i == pta_size)
+ else if (!strncmp (opts->x_ix86_arch_string, "generic", 7) || i == pta_size)
error ("bad value (%s) for %sarch=%s %s",
- ix86_arch_string, prefix, suffix, sw);
+ opts->x_ix86_arch_string, prefix, suffix, sw);
ix86_arch_mask = 1u << ix86_arch;
for (i = 0; i < X86_ARCH_LAST; ++i)
ix86_arch_features[i] = !!(initial_ix86_arch_features[i] & ix86_arch_mask);
for (i = 0; i < pta_size; i++)
- if (! strcmp (ix86_tune_string, processor_alias_table[i].name))
+ if (! strcmp (opts->x_ix86_tune_string, processor_alias_table[i].name))
{
ix86_schedule = processor_alias_table[i].schedule;
ix86_tune = processor_alias_table[i].processor;
- if (TARGET_64BIT)
+ if (TARGET_64BIT_P (opts->x_ix86_isa_flags))
{
if (!(processor_alias_table[i].flags & PTA_64BIT))
{
if (ix86_tune_defaulted)
{
- ix86_tune_string = "x86-64";
+ opts->x_ix86_tune_string = "x86-64";
for (i = 0; i < pta_size; i++)
- if (! strcmp (ix86_tune_string,
+ if (! strcmp (opts->x_ix86_tune_string,
processor_alias_table[i].name))
break;
ix86_schedule = processor_alias_table[i].schedule;
@@ -3554,9 +3568,9 @@ ix86_option_override_internal (bool main_args_p)
if (ix86_tune_specified && i == pta_size)
error ("bad value (%s) for %stune=%s %s",
- ix86_tune_string, prefix, suffix, sw);
+ opts->x_ix86_tune_string, prefix, suffix, sw);
- set_ix86_tune_features (ix86_tune, ix86_dump_tunes);
+ set_ix86_tune_features (ix86_tune, opts->x_ix86_dump_tunes);
#ifndef USE_IX86_FRAME_POINTER
#define USE_IX86_FRAME_POINTER 0
@@ -3568,27 +3582,29 @@ ix86_option_override_internal (bool main_args_p)
/* Set the default values for switches whose default depends on TARGET_64BIT
in case they weren't overwritten by command line options. */
- if (TARGET_64BIT)
+ if (TARGET_64BIT_P (opts->x_ix86_isa_flags))
{
- if (optimize >= 1 && !global_options_set.x_flag_omit_frame_pointer)
- flag_omit_frame_pointer = !USE_X86_64_FRAME_POINTER;
- if (flag_asynchronous_unwind_tables == 2)
- flag_unwind_tables = flag_asynchronous_unwind_tables = 1;
- if (flag_pcc_struct_return == 2)
- flag_pcc_struct_return = 0;
+ if (opts->x_optimize >= 1 && !opts_set->x_flag_omit_frame_pointer)
+ opts->x_flag_omit_frame_pointer = !USE_X86_64_FRAME_POINTER;
+ if (opts->x_flag_asynchronous_unwind_tables == 2)
+ opts->x_flag_unwind_tables
+ = opts->x_flag_asynchronous_unwind_tables = 1;
+ if (opts->x_flag_pcc_struct_return == 2)
+ opts->x_flag_pcc_struct_return = 0;
}
else
{
- if (optimize >= 1 && !global_options_set.x_flag_omit_frame_pointer)
- flag_omit_frame_pointer = !(USE_IX86_FRAME_POINTER || optimize_size);
- if (flag_asynchronous_unwind_tables == 2)
- flag_asynchronous_unwind_tables = !USE_IX86_FRAME_POINTER;
- if (flag_pcc_struct_return == 2)
- flag_pcc_struct_return = DEFAULT_PCC_STRUCT_RETURN;
+ if (opts->x_optimize >= 1 && !opts_set->x_flag_omit_frame_pointer)
+ opts->x_flag_omit_frame_pointer
+ = !(USE_IX86_FRAME_POINTER || opts->x_optimize_size);
+ if (opts->x_flag_asynchronous_unwind_tables == 2)
+ opts->x_flag_asynchronous_unwind_tables = !USE_IX86_FRAME_POINTER;
+ if (opts->x_flag_pcc_struct_return == 2)
+ opts->x_flag_pcc_struct_return = DEFAULT_PCC_STRUCT_RETURN;
}
ix86_tune_cost = processor_target_table[ix86_tune].cost;
- if (optimize_size)
+ if (opts->x_optimize_size)
ix86_cost = &ix86_size_cost;
else
ix86_cost = ix86_tune_cost;
@@ -3597,148 +3613,160 @@ ix86_option_override_internal (bool main_args_p)
init_machine_status = ix86_init_machine_status;
/* Validate -mregparm= value. */
- if (global_options_set.x_ix86_regparm)
+ if (opts_set->x_ix86_regparm)
{
- if (TARGET_64BIT)
+ if (TARGET_64BIT_P (opts->x_ix86_isa_flags))
warning (0, "-mregparm is ignored in 64-bit mode");
- if (ix86_regparm > REGPARM_MAX)
+ if (opts->x_ix86_regparm > REGPARM_MAX)
{
error ("-mregparm=%d is not between 0 and %d",
- ix86_regparm, REGPARM_MAX);
- ix86_regparm = 0;
+ opts->x_ix86_regparm, REGPARM_MAX);
+ opts->x_ix86_regparm = 0;
}
}
- if (TARGET_64BIT)
- ix86_regparm = REGPARM_MAX;
+ if (TARGET_64BIT_P (opts->x_ix86_isa_flags))
+ opts->x_ix86_regparm = REGPARM_MAX;
/* Default align_* from the processor table. */
- if (align_loops == 0)
+ if (opts->x_align_loops == 0)
{
- align_loops = processor_target_table[ix86_tune].align_loop;
+ opts->x_align_loops = processor_target_table[ix86_tune].align_loop;
align_loops_max_skip = processor_target_table[ix86_tune].align_loop_max_skip;
}
- if (align_jumps == 0)
+ if (opts->x_align_jumps == 0)
{
- align_jumps = processor_target_table[ix86_tune].align_jump;
+ opts->x_align_jumps = processor_target_table[ix86_tune].align_jump;
align_jumps_max_skip = processor_target_table[ix86_tune].align_jump_max_skip;
}
- if (align_functions == 0)
+ if (opts->x_align_functions == 0)
{
- align_functions = processor_target_table[ix86_tune].align_func;
+ opts->x_align_functions = processor_target_table[ix86_tune].align_func;
}
/* Provide default for -mbranch-cost= value. */
- if (!global_options_set.x_ix86_branch_cost)
- ix86_branch_cost = ix86_cost->branch_cost;
+ if (!opts_set->x_ix86_branch_cost)
+ opts->x_ix86_branch_cost = ix86_cost->branch_cost;
- if (TARGET_64BIT)
+ if (TARGET_64BIT_P (opts->x_ix86_isa_flags))
{
- target_flags |= TARGET_SUBTARGET64_DEFAULT & ~target_flags_explicit;
+ opts->x_target_flags
+ |= TARGET_SUBTARGET64_DEFAULT & ~opts_set->x_target_flags;
/* Enable by default the SSE and MMX builtins. Do allow the user to
explicitly disable any of these. In particular, disabling SSE and
MMX for kernel code is extremely useful. */
if (!ix86_arch_specified)
- ix86_isa_flags
+ opts->x_ix86_isa_flags
|= ((OPTION_MASK_ISA_SSE2 | OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_MMX
- | TARGET_SUBTARGET64_ISA_DEFAULT) & ~ix86_isa_flags_explicit);
+ | TARGET_SUBTARGET64_ISA_DEFAULT)
+ & ~opts->x_ix86_isa_flags_explicit);
- if (TARGET_RTD)
+ if (TARGET_RTD_P (opts->x_target_flags))
warning (0, "%srtd%s is ignored in 64bit mode", prefix, suffix);
}
else
{
- target_flags |= TARGET_SUBTARGET32_DEFAULT & ~target_flags_explicit;
+ opts->x_target_flags
+ |= TARGET_SUBTARGET32_DEFAULT & ~opts_set->x_target_flags;
if (!ix86_arch_specified)
- ix86_isa_flags
- |= TARGET_SUBTARGET32_ISA_DEFAULT & ~ix86_isa_flags_explicit;
+ opts->x_ix86_isa_flags
+ |= TARGET_SUBTARGET32_ISA_DEFAULT & ~opts->x_ix86_isa_flags_explicit;
/* i386 ABI does not specify red zone. It still makes sense to use it
when programmer takes care to stack from being destroyed. */
- if (!(target_flags_explicit & MASK_NO_RED_ZONE))
- target_flags |= MASK_NO_RED_ZONE;
+ if (!(opts_set->x_target_flags & MASK_NO_RED_ZONE))
+ opts->x_target_flags |= MASK_NO_RED_ZONE;
}
/* Keep nonleaf frame pointers. */
- if (flag_omit_frame_pointer)
- target_flags &= ~MASK_OMIT_LEAF_FRAME_POINTER;
- else if (TARGET_OMIT_LEAF_FRAME_POINTER)
- flag_omit_frame_pointer = 1;
+ if (opts->x_flag_omit_frame_pointer)
+ opts->x_target_flags &= ~MASK_OMIT_LEAF_FRAME_POINTER;
+ else if (TARGET_OMIT_LEAF_FRAME_POINTER_P (opts->x_target_flags))
+ opts->x_flag_omit_frame_pointer = 1;
/* If we're doing fast math, we don't care about comparison order
wrt NaNs. This lets us use a shorter comparison sequence. */
- if (flag_finite_math_only)
- target_flags &= ~MASK_IEEE_FP;
+ if (opts->x_flag_finite_math_only)
+ opts->x_target_flags &= ~MASK_IEEE_FP;
/* If the architecture always has an FPU, turn off NO_FANCY_MATH_387,
since the insns won't need emulation. */
- if (x86_arch_always_fancy_math_387 & ix86_arch_mask)
- target_flags &= ~MASK_NO_FANCY_MATH_387;
+ if (ix86_tune_features [X86_TUNE_ALWAYS_FANCY_MATH_387])
+ opts->x_target_flags &= ~MASK_NO_FANCY_MATH_387;
/* Likewise, if the target doesn't have a 387, or we've specified
software floating point, don't use 387 inline intrinsics. */
- if (!TARGET_80387)
- target_flags |= MASK_NO_FANCY_MATH_387;
+ if (!TARGET_80387_P (opts->x_target_flags))
+ opts->x_target_flags |= MASK_NO_FANCY_MATH_387;
/* Turn on MMX builtins for -msse. */
- if (TARGET_SSE)
- ix86_isa_flags |= OPTION_MASK_ISA_MMX & ~ix86_isa_flags_explicit;
+ if (TARGET_SSE_P (opts->x_ix86_isa_flags))
+ opts->x_ix86_isa_flags
+ |= OPTION_MASK_ISA_MMX & ~opts->x_ix86_isa_flags_explicit;
/* Enable SSE prefetch. */
- if (TARGET_SSE || (TARGET_PRFCHW && !TARGET_3DNOW))
+ if (TARGET_SSE_P (opts->x_ix86_isa_flags)
+ || (TARGET_PRFCHW && !TARGET_3DNOW_P (opts->x_ix86_isa_flags)))
x86_prefetch_sse = true;
/* Enable prefetch{,w} instructions for -m3dnow. */
- if (TARGET_3DNOW)
- ix86_isa_flags |= OPTION_MASK_ISA_PRFCHW & ~ix86_isa_flags_explicit;
+ if (TARGET_3DNOW_P (opts->x_ix86_isa_flags))
+ opts->x_ix86_isa_flags
+ |= OPTION_MASK_ISA_PRFCHW & ~opts->x_ix86_isa_flags_explicit;
/* Enable popcnt instruction for -msse4.2 or -mabm. */
- if (TARGET_SSE4_2 || TARGET_ABM)
- ix86_isa_flags |= OPTION_MASK_ISA_POPCNT & ~ix86_isa_flags_explicit;
+ if (TARGET_SSE4_2_P (opts->x_ix86_isa_flags)
+ || TARGET_ABM_P (opts->x_ix86_isa_flags))
+ opts->x_ix86_isa_flags
+ |= OPTION_MASK_ISA_POPCNT & ~opts->x_ix86_isa_flags_explicit;
/* Enable lzcnt instruction for -mabm. */
- if (TARGET_ABM)
- ix86_isa_flags |= OPTION_MASK_ISA_LZCNT & ~ix86_isa_flags_explicit;
+ if (TARGET_ABM_P(opts->x_ix86_isa_flags))
+ opts->x_ix86_isa_flags
+ |= OPTION_MASK_ISA_LZCNT & ~opts->x_ix86_isa_flags_explicit;
/* Validate -mpreferred-stack-boundary= value or default it to
PREFERRED_STACK_BOUNDARY_DEFAULT. */
ix86_preferred_stack_boundary = PREFERRED_STACK_BOUNDARY_DEFAULT;
- if (global_options_set.x_ix86_preferred_stack_boundary_arg)
+ if (opts_set->x_ix86_preferred_stack_boundary_arg)
{
- int min = (TARGET_64BIT ? (TARGET_SSE ? 4 : 3) : 2);
+ int min = (TARGET_64BIT_P (opts->x_ix86_isa_flags)
+ ? (TARGET_SSE_P (opts->x_ix86_isa_flags) ? 4 : 3) : 2);
int max = (TARGET_SEH ? 4 : 12);
- if (ix86_preferred_stack_boundary_arg < min
- || ix86_preferred_stack_boundary_arg > max)
+ if (opts->x_ix86_preferred_stack_boundary_arg < min
+ || opts->x_ix86_preferred_stack_boundary_arg > max)
{
if (min == max)
error ("-mpreferred-stack-boundary is not supported "
"for this target");
else
error ("-mpreferred-stack-boundary=%d is not between %d and %d",
- ix86_preferred_stack_boundary_arg, min, max);
+ opts->x_ix86_preferred_stack_boundary_arg, min, max);
}
else
ix86_preferred_stack_boundary
- = (1 << ix86_preferred_stack_boundary_arg) * BITS_PER_UNIT;
+ = (1 << opts->x_ix86_preferred_stack_boundary_arg) * BITS_PER_UNIT;
}
/* Set the default value for -mstackrealign. */
- if (ix86_force_align_arg_pointer == -1)
- ix86_force_align_arg_pointer = STACK_REALIGN_DEFAULT;
+ if (opts->x_ix86_force_align_arg_pointer == -1)
+ opts->x_ix86_force_align_arg_pointer = STACK_REALIGN_DEFAULT;
ix86_default_incoming_stack_boundary = PREFERRED_STACK_BOUNDARY;
/* Validate -mincoming-stack-boundary= value or default it to
MIN_STACK_BOUNDARY/PREFERRED_STACK_BOUNDARY. */
ix86_incoming_stack_boundary = ix86_default_incoming_stack_boundary;
- if (global_options_set.x_ix86_incoming_stack_boundary_arg)
+ if (opts_set->x_ix86_incoming_stack_boundary_arg)
{
- if (ix86_incoming_stack_boundary_arg < (TARGET_64BIT ? 4 : 2)
+ if (ix86_incoming_stack_boundary_arg
+ < (TARGET_64BIT_P (opts->x_ix86_isa_flags) ? 4 : 2)
|| ix86_incoming_stack_boundary_arg > 12)
error ("-mincoming-stack-boundary=%d is not between %d and 12",
- ix86_incoming_stack_boundary_arg, TARGET_64BIT ? 4 : 2);
+ ix86_incoming_stack_boundary_arg,
+ TARGET_64BIT_P (opts->x_ix86_isa_flags) ? 4 : 2);
else
{
ix86_user_incoming_stack_boundary
@@ -3749,36 +3777,50 @@ ix86_option_override_internal (bool main_args_p)
}
/* Accept -msseregparm only if at least SSE support is enabled. */
- if (TARGET_SSEREGPARM
- && ! TARGET_SSE)
+ if (TARGET_SSEREGPARM_P (opts->x_target_flags)
+ && ! TARGET_SSE_P (opts->x_ix86_isa_flags))
error ("%ssseregparm%s used without SSE enabled", prefix, suffix);
- if (global_options_set.x_ix86_fpmath)
+ if (opts_set->x_ix86_fpmath)
{
- if (ix86_fpmath & FPMATH_SSE)
+ if (opts->x_ix86_fpmath & FPMATH_SSE)
{
- if (!TARGET_SSE)
+ if (!TARGET_SSE_P (opts->x_ix86_isa_flags))
{
warning (0, "SSE instruction set disabled, using 387 arithmetics");
- ix86_fpmath = FPMATH_387;
+ opts->x_ix86_fpmath = FPMATH_387;
}
- else if ((ix86_fpmath & FPMATH_387) && !TARGET_80387)
+ else if ((opts->x_ix86_fpmath & FPMATH_387)
+ && !TARGET_80387_P (opts->x_target_flags))
{
warning (0, "387 instruction set disabled, using SSE arithmetics");
- ix86_fpmath = FPMATH_SSE;
+ opts->x_ix86_fpmath = FPMATH_SSE;
}
}
}
+ /* For all chips supporting SSE2, -mfpmath=sse performs better than
+ fpmath=387. The second is however default at many targets since the
+ extra 80bit precision of temporaries is considered to be part of ABI.
+ Overwrite the default at least for -ffast-math.
+ TODO: -mfpmath=both seems to produce same performing code with bit
+ smaller binaries. It is however not clear if register allocation is
+ ready for this setting.
+ Also -mfpmath=387 is overall a lot more compact (bout 4-5%) than SSE
+ codegen. We may switch to 387 with -ffast-math for size optimized
+ functions. */
+ else if (fast_math_flags_set_p (&global_options)
+ && TARGET_SSE2_P (opts->x_ix86_isa_flags))
+ ix86_fpmath = FPMATH_SSE;
else
- ix86_fpmath = TARGET_FPMATH_DEFAULT;
+ opts->x_ix86_fpmath = TARGET_FPMATH_DEFAULT_P (opts->x_ix86_isa_flags);
/* If the i387 is disabled, then do not return values in it. */
- if (!TARGET_80387)
- target_flags &= ~MASK_FLOAT_RETURNS;
+ if (!TARGET_80387_P (opts->x_target_flags))
+ opts->x_target_flags &= ~MASK_FLOAT_RETURNS;
/* Use external vectorized library in vectorizing intrinsics. */
- if (global_options_set.x_ix86_veclibabi_type)
- switch (ix86_veclibabi_type)
+ if (opts_set->x_ix86_veclibabi_type)
+ switch (opts->x_ix86_veclibabi_type)
{
case ix86_veclibabi_type_svml:
ix86_veclib_handler = ix86_veclibabi_svml;
@@ -3792,39 +3834,21 @@ ix86_option_override_internal (bool main_args_p)
gcc_unreachable ();
}
- ix86_tune_mask = 1u << ix86_tune;
- if ((!USE_IX86_FRAME_POINTER
- || (x86_accumulate_outgoing_args & ix86_tune_mask))
- && !(target_flags_explicit & MASK_ACCUMULATE_OUTGOING_ARGS)
- && !optimize_size)
- target_flags |= MASK_ACCUMULATE_OUTGOING_ARGS;
-
- /* ??? Unwind info is not correct around the CFG unless either a frame
- pointer is present or M_A_O_A is set. Fixing this requires rewriting
- unwind info generation to be aware of the CFG and propagating states
- around edges. */
- if ((flag_unwind_tables || flag_asynchronous_unwind_tables
- || flag_exceptions || flag_non_call_exceptions)
- && flag_omit_frame_pointer
- && !(target_flags & MASK_ACCUMULATE_OUTGOING_ARGS))
- {
- if (target_flags_explicit & MASK_ACCUMULATE_OUTGOING_ARGS)
- warning (0, "unwind tables currently require either a frame pointer "
- "or %saccumulate-outgoing-args%s for correctness",
- prefix, suffix);
- target_flags |= MASK_ACCUMULATE_OUTGOING_ARGS;
- }
+ if (ix86_tune_features [X86_TUNE_ACCUMULATE_OUTGOING_ARGS]
+ && !(opts_set->x_target_flags & MASK_ACCUMULATE_OUTGOING_ARGS)
+ && !opts->x_optimize_size)
+ opts->x_target_flags |= MASK_ACCUMULATE_OUTGOING_ARGS;
/* If stack probes are required, the space used for large function
arguments on the stack must also be probed, so enable
-maccumulate-outgoing-args so this happens in the prologue. */
- if (TARGET_STACK_PROBE
- && !(target_flags & MASK_ACCUMULATE_OUTGOING_ARGS))
+ if (TARGET_STACK_PROBE_P (opts->x_target_flags)
+ && !(opts->x_target_flags & MASK_ACCUMULATE_OUTGOING_ARGS))
{
- if (target_flags_explicit & MASK_ACCUMULATE_OUTGOING_ARGS)
+ if (opts_set->x_target_flags & MASK_ACCUMULATE_OUTGOING_ARGS)
warning (0, "stack probing requires %saccumulate-outgoing-args%s "
"for correctness", prefix, suffix);
- target_flags |= MASK_ACCUMULATE_OUTGOING_ARGS;
+ opts->x_target_flags |= MASK_ACCUMULATE_OUTGOING_ARGS;
}
/* Figure out what ASM_GENERATE_INTERNAL_LABEL builds as a prefix. */
@@ -3839,38 +3863,38 @@ ix86_option_override_internal (bool main_args_p)
/* When scheduling description is not available, disable scheduler pass
so it won't slow down the compilation and make x87 code slower. */
if (!TARGET_SCHEDULE)
- flag_schedule_insns_after_reload = flag_schedule_insns = 0;
+ opts->x_flag_schedule_insns_after_reload = opts->x_flag_schedule_insns = 0;
maybe_set_param_value (PARAM_SIMULTANEOUS_PREFETCHES,
ix86_tune_cost->simultaneous_prefetches,
- global_options.x_param_values,
- global_options_set.x_param_values);
+ opts->x_param_values,
+ opts_set->x_param_values);
maybe_set_param_value (PARAM_L1_CACHE_LINE_SIZE,
ix86_tune_cost->prefetch_block,
- global_options.x_param_values,
- global_options_set.x_param_values);
+ opts->x_param_values,
+ opts_set->x_param_values);
maybe_set_param_value (PARAM_L1_CACHE_SIZE,
ix86_tune_cost->l1_cache_size,
- global_options.x_param_values,
- global_options_set.x_param_values);
+ opts->x_param_values,
+ opts_set->x_param_values);
maybe_set_param_value (PARAM_L2_CACHE_SIZE,
ix86_tune_cost->l2_cache_size,
- global_options.x_param_values,
- global_options_set.x_param_values);
+ opts->x_param_values,
+ opts_set->x_param_values);
/* Enable sw prefetching at -O3 for CPUS that prefetching is helpful. */
- if (flag_prefetch_loop_arrays < 0
+ if (opts->x_flag_prefetch_loop_arrays < 0
&& HAVE_prefetch
- && (optimize >= 3 || flag_profile_use)
+ && (opts->x_optimize >= 3 || opts->x_flag_profile_use)
&& TARGET_SOFTWARE_PREFETCHING_BENEFICIAL)
- flag_prefetch_loop_arrays = 1;
+ opts->x_flag_prefetch_loop_arrays = 1;
/* If using typedef char *va_list, signal that __builtin_va_start (&ap, 0)
- can be optimized to ap = __builtin_next_arg (0). */
- if (!TARGET_64BIT && !flag_split_stack)
+ can be opts->x_optimized to ap = __builtin_next_arg (0). */
+ if (!TARGET_64BIT_P (opts->x_ix86_isa_flags) && !opts->x_flag_split_stack)
targetm.expand_builtin_va_start = NULL;
- if (TARGET_64BIT)
+ if (TARGET_64BIT_P (opts->x_ix86_isa_flags))
{
ix86_gen_leave = gen_leave_rex64;
if (Pmode == DImode)
@@ -3916,56 +3940,56 @@ ix86_option_override_internal (bool main_args_p)
#ifdef USE_IX86_CLD
/* Use -mcld by default for 32-bit code if configured with --enable-cld. */
- if (!TARGET_64BIT)
- target_flags |= MASK_CLD & ~target_flags_explicit;
+ if (!TARGET_64BIT_P (opts->x_ix86_isa_flags))
+ opts->x_target_flags |= MASK_CLD & ~opts_set->x_target_flags;
#endif
- if (!TARGET_64BIT && flag_pic)
+ if (!TARGET_64BIT_P (opts->x_ix86_isa_flags) && opts->x_flag_pic)
{
- if (flag_fentry > 0)
+ if (opts->x_flag_fentry > 0)
sorry ("-mfentry isn%'t supported for 32-bit in combination "
"with -fpic");
- flag_fentry = 0;
+ opts->x_flag_fentry = 0;
}
else if (TARGET_SEH)
{
- if (flag_fentry == 0)
+ if (opts->x_flag_fentry == 0)
sorry ("-mno-fentry isn%'t compatible with SEH");
- flag_fentry = 1;
+ opts->x_flag_fentry = 1;
}
- else if (flag_fentry < 0)
+ else if (opts->x_flag_fentry < 0)
{
#if defined(PROFILE_BEFORE_PROLOGUE)
- flag_fentry = 1;
+ opts->x_flag_fentry = 1;
#else
- flag_fentry = 0;
+ opts->x_flag_fentry = 0;
#endif
}
- /* When not optimize for size, enable vzeroupper optimization for
+ /* When not opts->x_optimize for size, enable vzeroupper optimization for
TARGET_AVX with -fexpensive-optimizations and split 32-byte
AVX unaligned load/store. */
- if (!optimize_size)
+ if (!opts->x_optimize_size)
{
if (flag_expensive_optimizations
- && !(target_flags_explicit & MASK_VZEROUPPER))
- target_flags |= MASK_VZEROUPPER;
- if ((x86_avx256_split_unaligned_load & ix86_tune_mask)
- && !(target_flags_explicit & MASK_AVX256_SPLIT_UNALIGNED_LOAD))
- target_flags |= MASK_AVX256_SPLIT_UNALIGNED_LOAD;
- if ((x86_avx256_split_unaligned_store & ix86_tune_mask)
- && !(target_flags_explicit & MASK_AVX256_SPLIT_UNALIGNED_STORE))
- target_flags |= MASK_AVX256_SPLIT_UNALIGNED_STORE;
+ && !(opts_set->x_target_flags & MASK_VZEROUPPER))
+ opts->x_target_flags |= MASK_VZEROUPPER;
+ if (!ix86_tune_features[X86_TUNE_SSE_UNALIGNED_LOAD_OPTIMAL]
+ && !(opts_set->x_target_flags & MASK_AVX256_SPLIT_UNALIGNED_LOAD))
+ opts->x_target_flags |= MASK_AVX256_SPLIT_UNALIGNED_LOAD;
+ if (!ix86_tune_features[X86_TUNE_SSE_UNALIGNED_STORE_OPTIMAL]
+ && !(opts_set->x_target_flags & MASK_AVX256_SPLIT_UNALIGNED_STORE))
+ opts->x_target_flags |= MASK_AVX256_SPLIT_UNALIGNED_STORE;
/* Enable 128-bit AVX instruction generation
for the auto-vectorizer. */
if (TARGET_AVX128_OPTIMAL
- && !(target_flags_explicit & MASK_PREFER_AVX128))
- target_flags |= MASK_PREFER_AVX128;
+ && !(opts_set->x_target_flags & MASK_PREFER_AVX128))
+ opts->x_target_flags |= MASK_PREFER_AVX128;
}
- if (ix86_recip_name)
+ if (opts->x_ix86_recip_name)
{
- char *p = ASTRDUP (ix86_recip_name);
+ char *p = ASTRDUP (opts->x_ix86_recip_name);
char *q;
unsigned int mask, i;
bool invert;
@@ -4000,45 +4024,46 @@ ix86_option_override_internal (bool main_args_p)
}
}
- recip_mask_explicit |= mask;
+ opts->x_recip_mask_explicit |= mask;
if (invert)
- recip_mask &= ~mask;
+ opts->x_recip_mask &= ~mask;
else
- recip_mask |= mask;
+ opts->x_recip_mask |= mask;
}
}
- if (TARGET_RECIP)
- recip_mask |= RECIP_MASK_ALL & ~recip_mask_explicit;
- else if (target_flags_explicit & MASK_RECIP)
- recip_mask &= ~(RECIP_MASK_ALL & ~recip_mask_explicit);
+ if (TARGET_RECIP_P (opts->x_target_flags))
+ opts->x_recip_mask |= RECIP_MASK_ALL & ~opts->x_recip_mask_explicit;
+ else if (opts_set->x_target_flags & MASK_RECIP)
+ opts->x_recip_mask &= ~(RECIP_MASK_ALL & ~opts->x_recip_mask_explicit);
/* Default long double to 64-bit for Bionic. */
if (TARGET_HAS_BIONIC
- && !(target_flags_explicit & MASK_LONG_DOUBLE_64))
- target_flags |= MASK_LONG_DOUBLE_64;
+ && !(opts_set->x_target_flags & MASK_LONG_DOUBLE_64))
+ opts->x_target_flags |= MASK_LONG_DOUBLE_64;
/* Save the initial options in case the user does function specific
options. */
if (main_args_p)
target_option_default_node = target_option_current_node
- = build_target_option_node ();
+ = build_target_option_node (opts);
/* Handle stack protector */
- if (!global_options_set.x_ix86_stack_protector_guard)
- ix86_stack_protector_guard = TARGET_HAS_BIONIC ? SSP_GLOBAL : SSP_TLS;
+ if (!opts_set->x_ix86_stack_protector_guard)
+ opts->x_ix86_stack_protector_guard
+ = TARGET_HAS_BIONIC ? SSP_GLOBAL : SSP_TLS;
/* Handle -mmemcpy-strategy= and -mmemset-strategy= */
- if (ix86_tune_memcpy_strategy)
+ if (opts->x_ix86_tune_memcpy_strategy)
{
- char *str = xstrdup (ix86_tune_memcpy_strategy);
+ char *str = xstrdup (opts->x_ix86_tune_memcpy_strategy);
ix86_parse_stringop_strategy_string (str, false);
free (str);
}
- if (ix86_tune_memset_strategy)
+ if (opts->x_ix86_tune_memset_strategy)
{
- char *str = xstrdup (ix86_tune_memset_strategy);
+ char *str = xstrdup (opts->x_ix86_tune_memset_strategy);
ix86_parse_stringop_strategy_string (str, true);
free (str);
}
@@ -4055,7 +4080,7 @@ ix86_option_override (void)
1, PASS_POS_INSERT_AFTER
};
- ix86_option_override_internal (true);
+ ix86_option_override_internal (true, &global_options, &global_options_set);
/* This needs to be done at start up. It's convenient to do it here. */
@@ -4134,13 +4159,19 @@ ix86_conditional_register_usage (void)
for (i = FIRST_MASK_REG; i <= LAST_MASK_REG; i++)
fixed_regs[i] = call_used_regs[i] = 1, reg_names[i] = "";
}
+
+ /* If MPX is disabled, squash the registers. */
+ if (! TARGET_MPX)
+ for (i = FIRST_BND_REG; i <= LAST_BND_REG; i++)
+ fixed_regs[i] = call_used_regs[i] = 1, reg_names[i] = "";
}
/* Save the current options */
static void
-ix86_function_specific_save (struct cl_target_option *ptr)
+ix86_function_specific_save (struct cl_target_option *ptr,
+ struct gcc_options *opts)
{
ptr->arch = ix86_arch;
ptr->schedule = ix86_schedule;
@@ -4148,9 +4179,9 @@ ix86_function_specific_save (struct cl_target_option *ptr)
ptr->branch_cost = ix86_branch_cost;
ptr->tune_defaulted = ix86_tune_defaulted;
ptr->arch_specified = ix86_arch_specified;
- ptr->x_ix86_isa_flags_explicit = ix86_isa_flags_explicit;
- ptr->ix86_target_flags_explicit = target_flags_explicit;
- ptr->x_recip_mask_explicit = recip_mask_explicit;
+ ptr->x_ix86_isa_flags_explicit = opts->x_ix86_isa_flags_explicit;
+ ptr->x_ix86_target_flags_explicit = opts->x_ix86_target_flags_explicit;
+ ptr->x_recip_mask_explicit = opts->x_recip_mask_explicit;
/* The fields are char but the variables are not; make sure the
values fit in the fields. */
@@ -4163,7 +4194,8 @@ ix86_function_specific_save (struct cl_target_option *ptr)
/* Restore the current options */
static void
-ix86_function_specific_restore (struct cl_target_option *ptr)
+ix86_function_specific_restore (struct gcc_options *opts,
+ struct cl_target_option *ptr)
{
enum processor_type old_tune = ix86_tune;
enum processor_type old_arch = ix86_arch;
@@ -4173,12 +4205,12 @@ ix86_function_specific_restore (struct cl_target_option *ptr)
ix86_arch = (enum processor_type) ptr->arch;
ix86_schedule = (enum attr_cpu) ptr->schedule;
ix86_tune = (enum processor_type) ptr->tune;
- ix86_branch_cost = ptr->branch_cost;
+ opts->x_ix86_branch_cost = ptr->branch_cost;
ix86_tune_defaulted = ptr->tune_defaulted;
ix86_arch_specified = ptr->arch_specified;
- ix86_isa_flags_explicit = ptr->x_ix86_isa_flags_explicit;
- target_flags_explicit = ptr->ix86_target_flags_explicit;
- recip_mask_explicit = ptr->x_recip_mask_explicit;
+ opts->x_ix86_isa_flags_explicit = ptr->x_ix86_isa_flags_explicit;
+ opts->x_ix86_target_flags_explicit = ptr->x_ix86_target_flags_explicit;
+ opts->x_recip_mask_explicit = ptr->x_recip_mask_explicit;
/* Recreate the arch feature tests if the arch changed */
if (old_arch != ix86_arch)
@@ -4234,6 +4266,8 @@ ix86_function_specific_print (FILE *file, int indent,
static bool
ix86_valid_target_attribute_inner_p (tree args, char *p_strings[],
+ struct gcc_options *opts,
+ struct gcc_options *opts_set,
struct gcc_options *enum_opts_set)
{
char *next_optstr;
@@ -4350,7 +4384,8 @@ ix86_valid_target_attribute_inner_p (tree args, char *p_strings[],
for (; args; args = TREE_CHAIN (args))
if (TREE_VALUE (args)
&& !ix86_valid_target_attribute_inner_p (TREE_VALUE (args),
- p_strings, enum_opts_set))
+ p_strings, opts, opts_set,
+ enum_opts_set))
ret = false;
return ret;
@@ -4433,7 +4468,7 @@ ix86_valid_target_attribute_inner_p (tree args, char *p_strings[],
struct cl_decoded_option decoded;
generate_option (opt, NULL, opt_set_p, CL_TARGET, &decoded);
- ix86_handle_option (&global_options, &global_options_set,
+ ix86_handle_option (opts, opts_set,
&decoded, input_location);
}
@@ -4443,9 +4478,9 @@ ix86_valid_target_attribute_inner_p (tree args, char *p_strings[],
opt_set_p = !opt_set_p;
if (opt_set_p)
- target_flags |= mask;
+ opts->x_target_flags |= mask;
else
- target_flags &= ~mask;
+ opts->x_target_flags &= ~mask;
}
else if (type == ix86_opt_str)
@@ -4466,7 +4501,7 @@ ix86_valid_target_attribute_inner_p (tree args, char *p_strings[],
arg_ok = opt_enum_arg_to_value (opt, p + opt_len, &value, CL_TARGET);
if (arg_ok)
- set_option (&global_options, enum_opts_set, opt, value,
+ set_option (opts, enum_opts_set, opt, value,
p + opt_len, DK_UNSPECIFIED, input_location,
global_dc);
else
@@ -4486,11 +4521,13 @@ ix86_valid_target_attribute_inner_p (tree args, char *p_strings[],
/* Return a TARGET_OPTION_NODE tree of the target options listed or NULL. */
tree
-ix86_valid_target_attribute_tree (tree args)
+ix86_valid_target_attribute_tree (tree args,
+ struct gcc_options *opts,
+ struct gcc_options *opts_set)
{
const char *orig_arch_string = ix86_arch_string;
const char *orig_tune_string = ix86_tune_string;
- enum fpmath_unit orig_fpmath_set = global_options_set.x_ix86_fpmath;
+ enum fpmath_unit orig_fpmath_set = opts_set->x_ix86_fpmath;
int orig_tune_defaulted = ix86_tune_defaulted;
int orig_arch_specified = ix86_arch_specified;
char *option_strings[IX86_FUNCTION_SPECIFIC_MAX] = { NULL, NULL };
@@ -4503,16 +4540,16 @@ ix86_valid_target_attribute_tree (tree args)
memset (&enum_opts_set, 0, sizeof (enum_opts_set));
/* Process each of the options on the chain. */
- if (! ix86_valid_target_attribute_inner_p (args, option_strings,
- &enum_opts_set))
+ if (! ix86_valid_target_attribute_inner_p (args, option_strings, opts,
+ opts_set, &enum_opts_set))
return error_mark_node;
/* If the changed options are different from the default, rerun
ix86_option_override_internal, and then save the options away.
The string options are are attribute options, and will be undone
when we copy the save structure. */
- if (ix86_isa_flags != def->x_ix86_isa_flags
- || target_flags != def->x_target_flags
+ if (opts->x_ix86_isa_flags != def->x_ix86_isa_flags
+ || opts->x_target_flags != def->x_target_flags
|| option_strings[IX86_FUNCTION_SPECIFIC_ARCH]
|| option_strings[IX86_FUNCTION_SPECIFIC_TUNE]
|| enum_opts_set.x_ix86_fpmath)
@@ -4520,37 +4557,38 @@ ix86_valid_target_attribute_tree (tree args)
/* If we are using the default tune= or arch=, undo the string assigned,
and use the default. */
if (option_strings[IX86_FUNCTION_SPECIFIC_ARCH])
- ix86_arch_string = option_strings[IX86_FUNCTION_SPECIFIC_ARCH];
+ opts->x_ix86_arch_string = option_strings[IX86_FUNCTION_SPECIFIC_ARCH];
else if (!orig_arch_specified)
- ix86_arch_string = NULL;
+ opts->x_ix86_arch_string = NULL;
if (option_strings[IX86_FUNCTION_SPECIFIC_TUNE])
- ix86_tune_string = option_strings[IX86_FUNCTION_SPECIFIC_TUNE];
+ opts->x_ix86_tune_string = option_strings[IX86_FUNCTION_SPECIFIC_TUNE];
else if (orig_tune_defaulted)
- ix86_tune_string = NULL;
+ opts->x_ix86_tune_string = NULL;
/* If fpmath= is not set, and we now have sse2 on 32-bit, use it. */
if (enum_opts_set.x_ix86_fpmath)
- global_options_set.x_ix86_fpmath = (enum fpmath_unit) 1;
- else if (!TARGET_64BIT && TARGET_SSE)
+ opts_set->x_ix86_fpmath = (enum fpmath_unit) 1;
+ else if (!TARGET_64BIT_P (opts->x_ix86_isa_flags)
+ && TARGET_SSE_P (opts->x_ix86_isa_flags))
{
- ix86_fpmath = (enum fpmath_unit) (FPMATH_SSE | FPMATH_387);
- global_options_set.x_ix86_fpmath = (enum fpmath_unit) 1;
+ opts->x_ix86_fpmath = (enum fpmath_unit) (FPMATH_SSE | FPMATH_387);
+ opts_set->x_ix86_fpmath = (enum fpmath_unit) 1;
}
/* Do any overrides, such as arch=xxx, or tune=xxx support. */
- ix86_option_override_internal (false);
+ ix86_option_override_internal (false, opts, opts_set);
/* Add any builtin functions with the new isa if any. */
- ix86_add_new_builtins (ix86_isa_flags);
+ ix86_add_new_builtins (opts->x_ix86_isa_flags);
/* Save the current options unless we are validating options for
#pragma. */
- t = build_target_option_node ();
+ t = build_target_option_node (opts);
- ix86_arch_string = orig_arch_string;
- ix86_tune_string = orig_tune_string;
- global_options_set.x_ix86_fpmath = orig_fpmath_set;
+ opts->x_ix86_arch_string = orig_arch_string;
+ opts->x_ix86_tune_string = orig_tune_string;
+ opts_set->x_ix86_fpmath = orig_fpmath_set;
/* Free up memory allocated to hold the strings */
for (i = 0; i < IX86_FUNCTION_SPECIFIC_MAX; i++)
@@ -4568,7 +4606,8 @@ ix86_valid_target_attribute_p (tree fndecl,
tree args,
int ARG_UNUSED (flags))
{
- struct cl_target_option cur_target;
+ struct gcc_options func_options;
+ tree new_target, new_optimize;
bool ret = true;
/* attribute((target("default"))) does nothing, beyond
@@ -4579,21 +4618,31 @@ ix86_valid_target_attribute_p (tree fndecl,
&& strcmp (TREE_STRING_POINTER (TREE_VALUE (args)), "default") == 0)
return true;
- tree old_optimize = build_optimization_node ();
- tree new_target, new_optimize;
+ tree old_optimize = build_optimization_node (&global_options);
+
+ /* Get the optimization options of the current function. */
tree func_optimize = DECL_FUNCTION_SPECIFIC_OPTIMIZATION (fndecl);
+
+ if (!func_optimize)
+ func_optimize = old_optimize;
- /* If the function changed the optimization levels as well as setting target
- options, start with the optimizations specified. */
- if (func_optimize && func_optimize != old_optimize)
- cl_optimization_restore (&global_options,
- TREE_OPTIMIZATION (func_optimize));
+ /* Init func_options. */
+ memset (&func_options, 0, sizeof (func_options));
+ init_options_struct (&func_options, NULL);
+ lang_hooks.init_options_struct (&func_options);
+
+ cl_optimization_restore (&func_options,
+ TREE_OPTIMIZATION (func_optimize));
+
+ /* Initialize func_options to the default before its target options can
+ be set. */
+ cl_target_option_restore (&func_options,
+ TREE_TARGET_OPTION (target_option_default_node));
+
+ new_target = ix86_valid_target_attribute_tree (args, &func_options,
+ &global_options_set);
- /* The target attributes may also change some optimization flags, so update
- the optimization options if necessary. */
- cl_target_option_save (&cur_target, &global_options);
- new_target = ix86_valid_target_attribute_tree (args);
- new_optimize = build_optimization_node ();
+ new_optimize = build_optimization_node (&func_options);
if (new_target == error_mark_node)
ret = false;
@@ -4606,12 +4655,6 @@ ix86_valid_target_attribute_p (tree fndecl,
DECL_FUNCTION_SPECIFIC_OPTIMIZATION (fndecl) = new_optimize;
}
- cl_target_option_restore (&global_options, &cur_target);
-
- if (old_optimize != new_optimize)
- cl_optimization_restore (&global_options,
- TREE_OPTIMIZATION (old_optimize));
-
return ret;
}
@@ -7367,9 +7410,15 @@ ix86_function_value_regno_p (const unsigned int regno)
switch (regno)
{
case AX_REG:
+ case DX_REG:
return true;
+ case DI_REG:
+ case SI_REG:
+ return TARGET_64BIT && ix86_abi != MS_ABI;
- case FIRST_FLOAT_REG:
+ /* Complex values are returned in %st(0)/%st(1) pair. */
+ case ST0_REG:
+ case ST1_REG:
/* TODO: The function should depend on current function ABI but
builtins.c would need updating then. Therefore we use the
default ABI. */
@@ -7377,10 +7426,12 @@ ix86_function_value_regno_p (const unsigned int regno)
return false;
return TARGET_FLOAT_RETURNS_IN_80387;
- case FIRST_SSE_REG:
+ /* Complex values are returned in %xmm0/%xmm1 pair. */
+ case XMM0_REG:
+ case XMM1_REG:
return TARGET_SSE;
- case FIRST_MMX_REG:
+ case MM0_REG:
if (TARGET_MACHO || TARGET_64BIT)
return false;
return TARGET_MMX;
@@ -8835,7 +8886,7 @@ ix86_code_end (void)
xops[0] = gen_rtx_REG (Pmode, regno);
xops[1] = gen_rtx_MEM (Pmode, stack_pointer_rtx);
output_asm_insn ("mov%z0\t{%1, %0|%0, %1}", xops);
- fputs ("\tret\n", asm_out_file);
+ output_asm_insn ("%!ret", NULL);
final_end_function ();
init_insn_lengths ();
free_after_compilation (cfun);
@@ -8893,7 +8944,7 @@ output_set_got (rtx dest, rtx label)
xops[2] = gen_rtx_SYMBOL_REF (Pmode, ggc_strdup (name));
xops[2] = gen_rtx_MEM (QImode, xops[2]);
- output_asm_insn ("call\t%X2", xops);
+ output_asm_insn ("%!call\t%X2", xops);
#if TARGET_MACHO
/* Output the Mach-O "canonical" pic base label name ("Lxx$pb") here.
@@ -10606,8 +10657,12 @@ ix86_expand_prologue (void)
if (STACK_CHECK_MOVING_SP)
{
- ix86_adjust_stack_and_probe (allocate);
- allocate = 0;
+ if (!(crtl->is_leaf && !cfun->calls_alloca
+ && allocate <= PROBE_INTERVAL))
+ {
+ ix86_adjust_stack_and_probe (allocate);
+ allocate = 0;
+ }
}
else
{
@@ -10617,9 +10672,26 @@ ix86_expand_prologue (void)
size = 0x80000000 - STACK_CHECK_PROTECT - 1;
if (TARGET_STACK_PROBE)
- ix86_emit_probe_stack_range (0, size + STACK_CHECK_PROTECT);
+ {
+ if (crtl->is_leaf && !cfun->calls_alloca)
+ {
+ if (size > PROBE_INTERVAL)
+ ix86_emit_probe_stack_range (0, size);
+ }
+ else
+ ix86_emit_probe_stack_range (0, size + STACK_CHECK_PROTECT);
+ }
else
- ix86_emit_probe_stack_range (STACK_CHECK_PROTECT, size);
+ {
+ if (crtl->is_leaf && !cfun->calls_alloca)
+ {
+ if (size > PROBE_INTERVAL && size > STACK_CHECK_PROTECT)
+ ix86_emit_probe_stack_range (STACK_CHECK_PROTECT,
+ size - STACK_CHECK_PROTECT);
+ }
+ else
+ ix86_emit_probe_stack_range (STACK_CHECK_PROTECT, size);
+ }
}
}
@@ -11511,8 +11583,8 @@ ix86_expand_split_stack_prologue (void)
JUMP_LABEL (jump_insn) = label;
/* Mark the jump as very likely to be taken. */
- add_reg_note (jump_insn, REG_BR_PROB,
- GEN_INT (REG_BR_PROB_BASE - REG_BR_PROB_BASE / 100));
+ add_int_reg_note (jump_insn, REG_BR_PROB,
+ REG_BR_PROB_BASE - REG_BR_PROB_BASE / 100);
if (split_stack_fn == NULL_RTX)
split_stack_fn = gen_rtx_SYMBOL_REF (Pmode, "__morestack");
@@ -14069,8 +14141,6 @@ put_condition_code (enum rtx_code code, enum machine_mode mode, bool reverse,
Those same assemblers have the same but opposite lossage on cmov. */
if (mode == CCmode)
suffix = fp ? "nbe" : "a";
- else if (mode == CCCmode)
- suffix = "b";
else
gcc_unreachable ();
break;
@@ -14092,8 +14162,12 @@ put_condition_code (enum rtx_code code, enum machine_mode mode, bool reverse,
}
break;
case LTU:
- gcc_assert (mode == CCmode || mode == CCCmode);
- suffix = "b";
+ if (mode == CCmode)
+ suffix = "b";
+ else if (mode == CCCmode)
+ suffix = "c";
+ else
+ gcc_unreachable ();
break;
case GE:
switch (mode)
@@ -14113,20 +14187,20 @@ put_condition_code (enum rtx_code code, enum machine_mode mode, bool reverse,
}
break;
case GEU:
- /* ??? As above. */
- gcc_assert (mode == CCmode || mode == CCCmode);
- suffix = fp ? "nb" : "ae";
+ if (mode == CCmode)
+ suffix = fp ? "nb" : "ae";
+ else if (mode == CCCmode)
+ suffix = "nc";
+ else
+ gcc_unreachable ();
break;
case LE:
gcc_assert (mode == CCmode || mode == CCGCmode || mode == CCNOmode);
suffix = "le";
break;
case LEU:
- /* ??? As above. */
if (mode == CCmode)
suffix = "be";
- else if (mode == CCCmode)
- suffix = fp ? "nb" : "ae";
else
gcc_unreachable ();
break;
@@ -14244,7 +14318,7 @@ print_reg (rtx x, int code, FILE *file)
case 8:
case 4:
case 12:
- if (! ANY_FP_REG_P (x))
+ if (! ANY_FP_REG_P (x) && ! ANY_BND_REG_P (x))
putc (code == 8 && TARGET_64BIT ? 'r' : 'e', file);
/* FALLTHRU */
case 16:
@@ -14367,6 +14441,7 @@ get_some_local_dynamic_name (void)
~ -- print "i" if TARGET_AVX2, "f" otherwise.
@ -- print a segment register of thread base pointer load
^ -- print addr32 prefix if TARGET_64BIT and Pmode != word_mode
+ ! -- print MPX prefix for jxx/call/ret instructions if required.
*/
void
@@ -14808,7 +14883,7 @@ ix86_print_operand (FILE *file, rtx x, int code)
x = find_reg_note (current_output_insn, REG_BR_PROB, 0);
if (x)
{
- int pred_val = INTVAL (XEXP (x, 0));
+ int pred_val = XINT (x, 0);
if (pred_val < REG_BR_PROB_BASE * 45 / 100
|| pred_val > REG_BR_PROB_BASE * 55 / 100)
@@ -14861,6 +14936,11 @@ ix86_print_operand (FILE *file, rtx x, int code)
fputs ("addr32 ", file);
return;
+ case '!':
+ if (ix86_bnd_prefixed_insn_p (NULL_RTX))
+ fputs ("bnd ", file);
+ return;
+
default:
output_operand_lossage ("invalid operand code '%c'", code);
}
@@ -15003,7 +15083,7 @@ static bool
ix86_print_operand_punct_valid_p (unsigned char code)
{
return (code == '@' || code == '*' || code == '+' || code == '&'
- || code == ';' || code == '~' || code == '^');
+ || code == ';' || code == '~' || code == '^' || code == '!');
}
/* Print a memory operand whose address is ADDR. */
@@ -15033,6 +15113,25 @@ ix86_print_operand_address (FILE *file, rtx addr)
ok = ix86_decompose_address (XVECEXP (addr, 0, 0), &parts);
code = 'q';
}
+ else if (GET_CODE (addr) == UNSPEC && XINT (addr, 1) == UNSPEC_BNDMK_ADDR)
+ {
+ ok = ix86_decompose_address (XVECEXP (addr, 0, 1), &parts);
+ gcc_assert (parts.base == NULL_RTX || parts.index == NULL_RTX);
+ if (parts.base != NULL_RTX)
+ {
+ parts.index = parts.base;
+ parts.scale = 1;
+ }
+ parts.base = XVECEXP (addr, 0, 0);
+ addr = XVECEXP (addr, 0, 0);
+ }
+ else if (GET_CODE (addr) == UNSPEC && XINT (addr, 1) == UNSPEC_BNDLDX_ADDR)
+ {
+ ok = ix86_decompose_address (XVECEXP (addr, 0, 0), &parts);
+ gcc_assert (parts.index == NULL_RTX);
+ parts.index = XVECEXP (addr, 0, 1);
+ addr = XVECEXP (addr, 0, 0);
+ }
else
ok = ix86_decompose_address (addr, &parts);
@@ -15609,7 +15708,7 @@ ix86_avx_u128_mode_needed (rtx insn)
rtx arg = XEXP (XEXP (link, 0), 0);
if (ix86_check_avx256_register (&arg, NULL))
- return AVX_U128_ANY;
+ return AVX_U128_DIRTY;
}
}
@@ -15729,8 +15828,8 @@ ix86_avx_u128_mode_after (int mode, rtx insn)
{
bool avx_reg256_found = false;
note_stores (pat, ix86_check_avx256_stores, &avx_reg256_found);
- if (!avx_reg256_found)
- return AVX_U128_CLEAN;
+
+ return avx_reg256_found ? AVX_U128_DIRTY : AVX_U128_CLEAN;
}
/* Otherwise, return current mode. Remember that if insn
@@ -16457,8 +16556,8 @@ ix86_avx256_split_vector_move_misalign (rtx op0, rtx op1)
gcc_unreachable ();
case V32QImode:
extract = gen_avx_vextractf128v32qi;
- load_unaligned = gen_avx_loaddqu256;
- store_unaligned = gen_avx_storedqu256;
+ load_unaligned = gen_avx_loaddquv32qi;
+ store_unaligned = gen_avx_storedquv32qi;
mode = V16QImode;
break;
case V8SFmode:
@@ -16486,6 +16585,12 @@ ix86_avx256_split_vector_move_misalign (rtx op0, rtx op1)
r = gen_rtx_VEC_CONCAT (GET_MODE (op0), r, m);
emit_move_insn (op0, r);
}
+ /* Normal *mov<mode>_internal pattern will handle
+ unaligned loads just fine if misaligned_operand
+ is true, and without the UNSPEC it can be combined
+ with arithmetic instructions. */
+ else if (misaligned_operand (op1, GET_MODE (op1)))
+ emit_insn (gen_rtx_SET (VOIDmode, op0, op1));
else
emit_insn (load_unaligned (op0, op1));
}
@@ -16560,11 +16665,68 @@ ix86_avx256_split_vector_move_misalign (rtx op0, rtx op1)
void
ix86_expand_vector_move_misalign (enum machine_mode mode, rtx operands[])
{
- rtx op0, op1, m;
+ rtx op0, op1, orig_op0 = NULL_RTX, m;
+ rtx (*load_unaligned) (rtx, rtx);
+ rtx (*store_unaligned) (rtx, rtx);
op0 = operands[0];
op1 = operands[1];
+ if (GET_MODE_SIZE (mode) == 64)
+ {
+ switch (GET_MODE_CLASS (mode))
+ {
+ case MODE_VECTOR_INT:
+ case MODE_INT:
+ if (GET_MODE (op0) != V16SImode)
+ {
+ if (!MEM_P (op0))
+ {
+ orig_op0 = op0;
+ op0 = gen_reg_rtx (V16SImode);
+ }
+ else
+ op0 = gen_lowpart (V16SImode, op0);
+ }
+ op1 = gen_lowpart (V16SImode, op1);
+ /* FALLTHRU */
+
+ case MODE_VECTOR_FLOAT:
+ switch (GET_MODE (op0))
+ {
+ default:
+ gcc_unreachable ();
+ case V16SImode:
+ load_unaligned = gen_avx512f_loaddquv16si;
+ store_unaligned = gen_avx512f_storedquv16si;
+ break;
+ case V16SFmode:
+ load_unaligned = gen_avx512f_loadups512;
+ store_unaligned = gen_avx512f_storeups512;
+ break;
+ case V8DFmode:
+ load_unaligned = gen_avx512f_loadupd512;
+ store_unaligned = gen_avx512f_storeupd512;
+ break;
+ }
+
+ if (MEM_P (op1))
+ emit_insn (load_unaligned (op0, op1));
+ else if (MEM_P (op0))
+ emit_insn (store_unaligned (op0, op1));
+ else
+ gcc_unreachable ();
+ if (orig_op0)
+ emit_move_insn (orig_op0, gen_lowpart (GET_MODE (orig_op0), op0));
+ break;
+
+ default:
+ gcc_unreachable ();
+ }
+
+ return;
+ }
+
if (TARGET_AVX
&& GET_MODE_SIZE (mode) == 32)
{
@@ -16572,12 +16734,23 @@ ix86_expand_vector_move_misalign (enum machine_mode mode, rtx operands[])
{
case MODE_VECTOR_INT:
case MODE_INT:
- op0 = gen_lowpart (V32QImode, op0);
+ if (GET_MODE (op0) != V32QImode)
+ {
+ if (!MEM_P (op0))
+ {
+ orig_op0 = op0;
+ op0 = gen_reg_rtx (V32QImode);
+ }
+ else
+ op0 = gen_lowpart (V32QImode, op0);
+ }
op1 = gen_lowpart (V32QImode, op1);
/* FALLTHRU */
case MODE_VECTOR_FLOAT:
ix86_avx256_split_vector_move_misalign (op0, op1);
+ if (orig_op0)
+ emit_move_insn (orig_op0, gen_lowpart (GET_MODE (orig_op0), op0));
break;
default:
@@ -16589,15 +16762,30 @@ ix86_expand_vector_move_misalign (enum machine_mode mode, rtx operands[])
if (MEM_P (op1))
{
+ /* Normal *mov<mode>_internal pattern will handle
+ unaligned loads just fine if misaligned_operand
+ is true, and without the UNSPEC it can be combined
+ with arithmetic instructions. */
+ if (TARGET_AVX
+ && (GET_MODE_CLASS (mode) == MODE_VECTOR_INT
+ || GET_MODE_CLASS (mode) == MODE_VECTOR_FLOAT)
+ && misaligned_operand (op1, GET_MODE (op1)))
+ emit_insn (gen_rtx_SET (VOIDmode, op0, op1));
/* ??? If we have typed data, then it would appear that using
movdqu is the only way to get unaligned data loaded with
integer type. */
- if (TARGET_SSE2 && GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
+ else if (TARGET_SSE2 && GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
{
- op0 = gen_lowpart (V16QImode, op0);
+ if (GET_MODE (op0) != V16QImode)
+ {
+ orig_op0 = op0;
+ op0 = gen_reg_rtx (V16QImode);
+ }
op1 = gen_lowpart (V16QImode, op1);
/* We will eventually emit movups based on insn attributes. */
- emit_insn (gen_sse2_loaddqu (op0, op1));
+ emit_insn (gen_sse2_loaddquv16qi (op0, op1));
+ if (orig_op0)
+ emit_move_insn (orig_op0, gen_lowpart (GET_MODE (orig_op0), op0));
}
else if (TARGET_SSE2 && mode == V2DFmode)
{
@@ -16640,29 +16828,42 @@ ix86_expand_vector_move_misalign (enum machine_mode mode, rtx operands[])
}
else
{
+ rtx t;
+
if (TARGET_AVX
|| TARGET_SSE_UNALIGNED_LOAD_OPTIMAL
|| TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL
|| optimize_insn_for_size_p ())
{
- op0 = gen_lowpart (V4SFmode, op0);
+ if (GET_MODE (op0) != V4SFmode)
+ {
+ orig_op0 = op0;
+ op0 = gen_reg_rtx (V4SFmode);
+ }
op1 = gen_lowpart (V4SFmode, op1);
emit_insn (gen_sse_loadups (op0, op1));
+ if (orig_op0)
+ emit_move_insn (orig_op0,
+ gen_lowpart (GET_MODE (orig_op0), op0));
return;
}
+ if (mode != V4SFmode)
+ t = gen_reg_rtx (V4SFmode);
+ else
+ t = op0;
+
if (TARGET_SSE_PARTIAL_REG_DEPENDENCY)
- emit_move_insn (op0, CONST0_RTX (mode));
+ emit_move_insn (t, CONST0_RTX (V4SFmode));
else
- emit_clobber (op0);
-
- if (mode != V4SFmode)
- op0 = gen_lowpart (V4SFmode, op0);
+ emit_clobber (t);
m = adjust_address (op1, V2SFmode, 0);
- emit_insn (gen_sse_loadlps (op0, op0, m));
+ emit_insn (gen_sse_loadlps (t, t, m));
m = adjust_address (op1, V2SFmode, 8);
- emit_insn (gen_sse_loadhps (op0, op0, m));
+ emit_insn (gen_sse_loadhps (t, t, m));
+ if (mode != V4SFmode)
+ emit_move_insn (op0, gen_lowpart (mode, t));
}
}
else if (MEM_P (op0))
@@ -16672,7 +16873,7 @@ ix86_expand_vector_move_misalign (enum machine_mode mode, rtx operands[])
op0 = gen_lowpart (V16QImode, op0);
op1 = gen_lowpart (V16QImode, op1);
/* We will eventually emit movups based on insn attributes. */
- emit_insn (gen_sse2_storedqu (op0, op1));
+ emit_insn (gen_sse2_storedquv16qi (op0, op1));
}
else if (TARGET_SSE2 && mode == V2DFmode)
{
@@ -16811,8 +17012,10 @@ ix86_fixup_binary_operands (enum rtx_code code, enum machine_mode mode,
src2 = force_reg (mode, src2);
src1 = src2;
}
- else
+ else if (rtx_equal_p (dst, src1))
src2 = force_reg (mode, src2);
+ else
+ src1 = force_reg (mode, src1);
}
/* If the destination is memory, and we do not have matching source
@@ -17162,6 +17365,14 @@ ix86_split_idivmod (enum machine_mode mode, rtx operands[],
emit_label (end_label);
}
+/* Whether it is OK to emit CFI directives when emitting asm code. */
+
+bool
+ix86_emit_cfi ()
+{
+ return dwarf2out_do_cfi_asm ();
+}
+
#define LEA_MAX_STALL (3)
#define LEA_SEARCH_THRESHOLD (LEA_MAX_STALL << 1)
@@ -18780,12 +18991,7 @@ ix86_cc_mode (enum rtx_code code, rtx op0, rtx op1)
return CCmode;
case GTU: /* CF=0 & ZF=0 */
case LEU: /* CF=1 | ZF=1 */
- /* Detect overflow checks. They need just the carry flag. */
- if (GET_CODE (op0) == MINUS
- && rtx_equal_p (op1, XEXP (op0, 0)))
- return CCCmode;
- else
- return CCmode;
+ return CCmode;
/* Codes possibly doable only with sign flag when
comparing against zero. */
case GE: /* SF=OF or SF=0 */
@@ -19454,7 +19660,7 @@ ix86_split_fp_branch (enum rtx_code code, rtx op1, rtx op2,
gen_rtx_IF_THEN_ELSE (VOIDmode,
condition, target1, target2)));
if (split_branch_probability >= 0)
- add_reg_note (i, REG_BR_PROB, GEN_INT (split_branch_probability));
+ add_int_reg_note (i, REG_BR_PROB, split_branch_probability);
}
void
@@ -20306,6 +20512,7 @@ ix86_expand_sse_movcc (rtx dest, rtx cmp, rtx op_true, rtx op_false)
else
{
rtx (*gen) (rtx, rtx, rtx, rtx) = NULL;
+ rtx d = dest;
if (!nonimmediate_operand (op_true, mode))
op_true = force_reg (mode, op_true);
@@ -20329,7 +20536,8 @@ ix86_expand_sse_movcc (rtx dest, rtx cmp, rtx op_true, rtx op_false)
if (TARGET_SSE4_1)
{
gen = gen_sse4_1_pblendvb;
- dest = gen_lowpart (V16QImode, dest);
+ if (mode != V16QImode)
+ d = gen_reg_rtx (V16QImode);
op_false = gen_lowpart (V16QImode, op_false);
op_true = gen_lowpart (V16QImode, op_true);
cmp = gen_lowpart (V16QImode, cmp);
@@ -20350,7 +20558,8 @@ ix86_expand_sse_movcc (rtx dest, rtx cmp, rtx op_true, rtx op_false)
if (TARGET_AVX2)
{
gen = gen_avx2_pblendvb;
- dest = gen_lowpart (V32QImode, dest);
+ if (mode != V32QImode)
+ d = gen_reg_rtx (V32QImode);
op_false = gen_lowpart (V32QImode, op_false);
op_true = gen_lowpart (V32QImode, op_true);
cmp = gen_lowpart (V32QImode, cmp);
@@ -20361,7 +20570,11 @@ ix86_expand_sse_movcc (rtx dest, rtx cmp, rtx op_true, rtx op_false)
}
if (gen != NULL)
- emit_insn (gen (dest, op_false, op_true, cmp));
+ {
+ emit_insn (gen (d, op_false, op_true, cmp));
+ if (d != dest)
+ emit_move_insn (dest, gen_lowpart (GET_MODE (dest), d));
+ }
else
{
op_true = force_reg (mode, op_true);
@@ -20682,8 +20895,7 @@ ix86_expand_int_vcond (rtx operands[])
else
{
gcc_assert (GET_MODE_SIZE (data_mode) == GET_MODE_SIZE (mode));
- x = ix86_expand_sse_cmp (gen_lowpart (mode, operands[0]),
- code, cop0, cop1,
+ x = ix86_expand_sse_cmp (gen_reg_rtx (mode), code, cop0, cop1,
operands[1+negate], operands[2-negate]);
x = gen_lowpart (data_mode, x);
}
@@ -20702,7 +20914,7 @@ ix86_expand_vec_perm (rtx operands[])
rtx op0 = operands[1];
rtx op1 = operands[2];
rtx mask = operands[3];
- rtx t1, t2, t3, t4, vt, vt2, vec[32];
+ rtx t1, t2, t3, t4, t5, t6, t7, t8, vt, vt2, vec[32];
enum machine_mode mode = GET_MODE (op0);
enum machine_mode maskmode = GET_MODE (mask);
int w, e, i;
@@ -20770,7 +20982,7 @@ ix86_expand_vec_perm (rtx operands[])
/* Continue as if V8SImode (resp. V32QImode) was used initially. */
operands[3] = mask = t1;
- target = gen_lowpart (mode, target);
+ target = gen_reg_rtx (mode);
op0 = gen_lowpart (mode, op0);
op1 = gen_lowpart (mode, op1);
}
@@ -20782,7 +20994,12 @@ ix86_expand_vec_perm (rtx operands[])
the high bits of the shuffle elements. No need for us to
perform an AND ourselves. */
if (one_operand_shuffle)
- emit_insn (gen_avx2_permvarv8si (target, op0, mask));
+ {
+ emit_insn (gen_avx2_permvarv8si (target, op0, mask));
+ if (target != operands[0])
+ emit_move_insn (operands[0],
+ gen_lowpart (GET_MODE (operands[0]), target));
+ }
else
{
t1 = gen_reg_rtx (V8SImode);
@@ -20855,13 +21072,13 @@ ix86_expand_vec_perm (rtx operands[])
stands for other 12 bytes. */
/* The bit whether element is from the same lane or the other
lane is bit 4, so shift it up by 3 to the MSB position. */
- emit_insn (gen_ashlv4di3 (gen_lowpart (V4DImode, t1),
- gen_lowpart (V4DImode, mask),
+ t5 = gen_reg_rtx (V4DImode);
+ emit_insn (gen_ashlv4di3 (t5, gen_lowpart (V4DImode, mask),
GEN_INT (3)));
/* Clear MSB bits from the mask just in case it had them set. */
emit_insn (gen_avx2_andnotv32qi3 (t2, vt, mask));
/* After this t1 will have MSB set for elements from other lane. */
- emit_insn (gen_xorv32qi3 (t1, t1, vt2));
+ emit_insn (gen_xorv32qi3 (t1, gen_lowpart (V32QImode, t5), vt2));
/* Clear bits other than MSB. */
emit_insn (gen_andv32qi3 (t1, t1, vt));
/* Or in the lower bits from mask into t3. */
@@ -20870,8 +21087,8 @@ ix86_expand_vec_perm (rtx operands[])
lane. */
emit_insn (gen_xorv32qi3 (t1, t1, vt));
/* Swap 128-bit lanes in t3. */
- emit_insn (gen_avx2_permv4di_1 (gen_lowpart (V4DImode, t3),
- gen_lowpart (V4DImode, t3),
+ t6 = gen_reg_rtx (V4DImode);
+ emit_insn (gen_avx2_permv4di_1 (t6, gen_lowpart (V4DImode, t3),
const2_rtx, GEN_INT (3),
const0_rtx, const1_rtx));
/* And or in the lower bits from mask into t1. */
@@ -20881,15 +21098,20 @@ ix86_expand_vec_perm (rtx operands[])
/* Each of these shuffles will put 0s in places where
element from the other 128-bit lane is needed, otherwise
will shuffle in the requested value. */
- emit_insn (gen_avx2_pshufbv32qi3 (t3, op0, t3));
+ emit_insn (gen_avx2_pshufbv32qi3 (t3, op0,
+ gen_lowpart (V32QImode, t6)));
emit_insn (gen_avx2_pshufbv32qi3 (t1, op0, t1));
/* For t3 the 128-bit lanes are swapped again. */
- emit_insn (gen_avx2_permv4di_1 (gen_lowpart (V4DImode, t3),
- gen_lowpart (V4DImode, t3),
+ t7 = gen_reg_rtx (V4DImode);
+ emit_insn (gen_avx2_permv4di_1 (t7, gen_lowpart (V4DImode, t3),
const2_rtx, GEN_INT (3),
const0_rtx, const1_rtx));
/* And oring both together leads to the result. */
- emit_insn (gen_iorv32qi3 (target, t1, t3));
+ emit_insn (gen_iorv32qi3 (target, t1,
+ gen_lowpart (V32QImode, t7)));
+ if (target != operands[0])
+ emit_move_insn (operands[0],
+ gen_lowpart (GET_MODE (operands[0]), target));
return;
}
@@ -20897,20 +21119,22 @@ ix86_expand_vec_perm (rtx operands[])
/* Similarly to the above one_operand_shuffle code,
just for repeated twice for each operand. merge_two:
code will merge the two results together. */
- emit_insn (gen_avx2_pshufbv32qi3 (t4, op0, t3));
- emit_insn (gen_avx2_pshufbv32qi3 (t3, op1, t3));
+ emit_insn (gen_avx2_pshufbv32qi3 (t4, op0,
+ gen_lowpart (V32QImode, t6)));
+ emit_insn (gen_avx2_pshufbv32qi3 (t3, op1,
+ gen_lowpart (V32QImode, t6)));
emit_insn (gen_avx2_pshufbv32qi3 (t2, op0, t1));
emit_insn (gen_avx2_pshufbv32qi3 (t1, op1, t1));
- emit_insn (gen_avx2_permv4di_1 (gen_lowpart (V4DImode, t4),
- gen_lowpart (V4DImode, t4),
+ t7 = gen_reg_rtx (V4DImode);
+ emit_insn (gen_avx2_permv4di_1 (t7, gen_lowpart (V4DImode, t4),
const2_rtx, GEN_INT (3),
const0_rtx, const1_rtx));
- emit_insn (gen_avx2_permv4di_1 (gen_lowpart (V4DImode, t3),
- gen_lowpart (V4DImode, t3),
+ t8 = gen_reg_rtx (V4DImode);
+ emit_insn (gen_avx2_permv4di_1 (t8, gen_lowpart (V4DImode, t3),
const2_rtx, GEN_INT (3),
const0_rtx, const1_rtx));
- emit_insn (gen_iorv32qi3 (t4, t2, t4));
- emit_insn (gen_iorv32qi3 (t3, t1, t3));
+ emit_insn (gen_iorv32qi3 (t4, t2, gen_lowpart (V32QImode, t7)));
+ emit_insn (gen_iorv32qi3 (t3, t1, gen_lowpart (V32QImode, t8)));
t1 = t4;
t2 = t3;
goto merge_two;
@@ -20979,15 +21203,24 @@ ix86_expand_vec_perm (rtx operands[])
/* The actual shuffle operations all operate on V16QImode. */
op0 = gen_lowpart (V16QImode, op0);
op1 = gen_lowpart (V16QImode, op1);
- target = gen_lowpart (V16QImode, target);
if (TARGET_XOP)
{
+ if (GET_MODE (target) != V16QImode)
+ target = gen_reg_rtx (V16QImode);
emit_insn (gen_xop_pperm (target, op0, op1, mask));
+ if (target != operands[0])
+ emit_move_insn (operands[0],
+ gen_lowpart (GET_MODE (operands[0]), target));
}
else if (one_operand_shuffle)
{
+ if (GET_MODE (target) != V16QImode)
+ target = gen_reg_rtx (V16QImode);
emit_insn (gen_ssse3_pshufbv16qi3 (target, op0, mask));
+ if (target != operands[0])
+ emit_move_insn (operands[0],
+ gen_lowpart (GET_MODE (operands[0]), target));
}
else
{
@@ -21027,7 +21260,9 @@ ix86_expand_vec_perm (rtx operands[])
mask = expand_simple_binop (maskmode, AND, mask, vt,
NULL_RTX, 0, OPTAB_DIRECT);
- xops[0] = gen_lowpart (mode, operands[0]);
+ if (GET_MODE (target) != mode)
+ target = gen_reg_rtx (mode);
+ xops[0] = target;
xops[1] = gen_lowpart (mode, t2);
xops[2] = gen_lowpart (mode, t1);
xops[3] = gen_rtx_EQ (maskmode, mask, vt);
@@ -21035,6 +21270,9 @@ ix86_expand_vec_perm (rtx operands[])
xops[5] = vt;
ok = ix86_expand_int_vcond (xops);
gcc_assert (ok);
+ if (target != operands[0])
+ emit_move_insn (operands[0],
+ gen_lowpart (GET_MODE (operands[0]), target));
}
}
@@ -21113,10 +21351,10 @@ ix86_expand_sse_unpack (rtx dest, rtx src, bool unsigned_p, bool high_p)
else if (high_p)
{
/* Shift higher 8 bytes to lower 8 bytes. */
- tmp = gen_reg_rtx (imode);
- emit_insn (gen_sse2_lshrv1ti3 (gen_lowpart (V1TImode, tmp),
- gen_lowpart (V1TImode, src),
+ tmp = gen_reg_rtx (V1TImode);
+ emit_insn (gen_sse2_lshrv1ti3 (tmp, gen_lowpart (V1TImode, src),
GEN_INT (64)));
+ tmp = gen_lowpart (imode, tmp);
}
else
tmp = src;
@@ -21157,7 +21395,9 @@ ix86_expand_sse_unpack (rtx dest, rtx src, bool unsigned_p, bool high_p)
tmp = ix86_expand_sse_cmp (gen_reg_rtx (imode), GT, CONST0_RTX (imode),
src, pc_rtx, pc_rtx);
- emit_insn (unpack (gen_lowpart (imode, dest), src, tmp));
+ rtx tmp2 = gen_reg_rtx (imode);
+ emit_insn (unpack (tmp2, src, tmp));
+ emit_move_insn (dest, gen_lowpart (GET_MODE (dest), tmp2));
}
}
@@ -21961,7 +22201,7 @@ predict_jump (int prob)
{
rtx insn = get_last_insn ();
gcc_assert (JUMP_P (insn));
- add_reg_note (insn, REG_BR_PROB, GEN_INT (prob));
+ add_int_reg_note (insn, REG_BR_PROB, prob);
}
/* Helper function for the string operations below. Dest VARIABLE whether
@@ -22034,10 +22274,25 @@ counter_mode (rtx count_exp)
return SImode;
}
-/* When SRCPTR is non-NULL, output simple loop to move memory
- pointer to SRCPTR to DESTPTR via chunks of MODE unrolled UNROLL times,
- overall size is COUNT specified in bytes. When SRCPTR is NULL, output the
- equivalent loop to set memory by VALUE (supposed to be in MODE).
+/* Copy the address to a Pmode register. This is used for x32 to
+ truncate DImode TLS address to a SImode register. */
+
+static rtx
+ix86_copy_addr_to_reg (rtx addr)
+{
+ if (GET_MODE (addr) == Pmode)
+ return copy_addr_to_reg (addr);
+ else
+ {
+ gcc_assert (GET_MODE (addr) == DImode && Pmode == SImode);
+ return gen_rtx_SUBREG (SImode, copy_to_mode_reg (DImode, addr), 0);
+ }
+}
+
+/* When ISSETMEM is FALSE, output simple loop to move memory pointer to SRCPTR
+ to DESTPTR via chunks of MODE unrolled UNROLL times, overall size is COUNT
+ specified in bytes. When ISSETMEM is TRUE, output the equivalent loop to set
+ memory by VALUE (supposed to be in MODE).
The size is rounded down to whole number of chunk size moved at once.
SRCMEM and DESTMEM provide MEMrtx to feed proper aliasing info. */
@@ -22047,7 +22302,7 @@ static void
expand_set_or_movmem_via_loop (rtx destmem, rtx srcmem,
rtx destptr, rtx srcptr, rtx value,
rtx count, enum machine_mode mode, int unroll,
- int expected_size)
+ int expected_size, bool issetmem)
{
rtx out_label, top_label, iter, tmp;
enum machine_mode iter_mode = counter_mode (count);
@@ -22083,7 +22338,7 @@ expand_set_or_movmem_via_loop (rtx destmem, rtx srcmem,
destmem = offset_address (destmem, tmp, piece_size_n);
destmem = adjust_address (destmem, mode, 0);
- if (srcmem)
+ if (!issetmem)
{
srcmem = offset_address (srcmem, copy_rtx (tmp), piece_size_n);
srcmem = adjust_address (srcmem, mode, 0);
@@ -22163,7 +22418,7 @@ expand_set_or_movmem_via_loop (rtx destmem, rtx srcmem,
true, OPTAB_LIB_WIDEN);
if (tmp != destptr)
emit_move_insn (destptr, tmp);
- if (srcptr)
+ if (!issetmem)
{
tmp = expand_simple_binop (Pmode, PLUS, srcptr, iter, srcptr,
true, OPTAB_LIB_WIDEN);
@@ -22173,96 +22428,85 @@ expand_set_or_movmem_via_loop (rtx destmem, rtx srcmem,
emit_label (out_label);
}
-/* Output "rep; mov" instruction.
- Arguments have same meaning as for previous function */
+/* Output "rep; mov" or "rep; stos" instruction depending on ISSETMEM argument.
+ When ISSETMEM is true, arguments SRCMEM and SRCPTR are ignored.
+ When ISSETMEM is false, arguments VALUE and ORIG_VALUE are ignored.
+ For setmem case, VALUE is a promoted to a wider size ORIG_VALUE.
+ ORIG_VALUE is the original value passed to memset to fill the memory with.
+ Other arguments have same meaning as for previous function. */
+
static void
-expand_movmem_via_rep_mov (rtx destmem, rtx srcmem,
- rtx destptr, rtx srcptr,
+expand_set_or_movmem_via_rep (rtx destmem, rtx srcmem,
+ rtx destptr, rtx srcptr, rtx value, rtx orig_value,
rtx count,
- enum machine_mode mode)
+ enum machine_mode mode, bool issetmem)
{
rtx destexp;
rtx srcexp;
rtx countreg;
HOST_WIDE_INT rounded_count;
- /* If the size is known, it is shorter to use rep movs. */
- if (mode == QImode && CONST_INT_P (count)
- && !(INTVAL (count) & 3))
+ /* If possible, it is shorter to use rep movs.
+ TODO: Maybe it is better to move this logic to decide_alg. */
+ if (mode == QImode && CONST_INT_P (count) && !(INTVAL (count) & 3)
+ && (!issetmem || orig_value == const0_rtx))
mode = SImode;
if (destptr != XEXP (destmem, 0) || GET_MODE (destmem) != BLKmode)
destmem = adjust_automodify_address_nv (destmem, BLKmode, destptr, 0);
- if (srcptr != XEXP (srcmem, 0) || GET_MODE (srcmem) != BLKmode)
- srcmem = adjust_automodify_address_nv (srcmem, BLKmode, srcptr, 0);
- countreg = ix86_zero_extend_to_Pmode (scale_counter (count, GET_MODE_SIZE (mode)));
+
+ countreg = ix86_zero_extend_to_Pmode (scale_counter (count,
+ GET_MODE_SIZE (mode)));
if (mode != QImode)
{
destexp = gen_rtx_ASHIFT (Pmode, countreg,
GEN_INT (exact_log2 (GET_MODE_SIZE (mode))));
destexp = gen_rtx_PLUS (Pmode, destexp, destptr);
- srcexp = gen_rtx_ASHIFT (Pmode, countreg,
- GEN_INT (exact_log2 (GET_MODE_SIZE (mode))));
- srcexp = gen_rtx_PLUS (Pmode, srcexp, srcptr);
}
else
- {
- destexp = gen_rtx_PLUS (Pmode, destptr, countreg);
- srcexp = gen_rtx_PLUS (Pmode, srcptr, countreg);
- }
- if (CONST_INT_P (count))
+ destexp = gen_rtx_PLUS (Pmode, destptr, countreg);
+ if ((!issetmem || orig_value == const0_rtx) && CONST_INT_P (count))
{
rounded_count = (INTVAL (count)
& ~((HOST_WIDE_INT) GET_MODE_SIZE (mode) - 1));
destmem = shallow_copy_rtx (destmem);
- srcmem = shallow_copy_rtx (srcmem);
set_mem_size (destmem, rounded_count);
- set_mem_size (srcmem, rounded_count);
- }
- else
- {
- if (MEM_SIZE_KNOWN_P (destmem))
- clear_mem_size (destmem);
- if (MEM_SIZE_KNOWN_P (srcmem))
- clear_mem_size (srcmem);
}
- emit_insn (gen_rep_mov (destptr, destmem, srcptr, srcmem, countreg,
- destexp, srcexp));
-}
-
-/* Output "rep; stos" instruction.
- Arguments have same meaning as for previous function */
-static void
-expand_setmem_via_rep_stos (rtx destmem, rtx destptr, rtx value,
- rtx count, enum machine_mode mode,
- rtx orig_value)
-{
- rtx destexp;
- rtx countreg;
- HOST_WIDE_INT rounded_count;
+ else if (MEM_SIZE_KNOWN_P (destmem))
+ clear_mem_size (destmem);
- if (destptr != XEXP (destmem, 0) || GET_MODE (destmem) != BLKmode)
- destmem = adjust_automodify_address_nv (destmem, BLKmode, destptr, 0);
- value = force_reg (mode, gen_lowpart (mode, value));
- countreg = ix86_zero_extend_to_Pmode (scale_counter (count, GET_MODE_SIZE (mode)));
- if (mode != QImode)
+ if (issetmem)
{
- destexp = gen_rtx_ASHIFT (Pmode, countreg,
- GEN_INT (exact_log2 (GET_MODE_SIZE (mode))));
- destexp = gen_rtx_PLUS (Pmode, destexp, destptr);
+ value = force_reg (mode, gen_lowpart (mode, value));
+ emit_insn (gen_rep_stos (destptr, countreg, destmem, value, destexp));
}
else
- destexp = gen_rtx_PLUS (Pmode, destptr, countreg);
- if (orig_value == const0_rtx && CONST_INT_P (count))
{
- rounded_count = (INTVAL (count)
- & ~((HOST_WIDE_INT) GET_MODE_SIZE (mode) - 1));
- destmem = shallow_copy_rtx (destmem);
- set_mem_size (destmem, rounded_count);
+ if (srcptr != XEXP (srcmem, 0) || GET_MODE (srcmem) != BLKmode)
+ srcmem = adjust_automodify_address_nv (srcmem, BLKmode, srcptr, 0);
+ if (mode != QImode)
+ {
+ srcexp = gen_rtx_ASHIFT (Pmode, countreg,
+ GEN_INT (exact_log2 (GET_MODE_SIZE (mode))));
+ srcexp = gen_rtx_PLUS (Pmode, srcexp, srcptr);
+ }
+ else
+ srcexp = gen_rtx_PLUS (Pmode, srcptr, countreg);
+ if (CONST_INT_P (count))
+ {
+ rounded_count = (INTVAL (count)
+ & ~((HOST_WIDE_INT) GET_MODE_SIZE (mode) - 1));
+ srcmem = shallow_copy_rtx (srcmem);
+ set_mem_size (srcmem, rounded_count);
+ }
+ else
+ {
+ if (MEM_SIZE_KNOWN_P (srcmem))
+ clear_mem_size (srcmem);
+ }
+ emit_insn (gen_rep_mov (destptr, destmem, srcptr, srcmem, countreg,
+ destexp, srcexp));
}
- else if (MEM_SIZE_KNOWN_P (destmem))
- clear_mem_size (destmem);
- emit_insn (gen_rep_stos (destptr, countreg, destmem, value, destexp));
}
/* This function emits moves to copy SIZE_TO_MOVE bytes from SRCMEM to
@@ -22365,7 +22609,7 @@ expand_movmem_epilogue (rtx destmem, rtx srcmem,
count = expand_simple_binop (GET_MODE (count), AND, count, GEN_INT (max_size - 1),
count, 1, OPTAB_DIRECT);
expand_set_or_movmem_via_loop (destmem, srcmem, destptr, srcptr, NULL,
- count, QImode, 1, 4);
+ count, QImode, 1, 4, false);
return;
}
@@ -22450,6 +22694,59 @@ expand_movmem_epilogue (rtx destmem, rtx srcmem,
}
}
+/* This function emits moves to fill SIZE_TO_MOVE bytes starting from DESTMEM
+ with value PROMOTED_VAL.
+ SRC is passed by pointer to be updated on return.
+ Return value is updated DST. */
+static rtx
+emit_memset (rtx destmem, rtx destptr, rtx promoted_val,
+ HOST_WIDE_INT size_to_move)
+{
+ rtx dst = destmem, adjust;
+ enum insn_code code;
+ enum machine_mode move_mode;
+ int piece_size, i;
+
+ /* Find the widest mode in which we could perform moves.
+ Start with the biggest power of 2 less than SIZE_TO_MOVE and half
+ it until move of such size is supported. */
+ move_mode = GET_MODE (promoted_val);
+ if (move_mode == VOIDmode)
+ move_mode = QImode;
+ if (size_to_move < GET_MODE_SIZE (move_mode))
+ {
+ move_mode = mode_for_size (size_to_move * BITS_PER_UNIT, MODE_INT, 0);
+ promoted_val = gen_lowpart (move_mode, promoted_val);
+ }
+ piece_size = GET_MODE_SIZE (move_mode);
+ code = optab_handler (mov_optab, move_mode);
+ gcc_assert (code != CODE_FOR_nothing && promoted_val != NULL_RTX);
+
+ dst = adjust_automodify_address_nv (dst, move_mode, destptr, 0);
+
+ /* Emit moves. We'll need SIZE_TO_MOVE/PIECE_SIZES moves. */
+ gcc_assert (size_to_move % piece_size == 0);
+ adjust = GEN_INT (piece_size);
+ for (i = 0; i < size_to_move; i += piece_size)
+ {
+ if (piece_size <= GET_MODE_SIZE (word_mode))
+ {
+ emit_insn (gen_strset (destptr, dst, promoted_val));
+ continue;
+ }
+
+ emit_insn (GEN_FCN (code) (dst, promoted_val));
+
+ emit_move_insn (destptr,
+ gen_rtx_PLUS (Pmode, copy_rtx (destptr), adjust));
+
+ dst = adjust_automodify_address_nv (dst, move_mode, destptr,
+ piece_size);
+ }
+
+ /* Update DST rtx. */
+ return dst;
+}
/* Output code to set at most count & (max_size - 1) bytes starting by DEST. */
static void
expand_setmem_epilogue_via_loop (rtx destmem, rtx destptr, rtx value,
@@ -22460,66 +22757,35 @@ expand_setmem_epilogue_via_loop (rtx destmem, rtx destptr, rtx value,
GEN_INT (max_size - 1), count, 1, OPTAB_DIRECT);
expand_set_or_movmem_via_loop (destmem, NULL, destptr, NULL,
gen_lowpart (QImode, value), count, QImode,
- 1, max_size / 2);
+ 1, max_size / 2, true);
}
/* Output code to set at most count & (max_size - 1) bytes starting by DEST. */
static void
-expand_setmem_epilogue (rtx destmem, rtx destptr, rtx value, rtx count, int max_size)
+expand_setmem_epilogue (rtx destmem, rtx destptr, rtx value, rtx vec_value,
+ rtx count, int max_size)
{
rtx dest;
if (CONST_INT_P (count))
{
HOST_WIDE_INT countval = INTVAL (count);
- int offset = 0;
+ HOST_WIDE_INT epilogue_size = countval % max_size;
+ int i;
- if ((countval & 0x10) && max_size > 16)
- {
- if (TARGET_64BIT)
- {
- dest = adjust_automodify_address_nv (destmem, DImode, destptr, offset);
- emit_insn (gen_strset (destptr, dest, value));
- dest = adjust_automodify_address_nv (destmem, DImode, destptr, offset + 8);
- emit_insn (gen_strset (destptr, dest, value));
- }
- else
- gcc_unreachable ();
- offset += 16;
- }
- if ((countval & 0x08) && max_size > 8)
+ /* For now MAX_SIZE should be a power of 2. This assert could be
+ relaxed, but it'll require a bit more complicated epilogue
+ expanding. */
+ gcc_assert ((max_size & (max_size - 1)) == 0);
+ for (i = max_size; i >= 1; i >>= 1)
{
- if (TARGET_64BIT)
- {
- dest = adjust_automodify_address_nv (destmem, DImode, destptr, offset);
- emit_insn (gen_strset (destptr, dest, value));
- }
- else
+ if (epilogue_size & i)
{
- dest = adjust_automodify_address_nv (destmem, SImode, destptr, offset);
- emit_insn (gen_strset (destptr, dest, value));
- dest = adjust_automodify_address_nv (destmem, SImode, destptr, offset + 4);
- emit_insn (gen_strset (destptr, dest, value));
+ if (vec_value && i > GET_MODE_SIZE (GET_MODE (value)))
+ destmem = emit_memset (destmem, destptr, vec_value, i);
+ else
+ destmem = emit_memset (destmem, destptr, value, i);
}
- offset += 8;
- }
- if ((countval & 0x04) && max_size > 4)
- {
- dest = adjust_automodify_address_nv (destmem, SImode, destptr, offset);
- emit_insn (gen_strset (destptr, dest, gen_lowpart (SImode, value)));
- offset += 4;
- }
- if ((countval & 0x02) && max_size > 2)
- {
- dest = adjust_automodify_address_nv (destmem, HImode, destptr, offset);
- emit_insn (gen_strset (destptr, dest, gen_lowpart (HImode, value)));
- offset += 2;
- }
- if ((countval & 0x01) && max_size > 1)
- {
- dest = adjust_automodify_address_nv (destmem, QImode, destptr, offset);
- emit_insn (gen_strset (destptr, dest, gen_lowpart (QImode, value)));
- offset += 1;
}
return;
}
@@ -22591,13 +22857,16 @@ expand_setmem_epilogue (rtx destmem, rtx destptr, rtx value, rtx count, int max_
}
}
-/* Copy enough from DEST to SRC to align DEST known to by aligned by ALIGN to
- DESIRED_ALIGNMENT.
+/* Depending on ISSETMEM, copy enough from SRCMEM to DESTMEM or set enough to
+ DESTMEM to align it to DESIRED_ALIGNMENT. Original alignment is ALIGN.
+ Depending on ISSETMEM, either arguments SRCMEM/SRCPTR or VALUE/VEC_VALUE are
+ ignored.
Return value is updated DESTMEM. */
static rtx
-expand_movmem_prologue (rtx destmem, rtx srcmem,
- rtx destptr, rtx srcptr, rtx count,
- int align, int desired_alignment)
+expand_set_or_movmem_prologue (rtx destmem, rtx srcmem,
+ rtx destptr, rtx srcptr, rtx value,
+ rtx vec_value, rtx count, int align,
+ int desired_alignment, bool issetmem)
{
int i;
for (i = 1; i < desired_alignment; i <<= 1)
@@ -22605,7 +22874,15 @@ expand_movmem_prologue (rtx destmem, rtx srcmem,
if (align <= i)
{
rtx label = ix86_expand_aligntest (destptr, i, false);
- destmem = emit_memmov (destmem, &srcmem, destptr, srcptr, i);
+ if (issetmem)
+ {
+ if (vec_value && i > GET_MODE_SIZE (GET_MODE (value)))
+ destmem = emit_memset (destmem, destptr, vec_value, i);
+ else
+ destmem = emit_memset (destmem, destptr, value, i);
+ }
+ else
+ destmem = emit_memmov (destmem, &srcmem, destptr, srcptr, i);
ix86_adjust_counter (count, i);
emit_label (label);
LABEL_NUSES (label) = 1;
@@ -22615,191 +22892,482 @@ expand_movmem_prologue (rtx destmem, rtx srcmem,
return destmem;
}
-/* Copy enough from DST to SRC to align DST known to DESIRED_ALIGN.
- ALIGN_BYTES is how many bytes need to be copied.
- The function updates DST and SRC, namely, it sets proper alignment.
- DST is returned via return value, SRC is updated via pointer SRCP. */
-static rtx
-expand_constant_movmem_prologue (rtx dst, rtx *srcp, rtx destreg, rtx srcreg,
- int desired_align, int align_bytes)
-{
- rtx src = *srcp;
- rtx orig_dst = dst;
- rtx orig_src = src;
- int piece_size = 1;
- int copied_bytes = 0;
- int src_align_bytes = get_mem_align_offset (src, desired_align * BITS_PER_UNIT);
- if (src_align_bytes >= 0)
- src_align_bytes = desired_align - src_align_bytes;
+/* Test if COUNT&SIZE is nonzero and if so, expand movme
+ or setmem sequence that is valid for SIZE..2*SIZE-1 bytes
+ and jump to DONE_LABEL. */
+static void
+expand_small_movmem_or_setmem (rtx destmem, rtx srcmem,
+ rtx destptr, rtx srcptr,
+ rtx value, rtx vec_value,
+ rtx count, int size,
+ rtx done_label, bool issetmem)
+{
+ rtx label = ix86_expand_aligntest (count, size, false);
+ enum machine_mode mode = mode_for_size (size * BITS_PER_UNIT, MODE_INT, 1);
+ rtx modesize;
+ int n;
- for (piece_size = 1;
- piece_size <= desired_align && copied_bytes < align_bytes;
- piece_size <<= 1)
+ /* If we do not have vector value to copy, we must reduce size. */
+ if (issetmem)
{
- if (align_bytes & piece_size)
+ if (!vec_value)
{
- dst = emit_memmov (dst, &src, destreg, srcreg, piece_size);
- copied_bytes += piece_size;
+ if (GET_MODE (value) == VOIDmode && size > 8)
+ mode = Pmode;
+ else if (GET_MODE_SIZE (mode) > GET_MODE_SIZE (GET_MODE (value)))
+ mode = GET_MODE (value);
}
+ else
+ mode = GET_MODE (vec_value), value = vec_value;
}
-
- if (MEM_ALIGN (dst) < (unsigned int) desired_align * BITS_PER_UNIT)
- set_mem_align (dst, desired_align * BITS_PER_UNIT);
- if (src_align_bytes >= 0)
+ else
+ {
+ /* Choose appropriate vector mode. */
+ if (size >= 32)
+ mode = TARGET_AVX ? V32QImode : TARGET_SSE ? V16QImode : DImode;
+ else if (size >= 16)
+ mode = TARGET_SSE ? V16QImode : DImode;
+ srcmem = change_address (srcmem, mode, srcptr);
+ }
+ destmem = change_address (destmem, mode, destptr);
+ modesize = GEN_INT (GET_MODE_SIZE (mode));
+ gcc_assert (GET_MODE_SIZE (mode) <= size);
+ for (n = 0; n * GET_MODE_SIZE (mode) < size; n++)
{
- unsigned int src_align;
- for (src_align = desired_align; src_align >= 2; src_align >>= 1)
+ if (issetmem)
+ emit_move_insn (destmem, gen_lowpart (mode, value));
+ else
{
- if ((src_align_bytes & (src_align - 1))
- == (align_bytes & (src_align - 1)))
- break;
+ emit_move_insn (destmem, srcmem);
+ srcmem = offset_address (srcmem, modesize, GET_MODE_SIZE (mode));
}
- if (src_align > (unsigned int) desired_align)
- src_align = desired_align;
- if (MEM_ALIGN (src) < src_align * BITS_PER_UNIT)
- set_mem_align (src, src_align * BITS_PER_UNIT);
+ destmem = offset_address (destmem, modesize, GET_MODE_SIZE (mode));
}
- if (MEM_SIZE_KNOWN_P (orig_dst))
- set_mem_size (dst, MEM_SIZE (orig_dst) - align_bytes);
- if (MEM_SIZE_KNOWN_P (orig_src))
- set_mem_size (src, MEM_SIZE (orig_src) - align_bytes);
- *srcp = src;
- return dst;
+
+ destmem = offset_address (destmem, count, 1);
+ destmem = offset_address (destmem, GEN_INT (-size - GET_MODE_SIZE (mode)),
+ GET_MODE_SIZE (mode));
+ if (issetmem)
+ emit_move_insn (destmem, gen_lowpart (mode, value));
+ else
+ {
+ srcmem = offset_address (srcmem, count, 1);
+ srcmem = offset_address (srcmem, GEN_INT (-size - GET_MODE_SIZE (mode)),
+ GET_MODE_SIZE (mode));
+ emit_move_insn (destmem, srcmem);
+ }
+ emit_jump_insn (gen_jump (done_label));
+ emit_barrier ();
+
+ emit_label (label);
+ LABEL_NUSES (label) = 1;
}
-/* Set enough from DEST to align DEST known to by aligned by ALIGN to
- DESIRED_ALIGNMENT. */
+/* Handle small memcpy (up to SIZE that is supposed to be small power of 2.
+ and get ready for the main memcpy loop by copying iniital DESIRED_ALIGN-ALIGN
+ bytes and last SIZE bytes adjusitng DESTPTR/SRCPTR/COUNT in a way we can
+ proceed with an loop copying SIZE bytes at once. Do moves in MODE.
+ DONE_LABEL is a label after the whole copying sequence. The label is created
+ on demand if *DONE_LABEL is NULL.
+ MIN_SIZE is minimal size of block copied. This value gets adjusted for new
+ bounds after the initial copies.
+
+ DESTMEM/SRCMEM are memory expressions pointing to the copies block,
+ DESTPTR/SRCPTR are pointers to the block. DYNAMIC_CHECK indicate whether
+ we will dispatch to a library call for large blocks.
+
+ In pseudocode we do:
+
+ if (COUNT < SIZE)
+ {
+ Assume that SIZE is 4. Bigger sizes are handled analogously
+ if (COUNT & 4)
+ {
+ copy 4 bytes from SRCPTR to DESTPTR
+ copy 4 bytes from SRCPTR + COUNT - 4 to DESTPTR + COUNT - 4
+ goto done_label
+ }
+ if (!COUNT)
+ goto done_label;
+ copy 1 byte from SRCPTR to DESTPTR
+ if (COUNT & 2)
+ {
+ copy 2 bytes from SRCPTR to DESTPTR
+ copy 2 bytes from SRCPTR + COUNT - 2 to DESTPTR + COUNT - 2
+ }
+ }
+ else
+ {
+ copy at least DESIRED_ALIGN-ALIGN bytes from SRCPTR to DESTPTR
+ copy SIZE bytes from SRCPTR + COUNT - SIZE to DESTPTR + COUNT -SIZE
+
+ OLD_DESPTR = DESTPTR;
+ Align DESTPTR up to DESIRED_ALIGN
+ SRCPTR += DESTPTR - OLD_DESTPTR
+ COUNT -= DEST_PTR - OLD_DESTPTR
+ if (DYNAMIC_CHECK)
+ Round COUNT down to multiple of SIZE
+ << optional caller supplied zero size guard is here >>
+ << optional caller suppplied dynamic check is here >>
+ << caller supplied main copy loop is here >>
+ }
+ done_label:
+ */
static void
-expand_setmem_prologue (rtx destmem, rtx destptr, rtx value, rtx count,
- int align, int desired_alignment)
-{
- if (align <= 1 && desired_alignment > 1)
- {
- rtx label = ix86_expand_aligntest (destptr, 1, false);
- destmem = change_address (destmem, QImode, destptr);
- emit_insn (gen_strset (destptr, destmem, gen_lowpart (QImode, value)));
- ix86_adjust_counter (count, 1);
+expand_set_or_movmem_prologue_epilogue_by_misaligned_moves (rtx destmem, rtx srcmem,
+ rtx *destptr, rtx *srcptr,
+ enum machine_mode mode,
+ rtx value, rtx vec_value,
+ rtx *count,
+ rtx *done_label,
+ int size,
+ int desired_align,
+ int align,
+ unsigned HOST_WIDE_INT *min_size,
+ bool dynamic_check,
+ bool issetmem)
+{
+ rtx loop_label = NULL, label;
+ int n;
+ rtx modesize;
+ int prolog_size = 0;
+ rtx mode_value;
+
+ /* Chose proper value to copy. */
+ if (issetmem && VECTOR_MODE_P (mode))
+ mode_value = vec_value;
+ else
+ mode_value = value;
+ gcc_assert (GET_MODE_SIZE (mode) <= size);
+
+ /* See if block is big or small, handle small blocks. */
+ if (!CONST_INT_P (*count) && *min_size < (unsigned HOST_WIDE_INT)size)
+ {
+ int size2 = size;
+ loop_label = gen_label_rtx ();
+
+ if (!*done_label)
+ *done_label = gen_label_rtx ();
+
+ emit_cmp_and_jump_insns (*count, GEN_INT (size2), GE, 0, GET_MODE (*count),
+ 1, loop_label);
+ size2 >>= 1;
+
+ /* Handle sizes > 3. */
+ for (;size2 > 2; size2 >>= 1)
+ expand_small_movmem_or_setmem (destmem, srcmem,
+ *destptr, *srcptr,
+ value, vec_value,
+ *count,
+ size2, *done_label, issetmem);
+ /* Nothing to copy? Jump to DONE_LABEL if so */
+ emit_cmp_and_jump_insns (*count, const0_rtx, EQ, 0, GET_MODE (*count),
+ 1, *done_label);
+
+ /* Do a byte copy. */
+ destmem = change_address (destmem, QImode, *destptr);
+ if (issetmem)
+ emit_move_insn (destmem, gen_lowpart (QImode, value));
+ else
+ {
+ srcmem = change_address (srcmem, QImode, *srcptr);
+ emit_move_insn (destmem, srcmem);
+ }
+
+ /* Handle sizes 2 and 3. */
+ label = ix86_expand_aligntest (*count, 2, false);
+ destmem = change_address (destmem, HImode, *destptr);
+ destmem = offset_address (destmem, *count, 1);
+ destmem = offset_address (destmem, GEN_INT (-2), 2);
+ if (issetmem)
+ emit_move_insn (destmem, gen_lowpart (HImode, value));
+ else
+ {
+ srcmem = change_address (srcmem, HImode, *srcptr);
+ srcmem = offset_address (srcmem, *count, 1);
+ srcmem = offset_address (srcmem, GEN_INT (-2), 2);
+ emit_move_insn (destmem, srcmem);
+ }
+
emit_label (label);
LABEL_NUSES (label) = 1;
+ emit_jump_insn (gen_jump (*done_label));
+ emit_barrier ();
}
- if (align <= 2 && desired_alignment > 2)
+ else
+ gcc_assert (*min_size >= (unsigned HOST_WIDE_INT)size
+ || UINTVAL (*count) >= (unsigned HOST_WIDE_INT)size);
+
+ /* Start memcpy for COUNT >= SIZE. */
+ if (loop_label)
{
- rtx label = ix86_expand_aligntest (destptr, 2, false);
- destmem = change_address (destmem, HImode, destptr);
- emit_insn (gen_strset (destptr, destmem, gen_lowpart (HImode, value)));
- ix86_adjust_counter (count, 2);
- emit_label (label);
- LABEL_NUSES (label) = 1;
+ emit_label (loop_label);
+ LABEL_NUSES (loop_label) = 1;
}
- if (align <= 4 && desired_alignment > 4)
+
+ /* Copy first desired_align bytes. */
+ if (!issetmem)
+ srcmem = change_address (srcmem, mode, *srcptr);
+ destmem = change_address (destmem, mode, *destptr);
+ modesize = GEN_INT (GET_MODE_SIZE (mode));
+ for (n = 0; prolog_size < desired_align - align; n++)
{
- rtx label = ix86_expand_aligntest (destptr, 4, false);
- destmem = change_address (destmem, SImode, destptr);
- emit_insn (gen_strset (destptr, destmem, gen_lowpart (SImode, value)));
- ix86_adjust_counter (count, 4);
- emit_label (label);
- LABEL_NUSES (label) = 1;
+ if (issetmem)
+ emit_move_insn (destmem, mode_value);
+ else
+ {
+ emit_move_insn (destmem, srcmem);
+ srcmem = offset_address (srcmem, modesize, GET_MODE_SIZE (mode));
+ }
+ destmem = offset_address (destmem, modesize, GET_MODE_SIZE (mode));
+ prolog_size += GET_MODE_SIZE (mode);
+ }
+
+
+ /* Copy last SIZE bytes. */
+ destmem = offset_address (destmem, *count, 1);
+ destmem = offset_address (destmem,
+ GEN_INT (-size - prolog_size),
+ 1);
+ if (issetmem)
+ emit_move_insn (destmem, mode_value);
+ else
+ {
+ srcmem = offset_address (srcmem, *count, 1);
+ srcmem = offset_address (srcmem,
+ GEN_INT (-size - prolog_size),
+ 1);
+ emit_move_insn (destmem, srcmem);
+ }
+ for (n = 1; n * GET_MODE_SIZE (mode) < size; n++)
+ {
+ destmem = offset_address (destmem, modesize, 1);
+ if (issetmem)
+ emit_move_insn (destmem, mode_value);
+ else
+ {
+ srcmem = offset_address (srcmem, modesize, 1);
+ emit_move_insn (destmem, srcmem);
+ }
+ }
+
+ /* Align destination. */
+ if (desired_align > 1 && desired_align > align)
+ {
+ rtx saveddest = *destptr;
+
+ gcc_assert (desired_align <= size);
+ /* Align destptr up, place it to new register. */
+ *destptr = expand_simple_binop (GET_MODE (*destptr), PLUS, *destptr,
+ GEN_INT (prolog_size),
+ NULL_RTX, 1, OPTAB_DIRECT);
+ *destptr = expand_simple_binop (GET_MODE (*destptr), AND, *destptr,
+ GEN_INT (-desired_align),
+ *destptr, 1, OPTAB_DIRECT);
+ /* See how many bytes we skipped. */
+ saveddest = expand_simple_binop (GET_MODE (*destptr), MINUS, saveddest,
+ *destptr,
+ saveddest, 1, OPTAB_DIRECT);
+ /* Adjust srcptr and count. */
+ if (!issetmem)
+ *srcptr = expand_simple_binop (GET_MODE (*srcptr), MINUS, *srcptr, saveddest,
+ *srcptr, 1, OPTAB_DIRECT);
+ *count = expand_simple_binop (GET_MODE (*count), PLUS, *count,
+ saveddest, *count, 1, OPTAB_DIRECT);
+ /* We copied at most size + prolog_size. */
+ if (*min_size > (unsigned HOST_WIDE_INT)(size + prolog_size))
+ *min_size = (*min_size - size) & ~(unsigned HOST_WIDE_INT)(size - 1);
+ else
+ *min_size = 0;
+
+ /* Our loops always round down the bock size, but for dispatch to library
+ we need precise value. */
+ if (dynamic_check)
+ *count = expand_simple_binop (GET_MODE (*count), AND, *count,
+ GEN_INT (-size), *count, 1, OPTAB_DIRECT);
+ }
+ else
+ {
+ gcc_assert (prolog_size == 0);
+ /* Decrease count, so we won't end up copying last word twice. */
+ if (!CONST_INT_P (*count))
+ *count = expand_simple_binop (GET_MODE (*count), PLUS, *count,
+ constm1_rtx, *count, 1, OPTAB_DIRECT);
+ else
+ *count = GEN_INT ((UINTVAL (*count) - 1) & ~(unsigned HOST_WIDE_INT)(size - 1));
+ if (*min_size)
+ *min_size = (*min_size - 1) & ~(unsigned HOST_WIDE_INT)(size - 1);
}
- gcc_assert (desired_alignment <= 8);
}
-/* Set enough from DST to align DST known to by aligned by ALIGN to
- DESIRED_ALIGN. ALIGN_BYTES is how many bytes need to be stored. */
+
+/* This function is like the previous one, except here we know how many bytes
+ need to be copied. That allows us to update alignment not only of DST, which
+ is returned, but also of SRC, which is passed as a pointer for that
+ reason. */
static rtx
-expand_constant_setmem_prologue (rtx dst, rtx destreg, rtx value,
- int desired_align, int align_bytes)
+expand_set_or_movmem_constant_prologue (rtx dst, rtx *srcp, rtx destreg,
+ rtx srcreg, rtx value, rtx vec_value,
+ int desired_align, int align_bytes,
+ bool issetmem)
{
- int off = 0;
+ rtx src = NULL;
rtx orig_dst = dst;
- if (align_bytes & 1)
- {
- dst = adjust_automodify_address_nv (dst, QImode, destreg, 0);
- off = 1;
- emit_insn (gen_strset (destreg, dst,
- gen_lowpart (QImode, value)));
- }
- if (align_bytes & 2)
+ rtx orig_src = NULL;
+ int piece_size = 1;
+ int copied_bytes = 0;
+
+ if (!issetmem)
{
- dst = adjust_automodify_address_nv (dst, HImode, destreg, off);
- if (MEM_ALIGN (dst) < 2 * BITS_PER_UNIT)
- set_mem_align (dst, 2 * BITS_PER_UNIT);
- off = 2;
- emit_insn (gen_strset (destreg, dst,
- gen_lowpart (HImode, value)));
+ gcc_assert (srcp != NULL);
+ src = *srcp;
+ orig_src = src;
}
- if (align_bytes & 4)
+
+ for (piece_size = 1;
+ piece_size <= desired_align && copied_bytes < align_bytes;
+ piece_size <<= 1)
{
- dst = adjust_automodify_address_nv (dst, SImode, destreg, off);
- if (MEM_ALIGN (dst) < 4 * BITS_PER_UNIT)
- set_mem_align (dst, 4 * BITS_PER_UNIT);
- off = 4;
- emit_insn (gen_strset (destreg, dst,
- gen_lowpart (SImode, value)));
+ if (align_bytes & piece_size)
+ {
+ if (issetmem)
+ {
+ if (vec_value && piece_size > GET_MODE_SIZE (GET_MODE (value)))
+ dst = emit_memset (dst, destreg, vec_value, piece_size);
+ else
+ dst = emit_memset (dst, destreg, value, piece_size);
+ }
+ else
+ dst = emit_memmov (dst, &src, destreg, srcreg, piece_size);
+ copied_bytes += piece_size;
+ }
}
- dst = adjust_automodify_address_nv (dst, BLKmode, destreg, off);
if (MEM_ALIGN (dst) < (unsigned int) desired_align * BITS_PER_UNIT)
set_mem_align (dst, desired_align * BITS_PER_UNIT);
if (MEM_SIZE_KNOWN_P (orig_dst))
set_mem_size (dst, MEM_SIZE (orig_dst) - align_bytes);
+
+ if (!issetmem)
+ {
+ int src_align_bytes = get_mem_align_offset (src, desired_align
+ * BITS_PER_UNIT);
+ if (src_align_bytes >= 0)
+ src_align_bytes = desired_align - src_align_bytes;
+ if (src_align_bytes >= 0)
+ {
+ unsigned int src_align;
+ for (src_align = desired_align; src_align >= 2; src_align >>= 1)
+ {
+ if ((src_align_bytes & (src_align - 1))
+ == (align_bytes & (src_align - 1)))
+ break;
+ }
+ if (src_align > (unsigned int) desired_align)
+ src_align = desired_align;
+ if (MEM_ALIGN (src) < src_align * BITS_PER_UNIT)
+ set_mem_align (src, src_align * BITS_PER_UNIT);
+ }
+ if (MEM_SIZE_KNOWN_P (orig_src))
+ set_mem_size (src, MEM_SIZE (orig_src) - align_bytes);
+ *srcp = src;
+ }
+
return dst;
}
-/* Given COUNT and EXPECTED_SIZE, decide on codegen of string operation. */
-static enum stringop_alg
-decide_alg (HOST_WIDE_INT count, HOST_WIDE_INT expected_size, bool memset,
- int *dynamic_check, bool *noalign)
+/* Return true if ALG can be used in current context.
+ Assume we expand memset if MEMSET is true. */
+static bool
+alg_usable_p (enum stringop_alg alg, bool memset)
{
- const struct stringop_algs * algs;
- bool optimize_for_speed;
+ if (alg == no_stringop)
+ return false;
+ if (alg == vector_loop)
+ return TARGET_SSE || TARGET_AVX;
/* Algorithms using the rep prefix want at least edi and ecx;
additionally, memset wants eax and memcpy wants esi. Don't
consider such algorithms if the user has appropriated those
registers for their own purposes. */
- bool rep_prefix_usable = !(fixed_regs[CX_REG] || fixed_regs[DI_REG]
- || (memset
- ? fixed_regs[AX_REG] : fixed_regs[SI_REG]));
- *noalign = false;
+ if (alg == rep_prefix_1_byte
+ || alg == rep_prefix_4_byte
+ || alg == rep_prefix_8_byte)
+ return !(fixed_regs[CX_REG] || fixed_regs[DI_REG]
+ || (memset ? fixed_regs[AX_REG] : fixed_regs[SI_REG]));
+ return true;
+}
-#define ALG_USABLE_P(alg) (rep_prefix_usable \
- || (alg != rep_prefix_1_byte \
- && alg != rep_prefix_4_byte \
- && alg != rep_prefix_8_byte))
+/* Given COUNT and EXPECTED_SIZE, decide on codegen of string operation. */
+static enum stringop_alg
+decide_alg (HOST_WIDE_INT count, HOST_WIDE_INT expected_size,
+ unsigned HOST_WIDE_INT min_size, unsigned HOST_WIDE_INT max_size,
+ bool memset, bool zero_memset, int *dynamic_check, bool *noalign)
+{
+ const struct stringop_algs * algs;
+ bool optimize_for_speed;
+ int max = -1;
const struct processor_costs *cost;
+ int i;
+ bool any_alg_usable_p = false;
+
+ *noalign = false;
+ *dynamic_check = -1;
/* Even if the string operation call is cold, we still might spend a lot
of time processing large blocks. */
if (optimize_function_for_size_p (cfun)
|| (optimize_insn_for_size_p ()
- && expected_size != -1 && expected_size < 256))
+ && (max_size < 256
+ || (expected_size != -1 && expected_size < 256))))
optimize_for_speed = false;
else
optimize_for_speed = true;
cost = optimize_for_speed ? ix86_cost : &ix86_size_cost;
-
- *dynamic_check = -1;
if (memset)
algs = &cost->memset[TARGET_64BIT != 0];
else
algs = &cost->memcpy[TARGET_64BIT != 0];
- if (ix86_stringop_alg != no_stringop && ALG_USABLE_P (ix86_stringop_alg))
+
+ /* See maximal size for user defined algorithm. */
+ for (i = 0; i < MAX_STRINGOP_ALGS; i++)
+ {
+ enum stringop_alg candidate = algs->size[i].alg;
+ bool usable = alg_usable_p (candidate, memset);
+ any_alg_usable_p |= usable;
+
+ if (candidate != libcall && candidate && usable)
+ max = algs->size[i].max;
+ }
+
+ /* If expected size is not known but max size is small enough
+ so inline version is a win, set expected size into
+ the range. */
+ if (max > 1 && (unsigned HOST_WIDE_INT)max >= max_size && expected_size == -1)
+ expected_size = min_size / 2 + max_size / 2;
+
+ /* If user specified the algorithm, honnor it if possible. */
+ if (ix86_stringop_alg != no_stringop
+ && alg_usable_p (ix86_stringop_alg, memset))
return ix86_stringop_alg;
/* rep; movq or rep; movl is the smallest variant. */
else if (!optimize_for_speed)
{
- if (!count || (count & 3))
- return rep_prefix_usable ? rep_prefix_1_byte : loop_1_byte;
+ *noalign = true;
+ if (!count || (count & 3) || (memset && !zero_memset))
+ return alg_usable_p (rep_prefix_1_byte, memset)
+ ? rep_prefix_1_byte : loop_1_byte;
else
- return rep_prefix_usable ? rep_prefix_4_byte : loop;
+ return alg_usable_p (rep_prefix_4_byte, memset)
+ ? rep_prefix_4_byte : loop;
}
- /* Very tiny blocks are best handled via the loop, REP is expensive to setup.
- */
+ /* Very tiny blocks are best handled via the loop, REP is expensive to
+ setup. */
else if (expected_size != -1 && expected_size < 4)
return loop_1_byte;
else if (expected_size != -1)
{
- unsigned int i;
enum stringop_alg alg = libcall;
+ bool alg_noalign = false;
for (i = 0; i < MAX_STRINGOP_ALGS; i++)
{
/* We get here if the algorithms that were not libcall-based
@@ -22812,8 +23380,11 @@ decide_alg (HOST_WIDE_INT count, HOST_WIDE_INT expected_size, bool memset,
{
enum stringop_alg candidate = algs->size[i].alg;
- if (candidate != libcall && ALG_USABLE_P (candidate))
- alg = candidate;
+ if (candidate != libcall && alg_usable_p (candidate, memset))
+ {
+ alg = candidate;
+ alg_noalign = algs->size[i].noalign;
+ }
/* Honor TARGET_INLINE_ALL_STRINGOPS by picking
last non-libcall inline algorithm. */
if (TARGET_INLINE_ALL_STRINGOPS)
@@ -22822,17 +23393,19 @@ decide_alg (HOST_WIDE_INT count, HOST_WIDE_INT expected_size, bool memset,
but we are still forced to inline, run the heuristic below
that will pick code for medium sized blocks. */
if (alg != libcall)
- return alg;
+ {
+ *noalign = alg_noalign;
+ return alg;
+ }
break;
}
- else if (ALG_USABLE_P (candidate))
+ else if (alg_usable_p (candidate, memset))
{
*noalign = algs->size[i].noalign;
return candidate;
}
}
}
- gcc_assert (TARGET_INLINE_ALL_STRINGOPS || !rep_prefix_usable);
}
/* When asked to inline the call anyway, try to pick meaningful choice.
We look for maximal size of block that is faster to copy by hand and
@@ -22842,22 +23415,11 @@ decide_alg (HOST_WIDE_INT count, HOST_WIDE_INT expected_size, bool memset,
If this turns out to be bad, we might simply specify the preferred
choice in ix86_costs. */
if ((TARGET_INLINE_ALL_STRINGOPS || TARGET_INLINE_STRINGOPS_DYNAMICALLY)
- && (algs->unknown_size == libcall || !ALG_USABLE_P (algs->unknown_size)))
+ && (algs->unknown_size == libcall
+ || !alg_usable_p (algs->unknown_size, memset)))
{
- int max = -1;
enum stringop_alg alg;
- int i;
- bool any_alg_usable_p = true;
- for (i = 0; i < MAX_STRINGOP_ALGS; i++)
- {
- enum stringop_alg candidate = algs->size[i].alg;
- any_alg_usable_p = any_alg_usable_p && ALG_USABLE_P (candidate);
-
- if (candidate != libcall && candidate
- && ALG_USABLE_P (candidate))
- max = algs->size[i].max;
- }
/* If there aren't any usable algorithms, then recursing on
smaller sizes isn't going to find anything. Just return the
simple byte-at-a-time copy loop. */
@@ -22870,15 +23432,16 @@ decide_alg (HOST_WIDE_INT count, HOST_WIDE_INT expected_size, bool memset,
}
if (max == -1)
max = 4096;
- alg = decide_alg (count, max / 2, memset, dynamic_check, noalign);
+ alg = decide_alg (count, max / 2, min_size, max_size, memset,
+ zero_memset, dynamic_check, noalign);
gcc_assert (*dynamic_check == -1);
gcc_assert (alg != libcall);
if (TARGET_INLINE_STRINGOPS_DYNAMICALLY)
*dynamic_check = max;
return alg;
}
- return ALG_USABLE_P (algs->unknown_size) ? algs->unknown_size : libcall;
-#undef ALG_USABLE_P
+ return (alg_usable_p (algs->unknown_size, memset)
+ ? algs->unknown_size : libcall);
}
/* Decide on alignment. We know that the operand is already aligned to ALIGN
@@ -22915,341 +23478,6 @@ decide_alignment (int align,
return desired_align;
}
-/* Expand string move (memcpy) operation. Use i386 string operations
- when profitable. expand_setmem contains similar code. The code
- depends upon architecture, block size and alignment, but always has
- the same overall structure:
-
- 1) Prologue guard: Conditional that jumps up to epilogues for small
- blocks that can be handled by epilogue alone. This is faster
- but also needed for correctness, since prologue assume the block
- is larger than the desired alignment.
-
- Optional dynamic check for size and libcall for large
- blocks is emitted here too, with -minline-stringops-dynamically.
-
- 2) Prologue: copy first few bytes in order to get destination
- aligned to DESIRED_ALIGN. It is emitted only when ALIGN is less
- than DESIRED_ALIGN and up to DESIRED_ALIGN - ALIGN bytes can be
- copied. We emit either a jump tree on power of two sized
- blocks, or a byte loop.
-
- 3) Main body: the copying loop itself, copying in SIZE_NEEDED chunks
- with specified algorithm.
-
- 4) Epilogue: code copying tail of the block that is too small to be
- handled by main body (or up to size guarded by prologue guard). */
-
-bool
-ix86_expand_movmem (rtx dst, rtx src, rtx count_exp, rtx align_exp,
- rtx expected_align_exp, rtx expected_size_exp)
-{
- rtx destreg;
- rtx srcreg;
- rtx label = NULL;
- rtx tmp;
- rtx jump_around_label = NULL;
- HOST_WIDE_INT align = 1;
- unsigned HOST_WIDE_INT count = 0;
- HOST_WIDE_INT expected_size = -1;
- int size_needed = 0, epilogue_size_needed;
- int desired_align = 0, align_bytes = 0;
- enum stringop_alg alg;
- int dynamic_check;
- bool need_zero_guard = false;
- bool noalign;
- enum machine_mode move_mode = VOIDmode;
- int unroll_factor = 1;
-
- if (CONST_INT_P (align_exp))
- align = INTVAL (align_exp);
- /* i386 can do misaligned access on reasonably increased cost. */
- if (CONST_INT_P (expected_align_exp)
- && INTVAL (expected_align_exp) > align)
- align = INTVAL (expected_align_exp);
- /* ALIGN is the minimum of destination and source alignment, but we care here
- just about destination alignment. */
- else if (MEM_ALIGN (dst) > (unsigned HOST_WIDE_INT) align * BITS_PER_UNIT)
- align = MEM_ALIGN (dst) / BITS_PER_UNIT;
-
- if (CONST_INT_P (count_exp))
- count = expected_size = INTVAL (count_exp);
- if (CONST_INT_P (expected_size_exp) && count == 0)
- expected_size = INTVAL (expected_size_exp);
-
- /* Make sure we don't need to care about overflow later on. */
- if (count > ((unsigned HOST_WIDE_INT) 1 << 30))
- return false;
-
- /* Step 0: Decide on preferred algorithm, desired alignment and
- size of chunks to be copied by main loop. */
- alg = decide_alg (count, expected_size, false, &dynamic_check, &noalign);
- if (alg == libcall)
- return false;
- gcc_assert (alg != no_stringop);
-
- if (!count)
- count_exp = copy_to_mode_reg (GET_MODE (count_exp), count_exp);
- destreg = copy_addr_to_reg (XEXP (dst, 0));
- srcreg = copy_addr_to_reg (XEXP (src, 0));
-
- unroll_factor = 1;
- move_mode = word_mode;
- switch (alg)
- {
- case libcall:
- case no_stringop:
- case last_alg:
- gcc_unreachable ();
- case loop_1_byte:
- need_zero_guard = true;
- move_mode = QImode;
- break;
- case loop:
- need_zero_guard = true;
- break;
- case unrolled_loop:
- need_zero_guard = true;
- unroll_factor = (TARGET_64BIT ? 4 : 2);
- break;
- case vector_loop:
- need_zero_guard = true;
- unroll_factor = 4;
- /* Find the widest supported mode. */
- move_mode = word_mode;
- while (optab_handler (mov_optab, GET_MODE_WIDER_MODE (move_mode))
- != CODE_FOR_nothing)
- move_mode = GET_MODE_WIDER_MODE (move_mode);
-
- /* Find the corresponding vector mode with the same size as MOVE_MODE.
- MOVE_MODE is an integer mode at the moment (SI, DI, TI, etc.). */
- if (GET_MODE_SIZE (move_mode) > GET_MODE_SIZE (word_mode))
- {
- int nunits = GET_MODE_SIZE (move_mode) / GET_MODE_SIZE (word_mode);
- move_mode = mode_for_vector (word_mode, nunits);
- if (optab_handler (mov_optab, move_mode) == CODE_FOR_nothing)
- move_mode = word_mode;
- }
- gcc_assert (optab_handler (mov_optab, move_mode) != CODE_FOR_nothing);
- break;
- case rep_prefix_8_byte:
- move_mode = DImode;
- break;
- case rep_prefix_4_byte:
- move_mode = SImode;
- break;
- case rep_prefix_1_byte:
- move_mode = QImode;
- break;
- }
- size_needed = GET_MODE_SIZE (move_mode) * unroll_factor;
- epilogue_size_needed = size_needed;
-
- desired_align = decide_alignment (align, alg, expected_size, move_mode);
- if (!TARGET_ALIGN_STRINGOPS || noalign)
- align = desired_align;
-
- /* Step 1: Prologue guard. */
-
- /* Alignment code needs count to be in register. */
- if (CONST_INT_P (count_exp) && desired_align > align)
- {
- if (INTVAL (count_exp) > desired_align
- && INTVAL (count_exp) > size_needed)
- {
- align_bytes
- = get_mem_align_offset (dst, desired_align * BITS_PER_UNIT);
- if (align_bytes <= 0)
- align_bytes = 0;
- else
- align_bytes = desired_align - align_bytes;
- }
- if (align_bytes == 0)
- count_exp = force_reg (counter_mode (count_exp), count_exp);
- }
- gcc_assert (desired_align >= 1 && align >= 1);
-
- /* Ensure that alignment prologue won't copy past end of block. */
- if (size_needed > 1 || (desired_align > 1 && desired_align > align))
- {
- epilogue_size_needed = MAX (size_needed - 1, desired_align - align);
- /* Epilogue always copies COUNT_EXP & EPILOGUE_SIZE_NEEDED bytes.
- Make sure it is power of 2. */
- epilogue_size_needed = 1 << (floor_log2 (epilogue_size_needed) + 1);
-
- if (count)
- {
- if (count < (unsigned HOST_WIDE_INT)epilogue_size_needed)
- {
- /* If main algorithm works on QImode, no epilogue is needed.
- For small sizes just don't align anything. */
- if (size_needed == 1)
- desired_align = align;
- else
- goto epilogue;
- }
- }
- else
- {
- label = gen_label_rtx ();
- emit_cmp_and_jump_insns (count_exp,
- GEN_INT (epilogue_size_needed),
- LTU, 0, counter_mode (count_exp), 1, label);
- if (expected_size == -1 || expected_size < epilogue_size_needed)
- predict_jump (REG_BR_PROB_BASE * 60 / 100);
- else
- predict_jump (REG_BR_PROB_BASE * 20 / 100);
- }
- }
-
- /* Emit code to decide on runtime whether library call or inline should be
- used. */
- if (dynamic_check != -1)
- {
- if (CONST_INT_P (count_exp))
- {
- if (UINTVAL (count_exp) >= (unsigned HOST_WIDE_INT)dynamic_check)
- {
- emit_block_move_via_libcall (dst, src, count_exp, false);
- count_exp = const0_rtx;
- goto epilogue;
- }
- }
- else
- {
- rtx hot_label = gen_label_rtx ();
- jump_around_label = gen_label_rtx ();
- emit_cmp_and_jump_insns (count_exp, GEN_INT (dynamic_check - 1),
- LEU, 0, GET_MODE (count_exp), 1, hot_label);
- predict_jump (REG_BR_PROB_BASE * 90 / 100);
- emit_block_move_via_libcall (dst, src, count_exp, false);
- emit_jump (jump_around_label);
- emit_label (hot_label);
- }
- }
-
- /* Step 2: Alignment prologue. */
-
- if (desired_align > align)
- {
- if (align_bytes == 0)
- {
- /* Except for the first move in epilogue, we no longer know
- constant offset in aliasing info. It don't seems to worth
- the pain to maintain it for the first move, so throw away
- the info early. */
- src = change_address (src, BLKmode, srcreg);
- dst = change_address (dst, BLKmode, destreg);
- dst = expand_movmem_prologue (dst, src, destreg, srcreg, count_exp, align,
- desired_align);
- }
- else
- {
- /* If we know how many bytes need to be stored before dst is
- sufficiently aligned, maintain aliasing info accurately. */
- dst = expand_constant_movmem_prologue (dst, &src, destreg, srcreg,
- desired_align, align_bytes);
- count_exp = plus_constant (counter_mode (count_exp),
- count_exp, -align_bytes);
- count -= align_bytes;
- }
- if (need_zero_guard
- && (count < (unsigned HOST_WIDE_INT) size_needed
- || (align_bytes == 0
- && count < ((unsigned HOST_WIDE_INT) size_needed
- + desired_align - align))))
- {
- /* It is possible that we copied enough so the main loop will not
- execute. */
- gcc_assert (size_needed > 1);
- if (label == NULL_RTX)
- label = gen_label_rtx ();
- emit_cmp_and_jump_insns (count_exp,
- GEN_INT (size_needed),
- LTU, 0, counter_mode (count_exp), 1, label);
- if (expected_size == -1
- || expected_size < (desired_align - align) / 2 + size_needed)
- predict_jump (REG_BR_PROB_BASE * 20 / 100);
- else
- predict_jump (REG_BR_PROB_BASE * 60 / 100);
- }
- }
- if (label && size_needed == 1)
- {
- emit_label (label);
- LABEL_NUSES (label) = 1;
- label = NULL;
- epilogue_size_needed = 1;
- }
- else if (label == NULL_RTX)
- epilogue_size_needed = size_needed;
-
- /* Step 3: Main loop. */
-
- switch (alg)
- {
- case libcall:
- case no_stringop:
- case last_alg:
- gcc_unreachable ();
- case loop_1_byte:
- case loop:
- case unrolled_loop:
- case vector_loop:
- expand_set_or_movmem_via_loop (dst, src, destreg, srcreg, NULL,
- count_exp, move_mode, unroll_factor,
- expected_size);
- break;
- case rep_prefix_8_byte:
- case rep_prefix_4_byte:
- case rep_prefix_1_byte:
- expand_movmem_via_rep_mov (dst, src, destreg, srcreg, count_exp,
- move_mode);
- break;
- }
- /* Adjust properly the offset of src and dest memory for aliasing. */
- if (CONST_INT_P (count_exp))
- {
- src = adjust_automodify_address_nv (src, BLKmode, srcreg,
- (count / size_needed) * size_needed);
- dst = adjust_automodify_address_nv (dst, BLKmode, destreg,
- (count / size_needed) * size_needed);
- }
- else
- {
- src = change_address (src, BLKmode, srcreg);
- dst = change_address (dst, BLKmode, destreg);
- }
-
- /* Step 4: Epilogue to copy the remaining bytes. */
- epilogue:
- if (label)
- {
- /* When the main loop is done, COUNT_EXP might hold original count,
- while we want to copy only COUNT_EXP & SIZE_NEEDED bytes.
- Epilogue code will actually copy COUNT_EXP & EPILOGUE_SIZE_NEEDED
- bytes. Compensate if needed. */
-
- if (size_needed < epilogue_size_needed)
- {
- tmp =
- expand_simple_binop (counter_mode (count_exp), AND, count_exp,
- GEN_INT (size_needed - 1), count_exp, 1,
- OPTAB_DIRECT);
- if (tmp != count_exp)
- emit_move_insn (count_exp, tmp);
- }
- emit_label (label);
- LABEL_NUSES (label) = 1;
- }
-
- if (count_exp != const0_rtx && epilogue_size_needed > 1)
- expand_movmem_epilogue (dst, src, destreg, srcreg, count_exp,
- epilogue_size_needed);
- if (jump_around_label)
- emit_label (jump_around_label);
- return true;
-}
/* Helper function for memcpy. For QImode value 0xXY produce
0xXYXYXYXY of wide specified by MODE. This is essentially
@@ -23263,9 +23491,9 @@ promote_duplicated_reg (enum machine_mode mode, rtx val)
rtx tmp;
int nops = mode == DImode ? 3 : 2;
- gcc_assert (mode == SImode || mode == DImode);
+ gcc_assert (mode == SImode || mode == DImode || val == const0_rtx);
if (val == const0_rtx)
- return copy_to_mode_reg (mode, const0_rtx);
+ return copy_to_mode_reg (mode, CONST0_RTX (mode));
if (CONST_INT_P (val))
{
HOST_WIDE_INT v = INTVAL (val) & 255;
@@ -23327,7 +23555,8 @@ promote_duplicated_reg (enum machine_mode mode, rtx val)
be needed by main loop copying SIZE_NEEDED chunks and prologue getting
alignment from ALIGN to DESIRED_ALIGN. */
static rtx
-promote_duplicated_reg_to_size (rtx val, int size_needed, int desired_align, int align)
+promote_duplicated_reg_to_size (rtx val, int size_needed, int desired_align,
+ int align)
{
rtx promoted_val;
@@ -23344,14 +23573,55 @@ promote_duplicated_reg_to_size (rtx val, int size_needed, int desired_align, int
return promoted_val;
}
-/* Expand string clear operation (bzero). Use i386 string operations when
- profitable. See expand_movmem comment for explanation of individual
- steps performed. */
-bool
-ix86_expand_setmem (rtx dst, rtx count_exp, rtx val_exp, rtx align_exp,
- rtx expected_align_exp, rtx expected_size_exp)
+/* Expand string move (memcpy) ot store (memset) operation. Use i386 string
+ operations when profitable. The code depends upon architecture, block size
+ and alignment, but always has one of the following overall structures:
+
+ Aligned move sequence:
+
+ 1) Prologue guard: Conditional that jumps up to epilogues for small
+ blocks that can be handled by epilogue alone. This is faster
+ but also needed for correctness, since prologue assume the block
+ is larger than the desired alignment.
+
+ Optional dynamic check for size and libcall for large
+ blocks is emitted here too, with -minline-stringops-dynamically.
+
+ 2) Prologue: copy first few bytes in order to get destination
+ aligned to DESIRED_ALIGN. It is emitted only when ALIGN is less
+ than DESIRED_ALIGN and up to DESIRED_ALIGN - ALIGN bytes can be
+ copied. We emit either a jump tree on power of two sized
+ blocks, or a byte loop.
+
+ 3) Main body: the copying loop itself, copying in SIZE_NEEDED chunks
+ with specified algorithm.
+
+ 4) Epilogue: code copying tail of the block that is too small to be
+ handled by main body (or up to size guarded by prologue guard).
+
+ Misaligned move sequence
+
+ 1) missaligned move prologue/epilogue containing:
+ a) Prologue handling small memory blocks and jumping to done_label
+ (skipped if blocks are known to be large enough)
+ b) Signle move copying first DESIRED_ALIGN-ALIGN bytes if alignment is
+ needed by single possibly misaligned move
+ (skipped if alignment is not needed)
+ c) Copy of last SIZE_NEEDED bytes by possibly misaligned moves
+
+ 2) Zero size guard dispatching to done_label, if needed
+
+ 3) dispatch to library call, if needed,
+
+ 3) Main body: the copying loop itself, copying in SIZE_NEEDED chunks
+ with specified algorithm. */
+static bool
+ix86_expand_set_or_movmem (rtx dst, rtx src, rtx count_exp, rtx val_exp,
+ rtx align_exp, rtx expected_align_exp,
+ rtx expected_size_exp, bool issetmem)
{
rtx destreg;
+ rtx srcreg = NULL;
rtx label = NULL;
rtx tmp;
rtx jump_around_label = NULL;
@@ -23362,12 +23632,17 @@ ix86_expand_setmem (rtx dst, rtx count_exp, rtx val_exp, rtx align_exp,
int desired_align = 0, align_bytes = 0;
enum stringop_alg alg;
rtx promoted_val = NULL;
+ rtx vec_promoted_val = NULL;
bool force_loopy_epilogue = false;
int dynamic_check;
bool need_zero_guard = false;
bool noalign;
enum machine_mode move_mode = VOIDmode;
- int unroll_factor;
+ int unroll_factor = 1;
+ /* TODO: Once vlaue ranges are available, fill in proper data. */
+ unsigned HOST_WIDE_INT min_size = 0;
+ unsigned HOST_WIDE_INT max_size = -1;
+ bool misaligned_prologue_used = false;
if (CONST_INT_P (align_exp))
align = INTVAL (align_exp);
@@ -23375,8 +23650,14 @@ ix86_expand_setmem (rtx dst, rtx count_exp, rtx val_exp, rtx align_exp,
if (CONST_INT_P (expected_align_exp)
&& INTVAL (expected_align_exp) > align)
align = INTVAL (expected_align_exp);
+ /* ALIGN is the minimum of destination and source alignment, but we care here
+ just about destination alignment. */
+ else if (!issetmem
+ && MEM_ALIGN (dst) > (unsigned HOST_WIDE_INT) align * BITS_PER_UNIT)
+ align = MEM_ALIGN (dst) / BITS_PER_UNIT;
+
if (CONST_INT_P (count_exp))
- count = expected_size = INTVAL (count_exp);
+ min_size = max_size = count = expected_size = INTVAL (count_exp);
if (CONST_INT_P (expected_size_exp) && count == 0)
expected_size = INTVAL (expected_size_exp);
@@ -23386,31 +23667,62 @@ ix86_expand_setmem (rtx dst, rtx count_exp, rtx val_exp, rtx align_exp,
/* Step 0: Decide on preferred algorithm, desired alignment and
size of chunks to be copied by main loop. */
-
- alg = decide_alg (count, expected_size, true, &dynamic_check, &noalign);
+ alg = decide_alg (count, expected_size, min_size, max_size, issetmem,
+ issetmem && val_exp == const0_rtx,
+ &dynamic_check, &noalign);
if (alg == libcall)
return false;
gcc_assert (alg != no_stringop);
+ /* For now vector-version of memset is generated only for memory zeroing, as
+ creating of promoted vector value is very cheap in this case. */
+ if (issetmem && alg == vector_loop && val_exp != const0_rtx)
+ alg = unrolled_loop;
+
if (!count)
- count_exp = copy_to_mode_reg (counter_mode (count_exp), count_exp);
- destreg = copy_addr_to_reg (XEXP (dst, 0));
+ count_exp = copy_to_mode_reg (GET_MODE (count_exp), count_exp);
+ destreg = ix86_copy_addr_to_reg (XEXP (dst, 0));
+ if (!issetmem)
+ srcreg = ix86_copy_addr_to_reg (XEXP (src, 0));
- move_mode = word_mode;
unroll_factor = 1;
+ move_mode = word_mode;
switch (alg)
{
case libcall:
case no_stringop:
case last_alg:
gcc_unreachable ();
+ case loop_1_byte:
+ need_zero_guard = true;
+ move_mode = QImode;
+ break;
case loop:
need_zero_guard = true;
break;
- case vector_loop:
case unrolled_loop:
need_zero_guard = true;
+ unroll_factor = (TARGET_64BIT ? 4 : 2);
+ break;
+ case vector_loop:
+ need_zero_guard = true;
unroll_factor = 4;
+ /* Find the widest supported mode. */
+ move_mode = word_mode;
+ while (optab_handler (mov_optab, GET_MODE_WIDER_MODE (move_mode))
+ != CODE_FOR_nothing)
+ move_mode = GET_MODE_WIDER_MODE (move_mode);
+
+ /* Find the corresponding vector mode with the same size as MOVE_MODE.
+ MOVE_MODE is an integer mode at the moment (SI, DI, TI, etc.). */
+ if (GET_MODE_SIZE (move_mode) > GET_MODE_SIZE (word_mode))
+ {
+ int nunits = GET_MODE_SIZE (move_mode) / GET_MODE_SIZE (word_mode);
+ move_mode = mode_for_vector (word_mode, nunits);
+ if (optab_handler (mov_optab, move_mode) == CODE_FOR_nothing)
+ move_mode = word_mode;
+ }
+ gcc_assert (optab_handler (mov_optab, move_mode) != CODE_FOR_nothing);
break;
case rep_prefix_8_byte:
move_mode = DImode;
@@ -23421,10 +23733,6 @@ ix86_expand_setmem (rtx dst, rtx count_exp, rtx val_exp, rtx align_exp,
case rep_prefix_1_byte:
move_mode = QImode;
break;
- case loop_1_byte:
- need_zero_guard = true;
- move_mode = QImode;
- break;
}
size_needed = GET_MODE_SIZE (move_mode) * unroll_factor;
epilogue_size_needed = size_needed;
@@ -23449,24 +23757,82 @@ ix86_expand_setmem (rtx dst, rtx count_exp, rtx val_exp, rtx align_exp,
align_bytes = desired_align - align_bytes;
}
if (align_bytes == 0)
- {
- enum machine_mode mode = SImode;
- if (TARGET_64BIT && (count & ~0xffffffff))
- mode = DImode;
- count_exp = force_reg (mode, count_exp);
- }
+ count_exp = force_reg (counter_mode (count_exp), count_exp);
}
+ gcc_assert (desired_align >= 1 && align >= 1);
+
+ /* Misaligned move sequences handle both prologue and epilogue at once.
+ Default code generation results in a smaller code for large alignments
+ and also avoids redundant job when sizes are known precisely. */
+ misaligned_prologue_used
+ = (TARGET_MISALIGNED_MOVE_STRING_PRO_EPILOGUES
+ && MAX (desired_align, epilogue_size_needed) <= 32
+ && desired_align <= epilogue_size_needed
+ && ((desired_align > align && !align_bytes)
+ || (!count && epilogue_size_needed > 1)));
+
/* Do the cheap promotion to allow better CSE across the
main loop and epilogue (ie one load of the big constant in the
- front of all code. */
- if (CONST_INT_P (val_exp))
- promoted_val = promote_duplicated_reg_to_size (val_exp, size_needed,
- desired_align, align);
+ front of all code.
+ For now the misaligned move sequences do not have fast path
+ without broadcasting. */
+ if (issetmem && ((CONST_INT_P (val_exp) || misaligned_prologue_used)))
+ {
+ if (alg == vector_loop)
+ {
+ gcc_assert (val_exp == const0_rtx);
+ vec_promoted_val = promote_duplicated_reg (move_mode, val_exp);
+ promoted_val = promote_duplicated_reg_to_size (val_exp,
+ GET_MODE_SIZE (word_mode),
+ desired_align, align);
+ }
+ else
+ {
+ promoted_val = promote_duplicated_reg_to_size (val_exp, size_needed,
+ desired_align, align);
+ }
+ }
+ /* Misaligned move sequences handles both prologues and epilogues at once.
+ Default code generation results in smaller code for large alignments and
+ also avoids redundant job when sizes are known precisely. */
+ if (misaligned_prologue_used)
+ {
+ /* Misaligned move prologue handled small blocks by itself. */
+ expand_set_or_movmem_prologue_epilogue_by_misaligned_moves
+ (dst, src, &destreg, &srcreg,
+ move_mode, promoted_val, vec_promoted_val,
+ &count_exp,
+ &jump_around_label,
+ desired_align < align
+ ? MAX (desired_align, epilogue_size_needed) : epilogue_size_needed,
+ desired_align, align, &min_size, dynamic_check, issetmem);
+ if (!issetmem)
+ src = change_address (src, BLKmode, srcreg);
+ dst = change_address (dst, BLKmode, destreg);
+ set_mem_align (dst, desired_align * BITS_PER_UNIT);
+ epilogue_size_needed = 0;
+ if (need_zero_guard && !min_size)
+ {
+ /* It is possible that we copied enough so the main loop will not
+ execute. */
+ gcc_assert (size_needed > 1);
+ if (jump_around_label == NULL_RTX)
+ jump_around_label = gen_label_rtx ();
+ emit_cmp_and_jump_insns (count_exp,
+ GEN_INT (size_needed),
+ LTU, 0, counter_mode (count_exp), 1, jump_around_label);
+ if (expected_size == -1
+ || expected_size < (desired_align - align) / 2 + size_needed)
+ predict_jump (REG_BR_PROB_BASE * 20 / 100);
+ else
+ predict_jump (REG_BR_PROB_BASE * 60 / 100);
+ }
+ }
/* Ensure that alignment prologue won't copy past end of block. */
- if (size_needed > 1 || (desired_align > 1 && desired_align > align))
+ else if (size_needed > 1 || (desired_align > 1 && desired_align > align))
{
epilogue_size_needed = MAX (size_needed - 1, desired_align - align);
- /* Epilogue always copies COUNT_EXP & (EPILOGUE_SIZE_NEEDED - 1) bytes.
+ /* Epilogue always copies COUNT_EXP & EPILOGUE_SIZE_NEEDED bytes.
Make sure it is power of 2. */
epilogue_size_needed = 1 << (floor_log2 (epilogue_size_needed) + 1);
@@ -23474,8 +23840,8 @@ ix86_expand_setmem (rtx dst, rtx count_exp, rtx val_exp, rtx align_exp,
promoting mode. This mean that if the promoted VAL is not constant,
we might not use it in the epilogue and have to use byte
loop variant. */
- if (epilogue_size_needed > 2 && !promoted_val)
- force_loopy_epilogue = true;
+ if (issetmem && epilogue_size_needed > 2 && !promoted_val)
+ force_loopy_epilogue = true;
if (count)
{
if (count < (unsigned HOST_WIDE_INT)epilogue_size_needed)
@@ -23488,61 +23854,96 @@ ix86_expand_setmem (rtx dst, rtx count_exp, rtx val_exp, rtx align_exp,
goto epilogue;
}
}
- else
+ else if (min_size < (unsigned HOST_WIDE_INT)epilogue_size_needed)
{
+ gcc_assert (max_size >= (unsigned HOST_WIDE_INT)epilogue_size_needed);
label = gen_label_rtx ();
emit_cmp_and_jump_insns (count_exp,
GEN_INT (epilogue_size_needed),
LTU, 0, counter_mode (count_exp), 1, label);
- if (expected_size == -1 || expected_size <= epilogue_size_needed)
+ if (expected_size == -1 || expected_size < epilogue_size_needed)
predict_jump (REG_BR_PROB_BASE * 60 / 100);
else
predict_jump (REG_BR_PROB_BASE * 20 / 100);
}
}
+
+ /* Emit code to decide on runtime whether library call or inline should be
+ used. */
if (dynamic_check != -1)
{
- rtx hot_label = gen_label_rtx ();
- jump_around_label = gen_label_rtx ();
- emit_cmp_and_jump_insns (count_exp, GEN_INT (dynamic_check - 1),
- LEU, 0, counter_mode (count_exp), 1, hot_label);
- predict_jump (REG_BR_PROB_BASE * 90 / 100);
- set_storage_via_libcall (dst, count_exp, val_exp, false);
- emit_jump (jump_around_label);
- emit_label (hot_label);
+ if (!issetmem && CONST_INT_P (count_exp))
+ {
+ if (UINTVAL (count_exp) >= (unsigned HOST_WIDE_INT)dynamic_check)
+ {
+ emit_block_move_via_libcall (dst, src, count_exp, false);
+ count_exp = const0_rtx;
+ goto epilogue;
+ }
+ }
+ else
+ {
+ rtx hot_label = gen_label_rtx ();
+ jump_around_label = gen_label_rtx ();
+ emit_cmp_and_jump_insns (count_exp, GEN_INT (dynamic_check - 1),
+ LEU, 0, GET_MODE (count_exp), 1, hot_label);
+ predict_jump (REG_BR_PROB_BASE * 90 / 100);
+ if (issetmem)
+ set_storage_via_libcall (dst, count_exp, val_exp, false);
+ else
+ emit_block_move_via_libcall (dst, src, count_exp, false);
+ emit_jump (jump_around_label);
+ emit_label (hot_label);
+ }
}
/* Step 2: Alignment prologue. */
-
/* Do the expensive promotion once we branched off the small blocks. */
- if (!promoted_val)
+ if (issetmem && !promoted_val)
promoted_val = promote_duplicated_reg_to_size (val_exp, size_needed,
desired_align, align);
- gcc_assert (desired_align >= 1 && align >= 1);
- if (desired_align > align)
+ if (desired_align > align && !misaligned_prologue_used)
{
if (align_bytes == 0)
{
- /* Except for the first move in epilogue, we no longer know
+ /* Except for the first move in prologue, we no longer know
constant offset in aliasing info. It don't seems to worth
the pain to maintain it for the first move, so throw away
the info early. */
dst = change_address (dst, BLKmode, destreg);
- expand_setmem_prologue (dst, destreg, promoted_val, count_exp, align,
- desired_align);
+ if (!issetmem)
+ src = change_address (src, BLKmode, srcreg);
+ dst = expand_set_or_movmem_prologue (dst, src, destreg, srcreg,
+ promoted_val, vec_promoted_val,
+ count_exp, align, desired_align,
+ issetmem);
+ /* At most desired_align - align bytes are copied. */
+ if (min_size < (unsigned)(desired_align - align))
+ min_size = 0;
+ else
+ min_size -= desired_align - align;
}
else
{
/* If we know how many bytes need to be stored before dst is
sufficiently aligned, maintain aliasing info accurately. */
- dst = expand_constant_setmem_prologue (dst, destreg, promoted_val,
- desired_align, align_bytes);
+ dst = expand_set_or_movmem_constant_prologue (dst, &src, destreg,
+ srcreg,
+ promoted_val,
+ vec_promoted_val,
+ desired_align,
+ align_bytes,
+ issetmem);
+
count_exp = plus_constant (counter_mode (count_exp),
count_exp, -align_bytes);
count -= align_bytes;
+ min_size -= align_bytes;
+ max_size -= align_bytes;
}
if (need_zero_guard
+ && !min_size
&& (count < (unsigned HOST_WIDE_INT) size_needed
|| (align_bytes == 0
&& count < ((unsigned HOST_WIDE_INT) size_needed
@@ -23568,10 +23969,11 @@ ix86_expand_setmem (rtx dst, rtx count_exp, rtx val_exp, rtx align_exp,
emit_label (label);
LABEL_NUSES (label) = 1;
label = NULL;
- promoted_val = val_exp;
epilogue_size_needed = 1;
+ if (issetmem)
+ promoted_val = val_exp;
}
- else if (label == NULL_RTX)
+ else if (label == NULL_RTX && !misaligned_prologue_used)
epilogue_size_needed = size_needed;
/* Step 3: Main loop. */
@@ -23584,38 +23986,45 @@ ix86_expand_setmem (rtx dst, rtx count_exp, rtx val_exp, rtx align_exp,
gcc_unreachable ();
case loop_1_byte:
case loop:
- case vector_loop:
case unrolled_loop:
- expand_set_or_movmem_via_loop (dst, NULL, destreg, NULL, promoted_val,
+ expand_set_or_movmem_via_loop (dst, src, destreg, srcreg, promoted_val,
count_exp, move_mode, unroll_factor,
- expected_size);
+ expected_size, issetmem);
break;
- case rep_prefix_8_byte:
- expand_setmem_via_rep_stos (dst, destreg, promoted_val, count_exp,
- DImode, val_exp);
+ case vector_loop:
+ expand_set_or_movmem_via_loop (dst, src, destreg, srcreg,
+ vec_promoted_val, count_exp, move_mode,
+ unroll_factor, expected_size, issetmem);
break;
+ case rep_prefix_8_byte:
case rep_prefix_4_byte:
- expand_setmem_via_rep_stos (dst, destreg, promoted_val, count_exp,
- SImode, val_exp);
- break;
case rep_prefix_1_byte:
- expand_setmem_via_rep_stos (dst, destreg, promoted_val, count_exp,
- QImode, val_exp);
+ expand_set_or_movmem_via_rep (dst, src, destreg, srcreg, promoted_val,
+ val_exp, count_exp, move_mode, issetmem);
break;
}
/* Adjust properly the offset of src and dest memory for aliasing. */
if (CONST_INT_P (count_exp))
- dst = adjust_automodify_address_nv (dst, BLKmode, destreg,
- (count / size_needed) * size_needed);
+ {
+ if (!issetmem)
+ src = adjust_automodify_address_nv (src, BLKmode, srcreg,
+ (count / size_needed) * size_needed);
+ dst = adjust_automodify_address_nv (dst, BLKmode, destreg,
+ (count / size_needed) * size_needed);
+ }
else
- dst = change_address (dst, BLKmode, destreg);
+ {
+ if (!issetmem)
+ src = change_address (src, BLKmode, srcreg);
+ dst = change_address (dst, BLKmode, destreg);
+ }
/* Step 4: Epilogue to copy the remaining bytes. */
-
+ epilogue:
if (label)
{
/* When the main loop is done, COUNT_EXP might hold original count,
- while we want to copy only COUNT_EXP & SIZE_NEEDED bytes.
+ while we want to copy only COUNT_EXP & SIZE_NEEDED bytes.
Epilogue code will actually copy COUNT_EXP & EPILOGUE_SIZE_NEEDED
bytes. Compensate if needed. */
@@ -23631,21 +24040,47 @@ ix86_expand_setmem (rtx dst, rtx count_exp, rtx val_exp, rtx align_exp,
emit_label (label);
LABEL_NUSES (label) = 1;
}
- epilogue:
+
if (count_exp != const0_rtx && epilogue_size_needed > 1)
{
if (force_loopy_epilogue)
expand_setmem_epilogue_via_loop (dst, destreg, val_exp, count_exp,
epilogue_size_needed);
else
- expand_setmem_epilogue (dst, destreg, promoted_val, count_exp,
- epilogue_size_needed);
+ {
+ if (issetmem)
+ expand_setmem_epilogue (dst, destreg, promoted_val,
+ vec_promoted_val, count_exp,
+ epilogue_size_needed);
+ else
+ expand_movmem_epilogue (dst, src, destreg, srcreg, count_exp,
+ epilogue_size_needed);
+ }
}
if (jump_around_label)
emit_label (jump_around_label);
return true;
}
+/* Wrapper for ix86_expand_set_or_movmem for memcpy case. */
+bool
+ix86_expand_movmem (rtx dst, rtx src, rtx count_exp, rtx align_exp,
+ rtx expected_align_exp, rtx expected_size_exp)
+{
+ return ix86_expand_set_or_movmem (dst, src, count_exp, NULL, align_exp,
+ expected_align_exp, expected_size_exp, false);
+}
+
+/* Wrapper for ix86_expand_set_or_movmem for memset case. */
+bool
+ix86_expand_setmem (rtx dst, rtx count_exp, rtx val_exp, rtx align_exp,
+ rtx expected_align_exp, rtx expected_size_exp)
+{
+ return ix86_expand_set_or_movmem (dst, NULL, count_exp, val_exp, align_exp,
+ expected_align_exp, expected_size_exp, true);
+}
+
+
/* Expand the appropriate insns for doing strlen if not just doing
repnz; scasb
@@ -24010,13 +24445,13 @@ ix86_output_call_insn (rtx insn, rtx call_op)
if (SIBLING_CALL_P (insn))
{
if (direct_p)
- xasm = "jmp\t%P0";
+ xasm = "%!jmp\t%P0";
/* SEH epilogue detection requires the indirect branch case
to include REX.W. */
else if (TARGET_SEH)
- xasm = "rex.W jmp %A0";
+ xasm = "%!rex.W jmp %A0";
else
- xasm = "jmp\t%A0";
+ xasm = "%!jmp\t%A0";
output_asm_insn (xasm, &call_op);
return "";
@@ -24053,9 +24488,9 @@ ix86_output_call_insn (rtx insn, rtx call_op)
}
if (direct_p)
- xasm = "call\t%P0";
+ xasm = "%!call\t%P0";
else
- xasm = "call\t%A0";
+ xasm = "%!call\t%A0";
output_asm_insn (xasm, &call_op);
@@ -24118,6 +24553,42 @@ ix86_instantiate_decls (void)
instantiate_decl_rtl (s->rtl);
}
+/* Check whether x86 address PARTS is a pc-relative address. */
+
+static bool
+rip_relative_addr_p (struct ix86_address *parts)
+{
+ rtx base, index, disp;
+
+ base = parts->base;
+ index = parts->index;
+ disp = parts->disp;
+
+ if (disp && !base && !index)
+ {
+ if (TARGET_64BIT)
+ {
+ rtx symbol = disp;
+
+ if (GET_CODE (disp) == CONST)
+ symbol = XEXP (disp, 0);
+ if (GET_CODE (symbol) == PLUS
+ && CONST_INT_P (XEXP (symbol, 1)))
+ symbol = XEXP (symbol, 0);
+
+ if (GET_CODE (symbol) == LABEL_REF
+ || (GET_CODE (symbol) == SYMBOL_REF
+ && SYMBOL_REF_TLS_MODEL (symbol) == 0)
+ || (GET_CODE (symbol) == UNSPEC
+ && (XINT (symbol, 1) == UNSPEC_GOTPCREL
+ || XINT (symbol, 1) == UNSPEC_PCREL
+ || XINT (symbol, 1) == UNSPEC_GOTNTPOFF)))
+ return true;
+ }
+ }
+ return false;
+}
+
/* Calculate the length of the memory address in the instruction encoding.
Includes addr32 prefix, does not include the one-byte modrm, opcode,
or other prefixes. We never generate addr32 prefix for LEA insn. */
@@ -24189,25 +24660,8 @@ memory_address_length (rtx addr, bool lea)
else if (disp && !base && !index)
{
len += 4;
- if (TARGET_64BIT)
- {
- rtx symbol = disp;
-
- if (GET_CODE (disp) == CONST)
- symbol = XEXP (disp, 0);
- if (GET_CODE (symbol) == PLUS
- && CONST_INT_P (XEXP (symbol, 1)))
- symbol = XEXP (symbol, 0);
-
- if (GET_CODE (symbol) != LABEL_REF
- && (GET_CODE (symbol) != SYMBOL_REF
- || SYMBOL_REF_TLS_MODEL (symbol) != 0)
- && (GET_CODE (symbol) != UNSPEC
- || (XINT (symbol, 1) != UNSPEC_GOTPCREL
- && XINT (symbol, 1) != UNSPEC_PCREL
- && XINT (symbol, 1) != UNSPEC_GOTNTPOFF)))
- len++;
- }
+ if (rip_relative_addr_p (&parts))
+ len++;
}
else
{
@@ -24389,23 +24843,26 @@ ix86_issue_rate (void)
case PROCESSOR_SLM:
case PROCESSOR_K6:
case PROCESSOR_BTVER2:
+ case PROCESSOR_PENTIUM4:
+ case PROCESSOR_NOCONA:
return 2;
case PROCESSOR_PENTIUMPRO:
- case PROCESSOR_PENTIUM4:
- case PROCESSOR_CORE2:
- case PROCESSOR_COREI7:
- case PROCESSOR_HASWELL:
case PROCESSOR_ATHLON:
case PROCESSOR_K8:
case PROCESSOR_AMDFAM10:
- case PROCESSOR_NOCONA:
case PROCESSOR_GENERIC:
+ case PROCESSOR_BTVER1:
+ return 3;
+
case PROCESSOR_BDVER1:
case PROCESSOR_BDVER2:
case PROCESSOR_BDVER3:
- case PROCESSOR_BTVER1:
- return 3;
+ case PROCESSOR_CORE2:
+ case PROCESSOR_COREI7:
+ case PROCESSOR_COREI7_AVX:
+ case PROCESSOR_HASWELL:
+ return 4;
default:
return 1;
@@ -24663,10 +25120,15 @@ ix86_adjust_cost (rtx insn, rtx link, rtx dep_insn, int cost)
case PROCESSOR_BDVER3:
case PROCESSOR_BTVER1:
case PROCESSOR_BTVER2:
- case PROCESSOR_ATOM:
case PROCESSOR_GENERIC:
memory = get_attr_memory (insn);
+ /* Stack engine allows to execute push&pop instructions in parall. */
+ if (((insn_type == TYPE_PUSH || insn_type == TYPE_POP)
+ && (dep_insn_type == TYPE_PUSH || dep_insn_type == TYPE_POP))
+ && (ix86_tune != PROCESSOR_ATHLON && ix86_tune != PROCESSOR_K8))
+ return 0;
+
/* Show ability of reorder buffer to hide latency of load by executing
in parallel with previous instruction in case
previous instruction is not needed to compute the address. */
@@ -24693,6 +25155,30 @@ ix86_adjust_cost (rtx insn, rtx link, rtx dep_insn, int cost)
}
break;
+ case PROCESSOR_CORE2:
+ case PROCESSOR_COREI7:
+ case PROCESSOR_COREI7_AVX:
+ case PROCESSOR_HASWELL:
+ memory = get_attr_memory (insn);
+
+ /* Stack engine allows to execute push&pop instructions in parall. */
+ if ((insn_type == TYPE_PUSH || insn_type == TYPE_POP)
+ && (dep_insn_type == TYPE_PUSH || dep_insn_type == TYPE_POP))
+ return 0;
+
+ /* Show ability of reorder buffer to hide latency of load by executing
+ in parallel with previous instruction in case
+ previous instruction is not needed to compute the address. */
+ if ((memory == MEMORY_LOAD || memory == MEMORY_BOTH)
+ && !ix86_agi_dependent (dep_insn, insn))
+ {
+ if (cost >= 4)
+ cost -= 4;
+ else
+ cost = 0;
+ }
+ break;
+
case PROCESSOR_SLM:
if (!reload_completed)
return cost;
@@ -24748,8 +25234,16 @@ ia32_multipass_dfa_lookahead (void)
case PROCESSOR_K6:
return 1;
+ case PROCESSOR_BDVER1:
+ case PROCESSOR_BDVER2:
+ case PROCESSOR_BDVER3:
+ /* We use lookahead value 4 for BD both before and after reload
+ schedules. Plan is to have value 8 included for O3. */
+ return 4;
+
case PROCESSOR_CORE2:
case PROCESSOR_COREI7:
+ case PROCESSOR_COREI7_AVX:
case PROCESSOR_HASWELL:
case PROCESSOR_ATOM:
case PROCESSOR_SLM:
@@ -24766,6 +25260,119 @@ ia32_multipass_dfa_lookahead (void)
}
}
+/* Return true if target platform supports macro-fusion. */
+
+static bool
+ix86_macro_fusion_p ()
+{
+ return TARGET_FUSE_CMP_AND_BRANCH;
+}
+
+/* Check whether current microarchitecture support macro fusion
+ for insn pair "CONDGEN + CONDJMP". Refer to
+ "Intel Architectures Optimization Reference Manual". */
+
+static bool
+ix86_macro_fusion_pair_p (rtx condgen, rtx condjmp)
+{
+ rtx src, dest;
+ rtx single_set = single_set (condgen);
+ enum rtx_code ccode;
+ rtx compare_set = NULL_RTX, test_if, cond;
+ rtx alu_set = NULL_RTX, addr = NULL_RTX;
+
+ if (get_attr_type (condgen) != TYPE_TEST
+ && get_attr_type (condgen) != TYPE_ICMP
+ && get_attr_type (condgen) != TYPE_INCDEC
+ && get_attr_type (condgen) != TYPE_ALU)
+ return false;
+
+ if (single_set == NULL_RTX
+ && !TARGET_FUSE_ALU_AND_BRANCH)
+ return false;
+
+ if (single_set != NULL_RTX)
+ compare_set = single_set;
+ else
+ {
+ int i;
+ rtx pat = PATTERN (condgen);
+ for (i = 0; i < XVECLEN (pat, 0); i++)
+ if (GET_CODE (XVECEXP (pat, 0, i)) == SET)
+ {
+ rtx set_src = SET_SRC (XVECEXP (pat, 0, i));
+ if (GET_CODE (set_src) == COMPARE)
+ compare_set = XVECEXP (pat, 0, i);
+ else
+ alu_set = XVECEXP (pat, 0, i);
+ }
+ }
+ if (compare_set == NULL_RTX)
+ return false;
+ src = SET_SRC (compare_set);
+ if (GET_CODE (src) != COMPARE)
+ return false;
+
+ /* Macro-fusion for cmp/test MEM-IMM + conditional jmp is not
+ supported. */
+ if ((MEM_P (XEXP (src, 0))
+ && CONST_INT_P (XEXP (src, 1)))
+ || (MEM_P (XEXP (src, 1))
+ && CONST_INT_P (XEXP (src, 0))))
+ return false;
+
+ /* No fusion for RIP-relative address. */
+ if (MEM_P (XEXP (src, 0)))
+ addr = XEXP (XEXP (src, 0), 0);
+ else if (MEM_P (XEXP (src, 1)))
+ addr = XEXP (XEXP (src, 1), 0);
+
+ if (addr) {
+ ix86_address parts;
+ int ok = ix86_decompose_address (addr, &parts);
+ gcc_assert (ok);
+
+ if (rip_relative_addr_p (&parts))
+ return false;
+ }
+
+ test_if = SET_SRC (pc_set (condjmp));
+ cond = XEXP (test_if, 0);
+ ccode = GET_CODE (cond);
+ /* Check whether conditional jump use Sign or Overflow Flags. */
+ if (!TARGET_FUSE_CMP_AND_BRANCH_SOFLAGS
+ && (ccode == GE
+ || ccode == GT
+ || ccode == LE
+ || ccode == LT))
+ return false;
+
+ /* Return true for TYPE_TEST and TYPE_ICMP. */
+ if (get_attr_type (condgen) == TYPE_TEST
+ || get_attr_type (condgen) == TYPE_ICMP)
+ return true;
+
+ /* The following is the case that macro-fusion for alu + jmp. */
+ if (!TARGET_FUSE_ALU_AND_BRANCH || !alu_set)
+ return false;
+
+ /* No fusion for alu op with memory destination operand. */
+ dest = SET_DEST (alu_set);
+ if (MEM_P (dest))
+ return false;
+
+ /* Macro-fusion for inc/dec + unsigned conditional jump is not
+ supported. */
+ if (get_attr_type (condgen) == TYPE_INCDEC
+ && (ccode == GEU
+ || ccode == GTU
+ || ccode == LEU
+ || ccode == LTU))
+ return false;
+
+ return true;
+}
+
/* Try to reorder ready list to take advantage of Atom pipelined IMUL
execution. It is applied if
(1) IMUL instruction is on the top of list;
@@ -25390,6 +25997,7 @@ ix86_sched_init_global (FILE *dump ATTRIBUTE_UNUSED,
{
case PROCESSOR_CORE2:
case PROCESSOR_COREI7:
+ case PROCESSOR_COREI7_AVX:
case PROCESSOR_HASWELL:
/* Do not perform multipass scheduling for pre-reload schedule
to save compile time. */
@@ -26387,6 +26995,11 @@ enum ix86_builtins
IX86_BUILTIN_LFENCE,
IX86_BUILTIN_PAUSE,
+ IX86_BUILTIN_FNSTENV,
+ IX86_BUILTIN_FLDENV,
+ IX86_BUILTIN_FNSTSW,
+ IX86_BUILTIN_FNCLEX,
+
IX86_BUILTIN_BSRSI,
IX86_BUILTIN_BSRDI,
IX86_BUILTIN_RDPMC,
@@ -27363,6 +27976,12 @@ static const struct builtin_description bdesc_special_args[] =
{ ~OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_rdtscp", IX86_BUILTIN_RDTSCP, UNKNOWN, (int) UINT64_FTYPE_PUNSIGNED },
{ ~OPTION_MASK_ISA_64BIT, CODE_FOR_pause, "__builtin_ia32_pause", IX86_BUILTIN_PAUSE, UNKNOWN, (int) VOID_FTYPE_VOID },
+ /* 80387 (for use internally for atomic compound assignment). */
+ { 0, CODE_FOR_fnstenv, "__builtin_ia32_fnstenv", IX86_BUILTIN_FNSTENV, UNKNOWN, (int) VOID_FTYPE_PVOID },
+ { 0, CODE_FOR_fldenv, "__builtin_ia32_fldenv", IX86_BUILTIN_FLDENV, UNKNOWN, (int) VOID_FTYPE_PCVOID },
+ { 0, CODE_FOR_fnstsw, "__builtin_ia32_fnstsw", IX86_BUILTIN_FNSTSW, UNKNOWN, (int) VOID_FTYPE_PUSHORT },
+ { 0, CODE_FOR_fnclex, "__builtin_ia32_fnclex", IX86_BUILTIN_FNCLEX, UNKNOWN, (int) VOID_FTYPE_VOID },
+
/* MMX */
{ OPTION_MASK_ISA_MMX, CODE_FOR_mmx_emms, "__builtin_ia32_emms", IX86_BUILTIN_EMMS, UNKNOWN, (int) VOID_FTYPE_VOID },
@@ -27400,13 +28019,13 @@ static const struct builtin_description bdesc_special_args[] =
{ OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_lfence, "__builtin_ia32_lfence", IX86_BUILTIN_LFENCE, UNKNOWN, (int) VOID_FTYPE_VOID },
{ OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_mfence, 0, IX86_BUILTIN_MFENCE, UNKNOWN, (int) VOID_FTYPE_VOID },
{ OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_storeupd, "__builtin_ia32_storeupd", IX86_BUILTIN_STOREUPD, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V2DF },
- { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_storedqu, "__builtin_ia32_storedqu", IX86_BUILTIN_STOREDQU, UNKNOWN, (int) VOID_FTYPE_PCHAR_V16QI },
+ { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_storedquv16qi, "__builtin_ia32_storedqu", IX86_BUILTIN_STOREDQU, UNKNOWN, (int) VOID_FTYPE_PCHAR_V16QI },
{ OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movntv2df, "__builtin_ia32_movntpd", IX86_BUILTIN_MOVNTPD, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V2DF },
{ OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movntv2di, "__builtin_ia32_movntdq", IX86_BUILTIN_MOVNTDQ, UNKNOWN, (int) VOID_FTYPE_PV2DI_V2DI },
{ OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movntisi, "__builtin_ia32_movnti", IX86_BUILTIN_MOVNTI, UNKNOWN, (int) VOID_FTYPE_PINT_INT },
{ OPTION_MASK_ISA_SSE2 | OPTION_MASK_ISA_64BIT, CODE_FOR_sse2_movntidi, "__builtin_ia32_movnti64", IX86_BUILTIN_MOVNTI64, UNKNOWN, (int) VOID_FTYPE_PLONGLONG_LONGLONG },
{ OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_loadupd, "__builtin_ia32_loadupd", IX86_BUILTIN_LOADUPD, UNKNOWN, (int) V2DF_FTYPE_PCDOUBLE },
- { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_loaddqu, "__builtin_ia32_loaddqu", IX86_BUILTIN_LOADDQU, UNKNOWN, (int) V16QI_FTYPE_PCCHAR },
+ { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_loaddquv16qi, "__builtin_ia32_loaddqu", IX86_BUILTIN_LOADDQU, UNKNOWN, (int) V16QI_FTYPE_PCCHAR },
{ OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_loadhpd_exp, "__builtin_ia32_loadhpd", IX86_BUILTIN_LOADHPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_PCDOUBLE },
{ OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_loadlpd_exp, "__builtin_ia32_loadlpd", IX86_BUILTIN_LOADLPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_PCDOUBLE },
@@ -27435,8 +28054,8 @@ static const struct builtin_description bdesc_special_args[] =
{ OPTION_MASK_ISA_AVX, CODE_FOR_avx_loadups256, "__builtin_ia32_loadups256", IX86_BUILTIN_LOADUPS256, UNKNOWN, (int) V8SF_FTYPE_PCFLOAT },
{ OPTION_MASK_ISA_AVX, CODE_FOR_avx_storeupd256, "__builtin_ia32_storeupd256", IX86_BUILTIN_STOREUPD256, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V4DF },
{ OPTION_MASK_ISA_AVX, CODE_FOR_avx_storeups256, "__builtin_ia32_storeups256", IX86_BUILTIN_STOREUPS256, UNKNOWN, (int) VOID_FTYPE_PFLOAT_V8SF },
- { OPTION_MASK_ISA_AVX, CODE_FOR_avx_loaddqu256, "__builtin_ia32_loaddqu256", IX86_BUILTIN_LOADDQU256, UNKNOWN, (int) V32QI_FTYPE_PCCHAR },
- { OPTION_MASK_ISA_AVX, CODE_FOR_avx_storedqu256, "__builtin_ia32_storedqu256", IX86_BUILTIN_STOREDQU256, UNKNOWN, (int) VOID_FTYPE_PCHAR_V32QI },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_avx_loaddquv32qi, "__builtin_ia32_loaddqu256", IX86_BUILTIN_LOADDQU256, UNKNOWN, (int) V32QI_FTYPE_PCCHAR },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_avx_storedquv32qi, "__builtin_ia32_storedqu256", IX86_BUILTIN_STOREDQU256, UNKNOWN, (int) VOID_FTYPE_PCHAR_V32QI },
{ OPTION_MASK_ISA_AVX, CODE_FOR_avx_lddqu256, "__builtin_ia32_lddqu256", IX86_BUILTIN_LDDQU256, UNKNOWN, (int) V32QI_FTYPE_PCCHAR },
{ OPTION_MASK_ISA_AVX, CODE_FOR_avx_movntv4di, "__builtin_ia32_movntdq256", IX86_BUILTIN_MOVNTDQ256, UNKNOWN, (int) VOID_FTYPE_PV4DI_V4DI },
@@ -27705,7 +28324,7 @@ static const struct builtin_description bdesc_args[] =
{ OPTION_MASK_ISA_SSE2 | OPTION_MASK_ISA_64BIT, CODE_FOR_sse2_cvtsd2siq, "__builtin_ia32_cvtsd2si64", IX86_BUILTIN_CVTSD2SI64, UNKNOWN, (int) INT64_FTYPE_V2DF },
{ OPTION_MASK_ISA_SSE2 | OPTION_MASK_ISA_64BIT, CODE_FOR_sse2_cvttsd2siq, "__builtin_ia32_cvttsd2si64", IX86_BUILTIN_CVTTSD2SI64, UNKNOWN, (int) INT64_FTYPE_V2DF },
- { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtps2dq, "__builtin_ia32_cvtps2dq", IX86_BUILTIN_CVTPS2DQ, UNKNOWN, (int) V4SI_FTYPE_V4SF },
+ { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_fix_notruncv4sfv4si, "__builtin_ia32_cvtps2dq", IX86_BUILTIN_CVTPS2DQ, UNKNOWN, (int) V4SI_FTYPE_V4SF },
{ OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtps2pd, "__builtin_ia32_cvtps2pd", IX86_BUILTIN_CVTPS2PD, UNKNOWN, (int) V2DF_FTYPE_V4SF },
{ OPTION_MASK_ISA_SSE2, CODE_FOR_fix_truncv4sfv4si2, "__builtin_ia32_cvttps2dq", IX86_BUILTIN_CVTTPS2DQ, UNKNOWN, (int) V4SI_FTYPE_V4SF },
@@ -28053,7 +28672,7 @@ static const struct builtin_description bdesc_args[] =
{ OPTION_MASK_ISA_AVX, CODE_FOR_floatv4siv4df2, "__builtin_ia32_cvtdq2pd256", IX86_BUILTIN_CVTDQ2PD256, UNKNOWN, (int) V4DF_FTYPE_V4SI },
{ OPTION_MASK_ISA_AVX, CODE_FOR_floatv8siv8sf2, "__builtin_ia32_cvtdq2ps256", IX86_BUILTIN_CVTDQ2PS256, UNKNOWN, (int) V8SF_FTYPE_V8SI },
{ OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtpd2ps256, "__builtin_ia32_cvtpd2ps256", IX86_BUILTIN_CVTPD2PS256, UNKNOWN, (int) V4SF_FTYPE_V4DF },
- { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtps2dq256, "__builtin_ia32_cvtps2dq256", IX86_BUILTIN_CVTPS2DQ256, UNKNOWN, (int) V8SI_FTYPE_V8SF },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_avx_fix_notruncv8sfv8si, "__builtin_ia32_cvtps2dq256", IX86_BUILTIN_CVTPS2DQ256, UNKNOWN, (int) V8SI_FTYPE_V8SF },
{ OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtps2pd256, "__builtin_ia32_cvtps2pd256", IX86_BUILTIN_CVTPS2PD256, UNKNOWN, (int) V4DF_FTYPE_V4SF },
{ OPTION_MASK_ISA_AVX, CODE_FOR_fix_truncv4dfv4si2, "__builtin_ia32_cvttpd2dq256", IX86_BUILTIN_CVTTPD2DQ256, UNKNOWN, (int) V4SI_FTYPE_V4DF },
{ OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtpd2dq256, "__builtin_ia32_cvtpd2dq256", IX86_BUILTIN_CVTPD2DQ256, UNKNOWN, (int) V4SI_FTYPE_V4DF },
@@ -29222,7 +29841,8 @@ get_builtin_code_for_version (tree decl, tree *predicate_list)
if (strstr (attrs_str, "arch=") != NULL)
{
cl_target_option_save (&cur_target, &global_options);
- target_node = ix86_valid_target_attribute_tree (attrs);
+ target_node = ix86_valid_target_attribute_tree (attrs, &global_options,
+ &global_options_set);
gcc_assert (target_node);
new_target = TREE_TARGET_OPTION (target_node);
@@ -29240,6 +29860,10 @@ get_builtin_code_for_version (tree decl, tree *predicate_list)
arg_str = "corei7";
priority = P_PROC_SSE4_2;
break;
+ case PROCESSOR_COREI7_AVX:
+ arg_str = "corei7-avx";
+ priority = P_PROC_SSE4_2;
+ break;
case PROCESSOR_ATOM:
arg_str = "atom";
priority = P_PROC_SSSE3;
@@ -29812,7 +30436,7 @@ ix86_get_function_versions_dispatcher (void *decl)
while (default_version_info != NULL)
{
if (is_function_default_version
- (default_version_info->this_node->symbol.decl))
+ (default_version_info->this_node->decl))
break;
default_version_info = default_version_info->next;
}
@@ -29842,7 +30466,7 @@ ix86_get_function_versions_dispatcher (void *decl)
struct cgraph_function_version_info *dispatcher_version_info = NULL;
/* Right now, the dispatching is done via ifunc. */
- dispatch_decl = make_dispatcher_decl (default_node->symbol.decl);
+ dispatch_decl = make_dispatcher_decl (default_node->decl);
dispatcher_node = cgraph_get_create_node (dispatch_decl);
gcc_assert (dispatcher_node != NULL);
@@ -29850,7 +30474,7 @@ ix86_get_function_versions_dispatcher (void *decl)
dispatcher_version_info
= insert_new_cgraph_node_version (dispatcher_node);
dispatcher_version_info->next = default_version_info;
- dispatcher_node->symbol.definition = 1;
+ dispatcher_node->definition = 1;
/* Set the dispatcher for all the versions. */
it_v = default_version_info;
@@ -29863,7 +30487,7 @@ ix86_get_function_versions_dispatcher (void *decl)
else
#endif
{
- error_at (DECL_SOURCE_LOCATION (default_node->symbol.decl),
+ error_at (DECL_SOURCE_LOCATION (default_node->decl),
"multiversioning needs ifunc which is not supported "
"on this target");
}
@@ -30002,13 +30626,13 @@ ix86_generate_version_dispatcher_body (void *node_p)
return node_version_info->dispatcher_resolver;
/* The first version in the chain corresponds to the default version. */
- default_ver_decl = node_version_info->next->this_node->symbol.decl;
+ default_ver_decl = node_version_info->next->this_node->decl;
/* node is going to be an alias, so remove the finalized bit. */
- node->symbol.definition = false;
+ node->definition = false;
resolver_decl = make_resolver_func (default_ver_decl,
- node->symbol.decl, &empty_bb);
+ node->decl, &empty_bb);
node_version_info->dispatcher_resolver = resolver_decl;
@@ -30025,10 +30649,10 @@ ix86_generate_version_dispatcher_body (void *node_p)
not. This happens for methods in derived classes that override
virtual methods in base classes but are not explicitly marked as
virtual. */
- if (DECL_VINDEX (versn->symbol.decl))
+ if (DECL_VINDEX (versn->decl))
sorry ("Virtual function multiversioning not supported");
- fn_ver_vec.safe_push (versn->symbol.decl);
+ fn_ver_vec.safe_push (versn->decl);
}
dispatch_function_versions (resolver_decl, &fn_ver_vec, &empty_bb);
@@ -31575,8 +32199,8 @@ ix86_expand_args_builtin (const struct builtin_description *d,
}
else
{
- target = gen_reg_rtx (rmode);
- real_target = simplify_gen_subreg (tmode, target, rmode, 0);
+ real_target = gen_reg_rtx (tmode);
+ target = simplify_gen_subreg (rmode, real_target, tmode, 0);
}
for (i = 0; i < nargs; i++)
@@ -32319,6 +32943,10 @@ ix86_expand_builtin (tree exp, rtx target, rtx subtarget,
case IX86_BUILTIN_FXRSTOR:
case IX86_BUILTIN_FXSAVE64:
case IX86_BUILTIN_FXRSTOR64:
+ case IX86_BUILTIN_FNSTENV:
+ case IX86_BUILTIN_FLDENV:
+ case IX86_BUILTIN_FNSTSW:
+ mode0 = BLKmode;
switch (fcode)
{
case IX86_BUILTIN_FXSAVE:
@@ -32333,6 +32961,16 @@ ix86_expand_builtin (tree exp, rtx target, rtx subtarget,
case IX86_BUILTIN_FXRSTOR64:
icode = CODE_FOR_fxrstor64;
break;
+ case IX86_BUILTIN_FNSTENV:
+ icode = CODE_FOR_fnstenv;
+ break;
+ case IX86_BUILTIN_FLDENV:
+ icode = CODE_FOR_fldenv;
+ break;
+ case IX86_BUILTIN_FNSTSW:
+ icode = CODE_FOR_fnstsw;
+ mode0 = HImode;
+ break;
default:
gcc_unreachable ();
}
@@ -32345,7 +32983,7 @@ ix86_expand_builtin (tree exp, rtx target, rtx subtarget,
op0 = convert_memory_address (Pmode, op0);
op0 = copy_addr_to_reg (op0);
}
- op0 = gen_rtx_MEM (BLKmode, op0);
+ op0 = gen_rtx_MEM (mode0, op0);
pat = GEN_FCN (icode) (op0);
if (pat)
@@ -34015,6 +34653,7 @@ ix86_class_likely_spilled_p (reg_class_t rclass)
case SSE_FIRST_REG:
case FP_TOP_REG:
case FP_SECOND_REG:
+ case BND_REGS:
return true;
default:
@@ -34363,6 +35002,8 @@ ix86_hard_regno_mode_ok (int regno, enum machine_mode mode)
return VALID_FP_MODE_P (mode);
if (MASK_REGNO_P (regno))
return VALID_MASK_REG_MODE (mode);
+ if (BND_REGNO_P (regno))
+ return VALID_BND_REG_MODE (mode);
if (SSE_REGNO_P (regno))
{
/* We implement the move patterns for all vector modes into and
@@ -34739,7 +35380,7 @@ ix86_rtx_costs (rtx x, int code_i, int outer_code_i, int opno, int *total,
rtx sub;
gcc_assert (FLOAT_MODE_P (mode));
- gcc_assert (TARGET_FMA || TARGET_FMA4);
+ gcc_assert (TARGET_FMA || TARGET_FMA4 || TARGET_AVX512F);
/* ??? SSE scalar/vector cost should be used here. */
/* ??? Bald assumption that fma has the same cost as fmul. */
@@ -35176,6 +35817,10 @@ x86_order_regs_for_local_alloc (void)
for (i = FIRST_MASK_REG; i <= LAST_MASK_REG; i++)
reg_alloc_order [pos++] = i;
+ /* MPX bound registers. */
+ for (i = FIRST_BND_REG; i <= LAST_BND_REG; i++)
+ reg_alloc_order [pos++] = i;
+
/* x87 registers. */
if (TARGET_SSE_MATH)
for (i = FIRST_STACK_REG; i <= LAST_STACK_REG; i++)
@@ -36292,8 +36937,9 @@ ix86_expand_vector_init_duplicate (bool mmx_ok, enum machine_mode mode,
emit_move_insn (tmp1, gen_lowpart (SImode, val));
/* Insert the SImode value as low element of a V4SImode vector. */
- tmp2 = gen_lowpart (V4SImode, dperm.op0);
+ tmp2 = gen_reg_rtx (V4SImode);
emit_insn (gen_vec_setv4si_0 (tmp2, CONST0_RTX (V4SImode), tmp1));
+ emit_move_insn (dperm.op0, gen_lowpart (mode, tmp2));
ok = (expand_vec_perm_1 (&dperm)
|| expand_vec_perm_broadcast_1 (&dperm));
@@ -36323,9 +36969,10 @@ ix86_expand_vector_init_duplicate (bool mmx_ok, enum machine_mode mode,
NULL_RTX, 1, OPTAB_LIB_WIDEN);
val = expand_simple_binop (wsmode, IOR, val, x, x, 1, OPTAB_LIB_WIDEN);
- x = gen_lowpart (wvmode, target);
+ x = gen_reg_rtx (wvmode);
ok = ix86_expand_vector_init_duplicate (mmx_ok, wvmode, x, val);
gcc_assert (ok);
+ emit_move_insn (target, gen_lowpart (GET_MODE (target), x));
return ok;
}
@@ -37200,8 +37847,9 @@ ix86_expand_vector_set (bool mmx_ok, rtx target, rtx val, int elt)
else
{
/* For SSE1, we have to reuse the V4SF code. */
- ix86_expand_vector_set (false, gen_lowpart (V4SFmode, target),
- gen_lowpart (SFmode, val), elt);
+ rtx t = gen_reg_rtx (V4SFmode);
+ ix86_expand_vector_set (false, t, gen_lowpart (SFmode, val), elt);
+ emit_move_insn (target, gen_lowpart (mode, t));
}
return;
@@ -37519,7 +38167,7 @@ ix86_expand_vector_extract (bool mmx_ok, rtx target, rtx vec, int elt)
static void
emit_reduc_half (rtx dest, rtx src, int i)
{
- rtx tem;
+ rtx tem, d = dest;
switch (GET_MODE (src))
{
case V4SFmode:
@@ -37536,8 +38184,8 @@ emit_reduc_half (rtx dest, rtx src, int i)
case V8HImode:
case V4SImode:
case V2DImode:
- tem = gen_sse2_lshrv1ti3 (gen_lowpart (V1TImode, dest),
- gen_lowpart (V1TImode, src),
+ d = gen_reg_rtx (V1TImode);
+ tem = gen_sse2_lshrv1ti3 (d, gen_lowpart (V1TImode, src),
GEN_INT (i / 2));
break;
case V8SFmode:
@@ -37558,19 +38206,26 @@ emit_reduc_half (rtx dest, rtx src, int i)
case V8SImode:
case V4DImode:
if (i == 256)
- tem = gen_avx2_permv2ti (gen_lowpart (V4DImode, dest),
- gen_lowpart (V4DImode, src),
- gen_lowpart (V4DImode, src),
- const1_rtx);
+ {
+ if (GET_MODE (dest) != V4DImode)
+ d = gen_reg_rtx (V4DImode);
+ tem = gen_avx2_permv2ti (d, gen_lowpart (V4DImode, src),
+ gen_lowpart (V4DImode, src),
+ const1_rtx);
+ }
else
- tem = gen_avx2_lshrv2ti3 (gen_lowpart (V2TImode, dest),
- gen_lowpart (V2TImode, src),
- GEN_INT (i / 2));
+ {
+ d = gen_reg_rtx (V2TImode);
+ tem = gen_avx2_lshrv2ti3 (d, gen_lowpart (V2TImode, src),
+ GEN_INT (i / 2));
+ }
break;
default:
gcc_unreachable ();
}
emit_insn (tem);
+ if (d != dest)
+ emit_move_insn (dest, gen_lowpart (GET_MODE (dest), d));
}
/* Expand a vector reduction. FN is the binary pattern to reduce;
@@ -38217,6 +38872,7 @@ static rtx
ix86_expand_sse_compare_and_jump (enum rtx_code code, rtx op0, rtx op1,
bool swap_operands)
{
+ enum machine_mode fpcmp_mode = ix86_fp_compare_mode (code);
rtx label, tmp;
if (swap_operands)
@@ -38227,9 +38883,9 @@ ix86_expand_sse_compare_and_jump (enum rtx_code code, rtx op0, rtx op1,
}
label = gen_label_rtx ();
- tmp = gen_rtx_REG (CCFPUmode, FLAGS_REG);
+ tmp = gen_rtx_REG (fpcmp_mode, FLAGS_REG);
emit_insn (gen_rtx_SET (VOIDmode, tmp,
- gen_rtx_COMPARE (CCFPUmode, op0, op1)));
+ gen_rtx_COMPARE (fpcmp_mode, op0, op1)));
tmp = gen_rtx_fmt_ee (code, VOIDmode, tmp, const0_rtx);
tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, tmp,
gen_rtx_LABEL_REF (VOIDmode, label), pc_rtx);
@@ -39062,6 +39718,8 @@ expand_vec_perm_blend (struct expand_vec_perm_d *d)
emit_insn (gen_sse4_1_pblendvb (target, op0, op1, vperm));
else
emit_insn (gen_avx2_pblendvb (target, op0, op1, vperm));
+ if (target != d->target)
+ emit_move_insn (d->target, gen_lowpart (d->vmode, target));
return true;
}
@@ -39071,7 +39729,7 @@ expand_vec_perm_blend (struct expand_vec_perm_d *d)
/* FALLTHRU */
do_subreg:
- target = gen_lowpart (vmode, target);
+ target = gen_reg_rtx (vmode);
op0 = gen_lowpart (vmode, op0);
op1 = gen_lowpart (vmode, op1);
break;
@@ -39125,7 +39783,7 @@ expand_vec_perm_blend (struct expand_vec_perm_d *d)
vmode = V32QImode;
nelt = 32;
- target = gen_lowpart (vmode, target);
+ target = gen_reg_rtx (vmode);
op0 = gen_lowpart (vmode, op0);
op1 = gen_lowpart (vmode, op1);
goto finish_pblendvb;
@@ -39158,6 +39816,8 @@ expand_vec_perm_blend (struct expand_vec_perm_d *d)
x = gen_rtx_VEC_MERGE (vmode, op1, op0, GEN_INT (mask));
x = gen_rtx_SET (VOIDmode, target, x);
emit_insn (x);
+ if (target != d->target)
+ emit_move_insn (d->target, gen_lowpart (d->vmode, target));
return true;
}
@@ -39263,13 +39923,17 @@ expand_vec_perm_pshufb (struct expand_vec_perm_d *d)
/* Use vperm2i128 insn. The pattern uses
V4DImode instead of V2TImode. */
- target = gen_lowpart (V4DImode, d->target);
+ target = d->target;
+ if (d->vmode != V4DImode)
+ target = gen_reg_rtx (V4DImode);
op0 = gen_lowpart (V4DImode, d->op0);
op1 = gen_lowpart (V4DImode, d->op1);
rperm[0]
= GEN_INT (((d->perm[0] & (nelt / 2)) ? 1 : 0)
|| ((d->perm[nelt / 2] & (nelt / 2)) ? 2 : 0));
emit_insn (gen_avx2_permv2ti (target, op0, op1, rperm[0]));
+ if (target != d->target)
+ emit_move_insn (d->target, gen_lowpart (d->vmode, target));
return true;
}
return false;
@@ -39304,9 +39968,15 @@ expand_vec_perm_pshufb (struct expand_vec_perm_d *d)
perm[i] = (d->perm[i * nelt / 4] * 4 / nelt) & 3;
if (d->testing_p)
return true;
- return expand_vselect (gen_lowpart (V4DImode, d->target),
- gen_lowpart (V4DImode, d->op0),
- perm, 4, false);
+ target = gen_reg_rtx (V4DImode);
+ if (expand_vselect (target, gen_lowpart (V4DImode, d->op0),
+ perm, 4, false))
+ {
+ emit_move_insn (d->target,
+ gen_lowpart (d->vmode, target));
+ return true;
+ }
+ return false;
}
/* Next see if vpermd can be used. */
@@ -39358,7 +40028,9 @@ expand_vec_perm_pshufb (struct expand_vec_perm_d *d)
gen_rtvec_v (GET_MODE_NUNITS (vmode), rperm));
vperm = force_reg (vmode, vperm);
- target = gen_lowpart (vmode, d->target);
+ target = d->target;
+ if (d->vmode != vmode)
+ target = gen_reg_rtx (vmode);
op0 = gen_lowpart (vmode, d->op0);
if (d->one_operand_p)
{
@@ -39376,6 +40048,8 @@ expand_vec_perm_pshufb (struct expand_vec_perm_d *d)
op1 = gen_lowpart (vmode, d->op1);
emit_insn (gen_xop_pperm (target, op0, op1, vperm));
}
+ if (target != d->target)
+ emit_move_insn (d->target, gen_lowpart (d->vmode, target));
return true;
}
@@ -39575,7 +40249,8 @@ expand_vec_perm_palignr (struct expand_vec_perm_d *d)
unsigned i, nelt = d->nelt;
unsigned min, max;
bool in_order, ok;
- rtx shift;
+ rtx shift, target;
+ struct expand_vec_perm_d dcopy;
/* Even with AVX, palignr only operates on 128-bit vectors. */
if (!TARGET_SSSE3 || GET_MODE_SIZE (d->vmode) != 16)
@@ -39598,29 +40273,33 @@ expand_vec_perm_palignr (struct expand_vec_perm_d *d)
if (d->testing_p)
return true;
+ dcopy = *d;
shift = GEN_INT (min * GET_MODE_BITSIZE (GET_MODE_INNER (d->vmode)));
- emit_insn (gen_ssse3_palignrti (gen_lowpart (TImode, d->target),
- gen_lowpart (TImode, d->op1),
+ target = gen_reg_rtx (TImode);
+ emit_insn (gen_ssse3_palignrti (target, gen_lowpart (TImode, d->op1),
gen_lowpart (TImode, d->op0), shift));
- d->op0 = d->op1 = d->target;
- d->one_operand_p = true;
+ dcopy.op0 = dcopy.op1 = gen_lowpart (d->vmode, target);
+ dcopy.one_operand_p = true;
in_order = true;
for (i = 0; i < nelt; ++i)
{
- unsigned e = d->perm[i] - min;
+ unsigned e = dcopy.perm[i] - min;
if (e != i)
in_order = false;
- d->perm[i] = e;
+ dcopy.perm[i] = e;
}
/* Test for the degenerate case where the alignment by itself
produces the desired permutation. */
if (in_order)
- return true;
+ {
+ emit_move_insn (d->target, dcopy.op0);
+ return true;
+ }
- ok = expand_vec_perm_1 (d);
+ ok = expand_vec_perm_1 (&dcopy);
gcc_assert (ok);
return ok;
@@ -39874,10 +40553,10 @@ expand_vec_perm_interleave2 (struct expand_vec_perm_d *d)
else
dfinal.perm[i] = e;
}
- dfinal.op0 = gen_reg_rtx (dfinal.vmode);
+ dremap.target = gen_reg_rtx (dremap.vmode);
+ dfinal.op0 = gen_lowpart (dfinal.vmode, dremap.target);
dfinal.op1 = dfinal.op0;
dfinal.one_operand_p = true;
- dremap.target = dfinal.op0;
/* Test if the final remap can be done with a single insn. For V4SFmode or
V4SImode this *will* succeed. For V8HImode or V16QImode it may not. */
@@ -39894,7 +40573,6 @@ expand_vec_perm_interleave2 (struct expand_vec_perm_d *d)
if (dremap.vmode != dfinal.vmode)
{
- dremap.target = gen_lowpart (dremap.vmode, dremap.target);
dremap.op0 = gen_lowpart (dremap.vmode, dremap.op0);
dremap.op1 = gen_lowpart (dremap.vmode, dremap.op1);
}
@@ -40345,8 +41023,12 @@ expand_vec_perm_pshufb2 (struct expand_vec_perm_d *d)
op = gen_lowpart (V16QImode, d->op1);
emit_insn (gen_ssse3_pshufbv16qi3 (h, op, vperm));
- op = gen_lowpart (V16QImode, d->target);
+ op = d->target;
+ if (d->vmode != V16QImode)
+ op = gen_reg_rtx (V16QImode);
emit_insn (gen_iorv16qi3 (op, l, h));
+ if (op != d->target)
+ emit_move_insn (d->target, gen_lowpart (d->vmode, op));
return true;
}
@@ -40412,8 +41094,12 @@ expand_vec_perm_vpshufb2_vpermq (struct expand_vec_perm_d *d)
op = gen_lowpart (V32QImode, d->op0);
emit_insn (gen_avx2_pshufbv32qi3 (l, op, vperm));
- op = gen_lowpart (V32QImode, d->target);
+ op = d->target;
+ if (d->vmode != V32QImode)
+ op = gen_reg_rtx (V32QImode);
emit_insn (gen_iorv32qi3 (op, l, gen_lowpart (V32QImode, hp)));
+ if (op != d->target)
+ emit_move_insn (d->target, gen_lowpart (d->vmode, op));
return true;
}
@@ -40489,10 +41175,11 @@ expand_vec_perm_vpshufb2_vpermq_even_odd (struct expand_vec_perm_d *d)
emit_insn (gen_iorv32qi3 (ior, l, h));
/* Permute the V4DImode quarters using { 0, 2, 1, 3 } permutation. */
- op = gen_lowpart (V4DImode, d->target);
+ op = gen_reg_rtx (V4DImode);
ior = gen_lowpart (V4DImode, ior);
emit_insn (gen_avx2_permv4di_1 (op, ior, const0_rtx, const2_rtx,
const1_rtx, GEN_INT (3)));
+ emit_move_insn (d->target, gen_lowpart (d->vmode, op));
return true;
}
@@ -40503,7 +41190,7 @@ expand_vec_perm_vpshufb2_vpermq_even_odd (struct expand_vec_perm_d *d)
static bool
expand_vec_perm_even_odd_1 (struct expand_vec_perm_d *d, unsigned odd)
{
- rtx t1, t2, t3;
+ rtx t1, t2, t3, t4, t5;
switch (d->vmode)
{
@@ -40615,10 +41302,17 @@ expand_vec_perm_even_odd_1 (struct expand_vec_perm_d *d, unsigned odd)
{
struct expand_vec_perm_d d_copy = *d;
d_copy.vmode = V4DFmode;
- d_copy.target = gen_lowpart (V4DFmode, d->target);
+ d_copy.target = gen_reg_rtx (V4DFmode);
d_copy.op0 = gen_lowpart (V4DFmode, d->op0);
d_copy.op1 = gen_lowpart (V4DFmode, d->op1);
- return expand_vec_perm_even_odd_1 (&d_copy, odd);
+ if (expand_vec_perm_even_odd_1 (&d_copy, odd))
+ {
+ if (!d->testing_p)
+ emit_move_insn (d->target,
+ gen_lowpart (V4DImode, d_copy.target));
+ return true;
+ }
+ return false;
}
t1 = gen_reg_rtx (V4DImode);
@@ -40641,44 +41335,51 @@ expand_vec_perm_even_odd_1 (struct expand_vec_perm_d *d, unsigned odd)
{
struct expand_vec_perm_d d_copy = *d;
d_copy.vmode = V8SFmode;
- d_copy.target = gen_lowpart (V8SFmode, d->target);
+ d_copy.target = gen_reg_rtx (V8SFmode);
d_copy.op0 = gen_lowpart (V8SFmode, d->op0);
d_copy.op1 = gen_lowpart (V8SFmode, d->op1);
- return expand_vec_perm_even_odd_1 (&d_copy, odd);
+ if (expand_vec_perm_even_odd_1 (&d_copy, odd))
+ {
+ if (!d->testing_p)
+ emit_move_insn (d->target,
+ gen_lowpart (V8SImode, d_copy.target));
+ return true;
+ }
+ return false;
}
t1 = gen_reg_rtx (V8SImode);
t2 = gen_reg_rtx (V8SImode);
+ t3 = gen_reg_rtx (V4DImode);
+ t4 = gen_reg_rtx (V4DImode);
+ t5 = gen_reg_rtx (V4DImode);
/* Shuffle the lanes around into
{ 0 1 2 3 8 9 a b } and { 4 5 6 7 c d e f }. */
- emit_insn (gen_avx2_permv2ti (gen_lowpart (V4DImode, t1),
- gen_lowpart (V4DImode, d->op0),
+ emit_insn (gen_avx2_permv2ti (t3, gen_lowpart (V4DImode, d->op0),
gen_lowpart (V4DImode, d->op1),
GEN_INT (0x20)));
- emit_insn (gen_avx2_permv2ti (gen_lowpart (V4DImode, t2),
- gen_lowpart (V4DImode, d->op0),
+ emit_insn (gen_avx2_permv2ti (t4, gen_lowpart (V4DImode, d->op0),
gen_lowpart (V4DImode, d->op1),
GEN_INT (0x31)));
/* Swap the 2nd and 3rd position in each lane into
{ 0 2 1 3 8 a 9 b } and { 4 6 5 7 c e d f }. */
- emit_insn (gen_avx2_pshufdv3 (t1, t1,
+ emit_insn (gen_avx2_pshufdv3 (t1, gen_lowpart (V8SImode, t3),
GEN_INT (2 * 4 + 1 * 16 + 3 * 64)));
- emit_insn (gen_avx2_pshufdv3 (t2, t2,
+ emit_insn (gen_avx2_pshufdv3 (t2, gen_lowpart (V8SImode, t4),
GEN_INT (2 * 4 + 1 * 16 + 3 * 64)));
/* Now an vpunpck[lh]qdq will produce
{ 0 2 4 6 8 a c e } resp. { 1 3 5 7 9 b d f }. */
if (odd)
- t3 = gen_avx2_interleave_highv4di (gen_lowpart (V4DImode, d->target),
- gen_lowpart (V4DImode, t1),
+ t3 = gen_avx2_interleave_highv4di (t5, gen_lowpart (V4DImode, t1),
gen_lowpart (V4DImode, t2));
else
- t3 = gen_avx2_interleave_lowv4di (gen_lowpart (V4DImode, d->target),
- gen_lowpart (V4DImode, t1),
+ t3 = gen_avx2_interleave_lowv4di (t5, gen_lowpart (V4DImode, t1),
gen_lowpart (V4DImode, t2));
emit_insn (t3);
+ emit_move_insn (d->target, gen_lowpart (V8SImode, t5));
break;
default:
@@ -40716,7 +41417,7 @@ expand_vec_perm_broadcast_1 (struct expand_vec_perm_d *d)
unsigned elt = d->perm[0], nelt2 = d->nelt / 2;
enum machine_mode vmode = d->vmode;
unsigned char perm2[4];
- rtx op0 = d->op0;
+ rtx op0 = d->op0, dest;
bool ok;
switch (vmode)
@@ -40762,9 +41463,11 @@ expand_vec_perm_broadcast_1 (struct expand_vec_perm_d *d)
while (vmode != V4SImode);
memset (perm2, elt, 4);
- ok = expand_vselect (gen_lowpart (V4SImode, d->target), op0, perm2, 4,
- d->testing_p);
+ dest = gen_reg_rtx (V4SImode);
+ ok = expand_vselect (dest, op0, perm2, 4, d->testing_p);
gcc_assert (ok);
+ if (!d->testing_p)
+ emit_move_insn (d->target, gen_lowpart (d->vmode, dest));
return true;
case V32QImode:
@@ -40906,8 +41609,12 @@ expand_vec_perm_vpshufb4_vpermq2 (struct expand_vec_perm_d *d)
}
gcc_assert (l[0] && l[1]);
- op = gen_lowpart (V32QImode, d->target);
+ op = d->target;
+ if (d->vmode != V32QImode)
+ op = gen_reg_rtx (V32QImode);
emit_insn (gen_iorv32qi3 (op, l[0], l[1]));
+ if (op != d->target)
+ emit_move_insn (d->target, gen_lowpart (d->vmode, op));
return true;
}
@@ -41475,7 +42182,9 @@ ix86_expand_mul_widen_hilo (rtx dest, rtx op1, rtx op2,
op1, op2, NULL_RTX, uns_p, OPTAB_DIRECT);
gcc_assert (t1 && t2);
- ix86_expand_vec_interleave (gen_lowpart (mode, dest), t1, t2, high_p);
+ t3 = gen_reg_rtx (mode);
+ ix86_expand_vec_interleave (t3, t1, t2, high_p);
+ emit_move_insn (dest, gen_lowpart (wmode, t3));
break;
case V16QImode:
@@ -41496,14 +42205,14 @@ ix86_expand_mul_widen_hilo (rtx dest, rtx op1, rtx op2,
void
ix86_expand_sse2_mulv4si3 (rtx op0, rtx op1, rtx op2)
{
- rtx res_1, res_2;
+ rtx res_1, res_2, res_3, res_4;
res_1 = gen_reg_rtx (V4SImode);
res_2 = gen_reg_rtx (V4SImode);
- ix86_expand_mul_widen_evenodd (gen_lowpart (V2DImode, res_1),
- op1, op2, true, false);
- ix86_expand_mul_widen_evenodd (gen_lowpart (V2DImode, res_2),
- op1, op2, true, true);
+ res_3 = gen_reg_rtx (V2DImode);
+ res_4 = gen_reg_rtx (V2DImode);
+ ix86_expand_mul_widen_evenodd (res_3, op1, op2, true, false);
+ ix86_expand_mul_widen_evenodd (res_4, op1, op2, true, true);
/* Move the results in element 2 down to element 1; we don't care
what goes in elements 2 and 3. Then we can merge the parts
@@ -41517,9 +42226,11 @@ ix86_expand_sse2_mulv4si3 (rtx op0, rtx op1, rtx op2)
In both cases the cost of the reformatting stall was too high
and the overall sequence slower. */
- emit_insn (gen_sse2_pshufd_1 (res_1, res_1, const0_rtx, const2_rtx,
+ emit_insn (gen_sse2_pshufd_1 (res_1, gen_lowpart (V4SImode, res_3),
+ const0_rtx, const2_rtx,
const0_rtx, const0_rtx));
- emit_insn (gen_sse2_pshufd_1 (res_2, res_2, const0_rtx, const2_rtx,
+ emit_insn (gen_sse2_pshufd_1 (res_2, gen_lowpart (V4SImode, res_4),
+ const0_rtx, const2_rtx,
const0_rtx, const0_rtx));
res_1 = emit_insn (gen_vec_interleave_lowv4si (op0, res_1, res_2));
@@ -41608,6 +42319,68 @@ ix86_expand_sse2_mulvxdi3 (rtx op0, rtx op1, rtx op2)
gen_rtx_MULT (mode, op1, op2));
}
+/* Return 1 if control tansfer instruction INSN
+ should be encoded with bnd prefix.
+ If insn is NULL then return 1 when control
+ transfer instructions should be prefixed with
+ bnd by default for current function. */
+
+bool
+ix86_bnd_prefixed_insn_p (rtx insn ATTRIBUTE_UNUSED)
+{
+ return false;
+}
+
+/* Calculate integer abs() using only SSE2 instructions. */
+
+void
+ix86_expand_sse2_abs (rtx target, rtx input)
+{
+ enum machine_mode mode = GET_MODE (target);
+ rtx tmp0, tmp1, x;
+
+ switch (mode)
+ {
+ /* For 32-bit signed integer X, the best way to calculate the absolute
+ value of X is (((signed) X >> (W-1)) ^ X) - ((signed) X >> (W-1)). */
+ case V4SImode:
+ tmp0 = expand_simple_binop (mode, ASHIFTRT, input,
+ GEN_INT (GET_MODE_BITSIZE
+ (GET_MODE_INNER (mode)) - 1),
+ NULL, 0, OPTAB_DIRECT);
+ tmp1 = expand_simple_binop (mode, XOR, tmp0, input,
+ NULL, 0, OPTAB_DIRECT);
+ x = expand_simple_binop (mode, MINUS, tmp1, tmp0,
+ target, 0, OPTAB_DIRECT);
+ break;
+
+ /* For 16-bit signed integer X, the best way to calculate the absolute
+ value of X is max (X, -X), as SSE2 provides the PMAXSW insn. */
+ case V8HImode:
+ tmp0 = expand_unop (mode, neg_optab, input, NULL_RTX, 0);
+
+ x = expand_simple_binop (mode, SMAX, tmp0, input,
+ target, 0, OPTAB_DIRECT);
+ break;
+
+ /* For 8-bit signed integer X, the best way to calculate the absolute
+ value of X is min ((unsigned char) X, (unsigned char) (-X)),
+ as SSE2 provides the PMINUB insn. */
+ case V16QImode:
+ tmp0 = expand_unop (mode, neg_optab, input, NULL_RTX, 0);
+
+ x = expand_simple_binop (V16QImode, UMIN, tmp0, input,
+ target, 0, OPTAB_DIRECT);
+ break;
+
+ default:
+ gcc_unreachable ();
+ }
+
+ if (x != target)
+ emit_move_insn (target, x);
+}
+
/* Expand an insert into a vector register through pinsr insn.
Return true if successful. */
@@ -41676,12 +42449,17 @@ ix86_expand_pinsr (rtx *operands)
return false;
}
- dst = gen_lowpart (dstmode, dst);
+ rtx d = dst;
+ if (GET_MODE (dst) != dstmode)
+ d = gen_reg_rtx (dstmode);
src = gen_lowpart (srcmode, src);
pos /= size;
- emit_insn (pinsr (dst, dst, src, GEN_INT (1 << pos)));
+ emit_insn (pinsr (d, gen_lowpart (dstmode, dst), src,
+ GEN_INT (1 << pos)));
+ if (d != dst)
+ emit_move_insn (dst, gen_lowpart (GET_MODE (dst), d));
return true;
}
@@ -42736,20 +43514,17 @@ ix86_add_stmt_cost (void *data, int count, enum vect_cost_for_stmt kind,
unsigned *cost = (unsigned *) data;
unsigned retval = 0;
- if (flag_vect_cost_model)
- {
- tree vectype = stmt_info ? stmt_vectype (stmt_info) : NULL_TREE;
- int stmt_cost = ix86_builtin_vectorization_cost (kind, vectype, misalign);
+ tree vectype = stmt_info ? stmt_vectype (stmt_info) : NULL_TREE;
+ int stmt_cost = ix86_builtin_vectorization_cost (kind, vectype, misalign);
- /* Statements in an inner loop relative to the loop being
- vectorized are weighted more heavily. The value here is
- arbitrary and could potentially be improved with analysis. */
- if (where == vect_body && stmt_info && stmt_in_inner_loop_p (stmt_info))
- count *= 50; /* FIXME. */
+ /* Statements in an inner loop relative to the loop being
+ vectorized are weighted more heavily. The value here is
+ arbitrary and could potentially be improved with analysis. */
+ if (where == vect_body && stmt_info && stmt_in_inner_loop_p (stmt_info))
+ count *= 50; /* FIXME. */
- retval = (unsigned) (count * stmt_cost);
- cost[where] += retval;
- }
+ retval = (unsigned) (count * stmt_cost);
+ cost[where] += retval;
return retval;
}
@@ -42806,6 +43581,115 @@ ix86_memmodel_check (unsigned HOST_WIDE_INT val)
return val;
}
+/* Implement TARGET_FLOAT_EXCEPTIONS_ROUNDING_SUPPORTED_P. */
+
+static bool
+ix86_float_exceptions_rounding_supported_p (void)
+{
+ /* For x87 floating point with standard excess precision handling,
+ there is no adddf3 pattern (since x87 floating point only has
+ XFmode operations) so the default hook implementation gets this
+ wrong. */
+ return TARGET_80387 || TARGET_SSE_MATH;
+}
+
+/* Implement TARGET_ATOMIC_ASSIGN_EXPAND_FENV. */
+
+static void
+ix86_atomic_assign_expand_fenv (tree *hold, tree *clear, tree *update)
+{
+ if (!TARGET_80387 && !TARGET_SSE_MATH)
+ return;
+ tree exceptions_var = create_tmp_var (integer_type_node, NULL);
+ if (TARGET_80387)
+ {
+ tree fenv_index_type = build_index_type (size_int (6));
+ tree fenv_type = build_array_type (unsigned_type_node, fenv_index_type);
+ tree fenv_var = create_tmp_var (fenv_type, NULL);
+ mark_addressable (fenv_var);
+ tree fenv_ptr = build_pointer_type (fenv_type);
+ tree fenv_addr = build1 (ADDR_EXPR, fenv_ptr, fenv_var);
+ fenv_addr = fold_convert (ptr_type_node, fenv_addr);
+ tree fnstenv = ix86_builtins[IX86_BUILTIN_FNSTENV];
+ tree fldenv = ix86_builtins[IX86_BUILTIN_FLDENV];
+ tree fnstsw = ix86_builtins[IX86_BUILTIN_FNSTSW];
+ tree fnclex = ix86_builtins[IX86_BUILTIN_FNCLEX];
+ tree hold_fnstenv = build_call_expr (fnstenv, 1, fenv_addr);
+ tree hold_fnclex = build_call_expr (fnclex, 0);
+ *hold = build2 (COMPOUND_EXPR, void_type_node, hold_fnstenv,
+ hold_fnclex);
+ *clear = build_call_expr (fnclex, 0);
+ tree sw_var = create_tmp_var (short_unsigned_type_node, NULL);
+ mark_addressable (sw_var);
+ tree su_ptr = build_pointer_type (short_unsigned_type_node);
+ tree sw_addr = build1 (ADDR_EXPR, su_ptr, sw_var);
+ tree fnstsw_call = build_call_expr (fnstsw, 1, sw_addr);
+ tree exceptions_x87 = fold_convert (integer_type_node, sw_var);
+ tree update_mod = build2 (MODIFY_EXPR, integer_type_node,
+ exceptions_var, exceptions_x87);
+ *update = build2 (COMPOUND_EXPR, integer_type_node,
+ fnstsw_call, update_mod);
+ tree update_fldenv = build_call_expr (fldenv, 1, fenv_addr);
+ *update = build2 (COMPOUND_EXPR, void_type_node, *update, update_fldenv);
+ }
+ if (TARGET_SSE_MATH)
+ {
+ tree mxcsr_orig_var = create_tmp_var (unsigned_type_node, NULL);
+ tree mxcsr_mod_var = create_tmp_var (unsigned_type_node, NULL);
+ tree stmxcsr = ix86_builtins[IX86_BUILTIN_STMXCSR];
+ tree ldmxcsr = ix86_builtins[IX86_BUILTIN_LDMXCSR];
+ tree stmxcsr_hold_call = build_call_expr (stmxcsr, 0);
+ tree hold_assign_orig = build2 (MODIFY_EXPR, unsigned_type_node,
+ mxcsr_orig_var, stmxcsr_hold_call);
+ tree hold_mod_val = build2 (BIT_IOR_EXPR, unsigned_type_node,
+ mxcsr_orig_var,
+ build_int_cst (unsigned_type_node, 0x1f80));
+ hold_mod_val = build2 (BIT_AND_EXPR, unsigned_type_node, hold_mod_val,
+ build_int_cst (unsigned_type_node, 0xffffffc0));
+ tree hold_assign_mod = build2 (MODIFY_EXPR, unsigned_type_node,
+ mxcsr_mod_var, hold_mod_val);
+ tree ldmxcsr_hold_call = build_call_expr (ldmxcsr, 1, mxcsr_mod_var);
+ tree hold_all = build2 (COMPOUND_EXPR, unsigned_type_node,
+ hold_assign_orig, hold_assign_mod);
+ hold_all = build2 (COMPOUND_EXPR, void_type_node, hold_all,
+ ldmxcsr_hold_call);
+ if (*hold)
+ *hold = build2 (COMPOUND_EXPR, void_type_node, *hold, hold_all);
+ else
+ *hold = hold_all;
+ tree ldmxcsr_clear_call = build_call_expr (ldmxcsr, 1, mxcsr_mod_var);
+ if (*clear)
+ *clear = build2 (COMPOUND_EXPR, void_type_node, *clear,
+ ldmxcsr_clear_call);
+ else
+ *clear = ldmxcsr_clear_call;
+ tree stxmcsr_update_call = build_call_expr (stmxcsr, 0);
+ tree exceptions_sse = fold_convert (integer_type_node,
+ stxmcsr_update_call);
+ if (*update)
+ {
+ tree exceptions_mod = build2 (BIT_IOR_EXPR, integer_type_node,
+ exceptions_var, exceptions_sse);
+ tree exceptions_assign = build2 (MODIFY_EXPR, integer_type_node,
+ exceptions_var, exceptions_mod);
+ *update = build2 (COMPOUND_EXPR, integer_type_node, *update,
+ exceptions_assign);
+ }
+ else
+ *update = build2 (MODIFY_EXPR, integer_type_node,
+ exceptions_var, exceptions_sse);
+ tree ldmxcsr_update_call = build_call_expr (ldmxcsr, 1, mxcsr_orig_var);
+ *update = build2 (COMPOUND_EXPR, void_type_node, *update,
+ ldmxcsr_update_call);
+ }
+ tree atomic_feraiseexcept
+ = builtin_decl_implicit (BUILT_IN_ATOMIC_FERAISEEXCEPT);
+ tree atomic_feraiseexcept_call = build_call_expr (atomic_feraiseexcept,
+ 1, exceptions_var);
+ *update = build2 (COMPOUND_EXPR, void_type_node, *update,
+ atomic_feraiseexcept_call);
+}
+
/* Initialize the GCC target structure. */
#undef TARGET_RETURN_IN_MEMORY
#define TARGET_RETURN_IN_MEMORY ix86_return_in_memory
@@ -42906,6 +43790,10 @@ ix86_memmodel_check (unsigned HOST_WIDE_INT val)
#undef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD
#define TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD \
ia32_multipass_dfa_lookahead
+#undef TARGET_SCHED_MACRO_FUSION_P
+#define TARGET_SCHED_MACRO_FUSION_P ix86_macro_fusion_p
+#undef TARGET_SCHED_MACRO_FUSION_PAIR_P
+#define TARGET_SCHED_MACRO_FUSION_PAIR_P ix86_macro_fusion_pair_p
#undef TARGET_FUNCTION_OK_FOR_SIBCALL
#define TARGET_FUNCTION_OK_FOR_SIBCALL ix86_function_ok_for_sibcall
@@ -42913,6 +43801,9 @@ ix86_memmodel_check (unsigned HOST_WIDE_INT val)
#undef TARGET_MEMMODEL_CHECK
#define TARGET_MEMMODEL_CHECK ix86_memmodel_check
+#undef TARGET_ATOMIC_ASSIGN_EXPAND_FENV
+#define TARGET_ATOMIC_ASSIGN_EXPAND_FENV ix86_atomic_assign_expand_fenv
+
#ifdef HAVE_AS_TLS
#undef TARGET_HAVE_TLS
#define TARGET_HAVE_TLS true
@@ -43178,6 +44069,10 @@ ix86_memmodel_check (unsigned HOST_WIDE_INT val)
#undef TARGET_SPILL_CLASS
#define TARGET_SPILL_CLASS ix86_spill_class
+#undef TARGET_FLOAT_EXCEPTIONS_ROUNDING_SUPPORTED_P
+#define TARGET_FLOAT_EXCEPTIONS_ROUNDING_SUPPORTED_P \
+ ix86_float_exceptions_rounding_supported_p
+
struct gcc_target targetm = TARGET_INITIALIZER;
#include "gt-i386.h"
diff --git a/gcc/config/i386/i386.h b/gcc/config/i386/i386.h
index 440844e7735..123e3faed4b 100644
--- a/gcc/config/i386/i386.h
+++ b/gcc/config/i386/i386.h
@@ -40,53 +40,101 @@ see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
/* Redefines for option macros. */
#define TARGET_64BIT TARGET_ISA_64BIT
+#define TARGET_64BIT_P(x) TARGET_ISA_64BIT_P(x)
#define TARGET_MMX TARGET_ISA_MMX
+#define TARGET_MMX_P(x) TARGET_ISA_MMX_P(x)
#define TARGET_3DNOW TARGET_ISA_3DNOW
+#define TARGET_3DNOW_P(x) TARGET_ISA_3DNOW_P(x)
#define TARGET_3DNOW_A TARGET_ISA_3DNOW_A
+#define TARGET_3DNOW_A_P(x) TARGET_ISA_3DNOW_A_P(x)
#define TARGET_SSE TARGET_ISA_SSE
+#define TARGET_SSE_P(x) TARGET_ISA_SSE_P(x)
#define TARGET_SSE2 TARGET_ISA_SSE2
+#define TARGET_SSE2_P(x) TARGET_ISA_SSE2_P(x)
#define TARGET_SSE3 TARGET_ISA_SSE3
+#define TARGET_SSE3_P(x) TARGET_ISA_SSE3_P(x)
#define TARGET_SSSE3 TARGET_ISA_SSSE3
+#define TARGET_SSSE3_P(x) TARGET_ISA_SSSE3_P(x)
#define TARGET_SSE4_1 TARGET_ISA_SSE4_1
+#define TARGET_SSE4_1_P(x) TARGET_ISA_SSE4_1_P(x)
#define TARGET_SSE4_2 TARGET_ISA_SSE4_2
+#define TARGET_SSE4_2_P(x) TARGET_ISA_SSE4_2_P(x)
#define TARGET_AVX TARGET_ISA_AVX
+#define TARGET_AVX_P(x) TARGET_ISA_AVX_P(x)
#define TARGET_AVX2 TARGET_ISA_AVX2
-#define TARGET_AVX512F TARGET_ISA_AVX512F
-#define TARGET_AVX512PF TARGET_ISA_AVX512PF
-#define TARGET_AVX512ER TARGET_ISA_AVX512ER
-#define TARGET_AVX512CD TARGET_ISA_AVX512CD
+#define TARGET_AVX2_P(x) TARGET_ISA_AVX2_P(x)
+#define TARGET_AVX512F TARGET_ISA_AVX512F
+#define TARGET_AVX512F_P(x) TARGET_ISA_AVX512F_P(x)
+#define TARGET_AVX512PF TARGET_ISA_AVX512PF
+#define TARGET_AVX512PF_P(x) TARGET_ISA_AVX512PF_P(x)
+#define TARGET_AVX512ER TARGET_ISA_AVX512ER
+#define TARGET_AVX512ER_P(x) TARGET_ISA_AVX512ER_P(x)
+#define TARGET_AVX512CD TARGET_ISA_AVX512CD
+#define TARGET_AVX512CD_P(x) TARGET_ISA_AVX512CD_P(x)
#define TARGET_FMA TARGET_ISA_FMA
+#define TARGET_FMA_P(x) TARGET_ISA_FMA_P(x)
#define TARGET_SSE4A TARGET_ISA_SSE4A
+#define TARGET_SSE4A_P(x) TARGET_ISA_SSE4A_P(x)
#define TARGET_FMA4 TARGET_ISA_FMA4
+#define TARGET_FMA4_P(x) TARGET_ISA_FMA4_P(x)
#define TARGET_XOP TARGET_ISA_XOP
+#define TARGET_XOP_P(x) TARGET_ISA_XOP_P(x)
#define TARGET_LWP TARGET_ISA_LWP
+#define TARGET_LWP_P(x) TARGET_ISA_LWP_P(x)
#define TARGET_ROUND TARGET_ISA_ROUND
#define TARGET_ABM TARGET_ISA_ABM
+#define TARGET_ABM_P(x) TARGET_ISA_ABM_P(x)
#define TARGET_BMI TARGET_ISA_BMI
+#define TARGET_BMI_P(x) TARGET_ISA_BMI_P(x)
#define TARGET_BMI2 TARGET_ISA_BMI2
+#define TARGET_BMI2_P(x) TARGET_ISA_BMI2_P(x)
#define TARGET_LZCNT TARGET_ISA_LZCNT
+#define TARGET_LZCNT_P(x) TARGET_ISA_LZCNT_P(x)
#define TARGET_TBM TARGET_ISA_TBM
+#define TARGET_TBM_P(x) TARGET_ISA_TBM_P(x)
#define TARGET_POPCNT TARGET_ISA_POPCNT
+#define TARGET_POPCNT_P(x) TARGET_ISA_POPCNT_P(x)
#define TARGET_SAHF TARGET_ISA_SAHF
+#define TARGET_SAHF_P(x) TARGET_ISA_SAHF_P(x)
#define TARGET_MOVBE TARGET_ISA_MOVBE
+#define TARGET_MOVBE_P(x) TARGET_ISA_MOVBE_P(x)
#define TARGET_CRC32 TARGET_ISA_CRC32
+#define TARGET_CRC32_P(x) TARGET_ISA_CRC32_P(x)
#define TARGET_AES TARGET_ISA_AES
+#define TARGET_AES_P(x) TARGET_ISA_AES_P(x)
#define TARGET_PCLMUL TARGET_ISA_PCLMUL
-#define TARGET_CMPXCHG16B TARGET_ISA_CX16
+#define TARGET_PCLMUL_P(x) TARGET_ISA_PCLMUL_P(x)
+#define TARGET_CMPXCHG16B TARGET_ISA_CX16
+#define TARGET_CMPXCHG16B_P(x) TARGET_ISA_CX16_P(x)
#define TARGET_FSGSBASE TARGET_ISA_FSGSBASE
+#define TARGET_FSGSBASE_P(x) TARGET_ISA_FSGSBASE_P(x)
#define TARGET_RDRND TARGET_ISA_RDRND
+#define TARGET_RDRND_P(x) TARGET_ISA_RDRND_P(x)
#define TARGET_F16C TARGET_ISA_F16C
-#define TARGET_RTM TARGET_ISA_RTM
+#define TARGET_F16C_P(x) TARGET_ISA_F16C_P(x)
+#define TARGET_RTM TARGET_ISA_RTM
+#define TARGET_RTM_P(x) TARGET_ISA_RTM_P(x)
#define TARGET_HLE TARGET_ISA_HLE
+#define TARGET_HLE_P(x) TARGET_ISA_HLE_P(x)
#define TARGET_RDSEED TARGET_ISA_RDSEED
+#define TARGET_RDSEED_P(x) TARGET_ISA_RDSEED_P(x)
#define TARGET_PRFCHW TARGET_ISA_PRFCHW
+#define TARGET_PRFCHW_P(x) TARGET_ISA_PRFCHW_P(x)
#define TARGET_ADX TARGET_ISA_ADX
+#define TARGET_ADX_P(x) TARGET_ISA_ADX_P(x)
#define TARGET_FXSR TARGET_ISA_FXSR
+#define TARGET_FXSR_P(x) TARGET_ISA_FXSR_P(x)
#define TARGET_XSAVE TARGET_ISA_XSAVE
+#define TARGET_XSAVE_P(x) TARGET_ISA_XSAVE_P(x)
#define TARGET_XSAVEOPT TARGET_ISA_XSAVEOPT
+#define TARGET_XSAVEOPT_P(x) TARGET_ISA_XSAVEOPT_P(x)
+#define TARGET_MPX TARGET_ISA_MPX
+#define TARGET_MPX_P(x) TARGET_ISA_MPX_P(x)
#define TARGET_LP64 TARGET_ABI_64
+#define TARGET_LP64_P(x) TARGET_ABI_64_P(x)
#define TARGET_X32 TARGET_ABI_X32
+#define TARGET_X32_P(x) TARGET_ABI_X32_P(x)
/* SSE4.1 defines round instructions */
#define OPTION_MASK_ISA_ROUND OPTION_MASK_ISA_SSE4_1
@@ -212,7 +260,13 @@ extern const struct processor_costs ix86_size_cost;
(TARGET_64BIT && TARGET_SSE ? FPMATH_SSE : FPMATH_387)
#endif
+#ifndef TARGET_FPMATH_DEFAULT_P
+#define TARGET_FPMATH_DEFAULT_P(x) \
+ (TARGET_64BIT_P(x) && TARGET_SSE_P(x) ? FPMATH_SSE : FPMATH_387)
+#endif
+
#define TARGET_FLOAT_RETURNS_IN_80387 TARGET_FLOAT_RETURNS
+#define TARGET_FLOAT_RETURNS_IN_80387_P(x) TARGET_FLOAT_RETURNS_P(x)
/* 64bit Sledgehammer mode. For libgcc2 we make sure this is a
compile-time constant. */
@@ -250,6 +304,7 @@ extern const struct processor_costs ix86_size_cost;
#define TARGET_NOCONA (ix86_tune == PROCESSOR_NOCONA)
#define TARGET_CORE2 (ix86_tune == PROCESSOR_CORE2)
#define TARGET_COREI7 (ix86_tune == PROCESSOR_COREI7)
+#define TARGET_COREI7_AVX (ix86_tune == PROCESSOR_COREI7_AVX)
#define TARGET_HASWELL (ix86_tune == PROCESSOR_HASWELL)
#define TARGET_GENERIC (ix86_tune == PROCESSOR_GENERIC)
#define TARGET_AMDFAM10 (ix86_tune == PROCESSOR_AMDFAM10)
@@ -298,6 +353,8 @@ extern unsigned char ix86_tune_features[X86_TUNE_LAST];
#define TARGET_PROMOTE_QImode ix86_tune_features[X86_TUNE_PROMOTE_QIMODE]
#define TARGET_FAST_PREFIX ix86_tune_features[X86_TUNE_FAST_PREFIX]
#define TARGET_SINGLE_STRINGOP ix86_tune_features[X86_TUNE_SINGLE_STRINGOP]
+#define TARGET_MISALIGNED_MOVE_STRING_PRO_EPILOGUES \
+ ix86_tune_features[X86_TUNE_MISALIGNED_MOVE_STRING_PRO_EPILOGUES]
#define TARGET_QIMODE_MATH ix86_tune_features[X86_TUNE_QIMODE_MATH]
#define TARGET_HIMODE_MATH ix86_tune_features[X86_TUNE_HIMODE_MATH]
#define TARGET_PROMOTE_QI_REGS ix86_tune_features[X86_TUNE_PROMOTE_QI_REGS]
@@ -359,8 +416,17 @@ extern unsigned char ix86_tune_features[X86_TUNE_LAST];
ix86_tune_features[X86_TUNE_USE_VECTOR_FP_CONVERTS]
#define TARGET_USE_VECTOR_CONVERTS \
ix86_tune_features[X86_TUNE_USE_VECTOR_CONVERTS]
+#define TARGET_FUSE_CMP_AND_BRANCH_32 \
+ ix86_tune_features[X86_TUNE_FUSE_CMP_AND_BRANCH_32]
+#define TARGET_FUSE_CMP_AND_BRANCH_64 \
+ ix86_tune_features[X86_TUNE_FUSE_CMP_AND_BRANCH_64]
#define TARGET_FUSE_CMP_AND_BRANCH \
- ix86_tune_features[X86_TUNE_FUSE_CMP_AND_BRANCH]
+ (TARGET_64BIT ? TARGET_FUSE_CMP_AND_BRANCH_64 \
+ : TARGET_FUSE_CMP_AND_BRANCH_32)
+#define TARGET_FUSE_CMP_AND_BRANCH_SOFLAGS \
+ ix86_tune_features[X86_TUNE_FUSE_CMP_AND_BRANCH_SOFLAGS]
+#define TARGET_FUSE_ALU_AND_BRANCH \
+ ix86_tune_features[X86_TUNE_FUSE_ALU_AND_BRANCH]
#define TARGET_OPT_AGU ix86_tune_features[X86_TUNE_OPT_AGU]
#define TARGET_VECTORIZE_DOUBLE \
ix86_tune_features[X86_TUNE_VECTORIZE_DOUBLE]
@@ -557,6 +623,7 @@ enum target_cpu_default
TARGET_CPU_DEFAULT_nocona,
TARGET_CPU_DEFAULT_core2,
TARGET_CPU_DEFAULT_corei7,
+ TARGET_CPU_DEFAULT_corei7_avx,
TARGET_CPU_DEFAULT_haswell,
TARGET_CPU_DEFAULT_atom,
TARGET_CPU_DEFAULT_slm,
@@ -891,7 +958,7 @@ enum target_cpu_default
eliminated during reloading in favor of either the stack or frame
pointer. */
-#define FIRST_PSEUDO_REGISTER 77
+#define FIRST_PSEUDO_REGISTER 81
/* Number of hardware registers that go into the DWARF-2 unwind info.
If not defined, equals FIRST_PSEUDO_REGISTER. */
@@ -923,7 +990,9 @@ enum target_cpu_default
/*xmm24,xmm25,xmm26,xmm27,xmm28,xmm29,xmm30,xmm31*/ \
0, 0, 0, 0, 0, 0, 0, 0, \
/* k0, k1, k2, k3, k4, k5, k6, k7*/ \
- 0, 0, 0, 0, 0, 0, 0, 0 }
+ 0, 0, 0, 0, 0, 0, 0, 0, \
+/* b0, b1, b2, b3*/ \
+ 0, 0, 0, 0 }
/* 1 for registers not available across function calls.
These must include the FIXED_REGISTERS and also any
@@ -957,7 +1026,9 @@ enum target_cpu_default
/*xmm24,xmm25,xmm26,xmm27,xmm28,xmm29,xmm30,xmm31*/ \
6, 6, 6, 6, 6, 6, 6, 6, \
/* k0, k1, k2, k3, k4, k5, k6, k7*/ \
- 1, 1, 1, 1, 1, 1, 1, 1 }
+ 1, 1, 1, 1, 1, 1, 1, 1, \
+/* b0, b1, b2, b3*/ \
+ 1, 1, 1, 1 }
/* Order in which to allocate registers. Each register must be
listed once, even those in FIXED_REGISTERS. List frame pointer
@@ -973,7 +1044,8 @@ enum target_cpu_default
18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, \
33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, \
48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, \
- 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76 }
+ 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, \
+ 78, 79, 80 }
/* ADJUST_REG_ALLOC_ORDER is a macro which permits reg_alloc_order
to be rearranged based on a particular function. When using sse math,
@@ -995,6 +1067,7 @@ enum target_cpu_default
#define HARD_REGNO_NREGS(REGNO, MODE) \
(STACK_REGNO_P (REGNO) || SSE_REGNO_P (REGNO) || MMX_REGNO_P (REGNO) \
+ || BND_REGNO_P (REGNO) \
? (COMPLEX_MODE_P (MODE) ? 2 : 1) \
: ((MODE) == XFmode \
? (TARGET_64BIT ? 2 : 3) \
@@ -1044,6 +1117,9 @@ enum target_cpu_default
|| (MODE) == V2SImode || (MODE) == SImode \
|| (MODE) == V4HImode || (MODE) == V8QImode)
+#define VALID_BND_REG_MODE(MODE) \
+ (TARGET_64BIT ? (MODE) == BND64mode : (MODE) == BND32mode)
+
#define VALID_DFP_MODE_P(MODE) \
((MODE) == SDmode || (MODE) == DDmode || (MODE) == TDmode)
@@ -1150,6 +1226,9 @@ enum target_cpu_default
#define FIRST_MASK_REG (LAST_EXT_REX_SSE_REG + 1) /*69*/
#define LAST_MASK_REG (FIRST_MASK_REG + 7) /*76*/
+#define FIRST_BND_REG (LAST_MASK_REG + 1) /*77*/
+#define LAST_BND_REG (FIRST_BND_REG + 3) /*80*/
+
/* Override this in other tm.h files to cope with various OS lossage
requiring a frame pointer. */
#ifndef SUBTARGET_FRAME_POINTER_REQUIRED
@@ -1230,6 +1309,7 @@ enum reg_class
SSE_FIRST_REG,
SSE_REGS,
EVEX_SSE_REGS,
+ BND_REGS,
ALL_SSE_REGS,
MMX_REGS,
FP_TOP_SSE_REGS,
@@ -1287,6 +1367,7 @@ enum reg_class
"SSE_FIRST_REG", \
"SSE_REGS", \
"EVEX_SSE_REGS", \
+ "BND_REGS", \
"ALL_SSE_REGS", \
"MMX_REGS", \
"FP_TOP_SSE_REGS", \
@@ -1306,37 +1387,38 @@ enum reg_class
TARGET_CONDITIONAL_REGISTER_USAGE. */
#define REG_CLASS_CONTENTS \
-{ { 0x00, 0x0, 0x0 }, \
- { 0x01, 0x0, 0x0 }, /* AREG */ \
- { 0x02, 0x0, 0x0 }, /* DREG */ \
- { 0x04, 0x0, 0x0 }, /* CREG */ \
- { 0x08, 0x0, 0x0 }, /* BREG */ \
- { 0x10, 0x0, 0x0 }, /* SIREG */ \
- { 0x20, 0x0, 0x0 }, /* DIREG */ \
- { 0x03, 0x0, 0x0 }, /* AD_REGS */ \
- { 0x0f, 0x0, 0x0 }, /* Q_REGS */ \
- { 0x1100f0, 0x1fe0, 0x0 }, /* NON_Q_REGS */ \
- { 0x7f, 0x1fe0, 0x0 }, /* INDEX_REGS */ \
- { 0x1100ff, 0x0, 0x0 }, /* LEGACY_REGS */ \
- { 0x07, 0x0, 0x0 }, /* CLOBBERED_REGS */ \
- { 0x1100ff, 0x1fe0, 0x0 }, /* GENERAL_REGS */ \
- { 0x100, 0x0, 0x0 }, /* FP_TOP_REG */ \
- { 0x0200, 0x0, 0x0 }, /* FP_SECOND_REG */ \
- { 0xff00, 0x0, 0x0 }, /* FLOAT_REGS */ \
- { 0x200000, 0x0, 0x0 }, /* SSE_FIRST_REG */ \
-{ 0x1fe00000, 0x1fe000, 0x0 }, /* SSE_REGS */ \
- { 0x0,0xffe00000, 0x1f }, /* EVEX_SSE_REGS */ \
-{ 0x1fe00000,0xffffe000, 0x1f }, /* ALL_SSE_REGS */ \
-{ 0xe0000000, 0x1f, 0x0 }, /* MMX_REGS */ \
-{ 0x1fe00100,0xffffe000, 0x1f }, /* FP_TOP_SSE_REG */ \
-{ 0x1fe00200,0xffffe000, 0x1f }, /* FP_SECOND_SSE_REG */ \
-{ 0x1fe0ff00,0xffffe000, 0x1f }, /* FLOAT_SSE_REGS */ \
-{ 0x11ffff, 0x1fe0, 0x0 }, /* FLOAT_INT_REGS */ \
-{ 0x1ff100ff,0xffffffe0, 0x1f }, /* INT_SSE_REGS */ \
-{ 0x1ff1ffff,0xffffffe0, 0x1f }, /* FLOAT_INT_SSE_REGS */ \
- { 0x0, 0x0,0x1fc0 }, /* MASK_EVEX_REGS */ \
- { 0x0, 0x0,0x1fe0 }, /* MASK_REGS */ \
-{ 0xffffffff,0xffffffff,0x1fff } \
+{ { 0x00, 0x0, 0x0 }, \
+ { 0x01, 0x0, 0x0 }, /* AREG */ \
+ { 0x02, 0x0, 0x0 }, /* DREG */ \
+ { 0x04, 0x0, 0x0 }, /* CREG */ \
+ { 0x08, 0x0, 0x0 }, /* BREG */ \
+ { 0x10, 0x0, 0x0 }, /* SIREG */ \
+ { 0x20, 0x0, 0x0 }, /* DIREG */ \
+ { 0x03, 0x0, 0x0 }, /* AD_REGS */ \
+ { 0x0f, 0x0, 0x0 }, /* Q_REGS */ \
+ { 0x1100f0, 0x1fe0, 0x0 }, /* NON_Q_REGS */ \
+ { 0x7f, 0x1fe0, 0x0 }, /* INDEX_REGS */ \
+ { 0x1100ff, 0x0, 0x0 }, /* LEGACY_REGS */ \
+ { 0x07, 0x0, 0x0 }, /* CLOBBERED_REGS */ \
+ { 0x1100ff, 0x1fe0, 0x0 }, /* GENERAL_REGS */ \
+ { 0x100, 0x0, 0x0 }, /* FP_TOP_REG */ \
+ { 0x0200, 0x0, 0x0 }, /* FP_SECOND_REG */ \
+ { 0xff00, 0x0, 0x0 }, /* FLOAT_REGS */ \
+ { 0x200000, 0x0, 0x0 }, /* SSE_FIRST_REG */ \
+{ 0x1fe00000, 0x1fe000, 0x0 }, /* SSE_REGS */ \
+ { 0x0,0xffe00000, 0x1f }, /* EVEX_SSE_REGS */ \
+ { 0x0, 0x0,0x1e000 }, /* BND_REGS */ \
+{ 0x1fe00000,0xffffe000, 0x1f }, /* ALL_SSE_REGS */ \
+{ 0xe0000000, 0x1f, 0x0 }, /* MMX_REGS */ \
+{ 0x1fe00100,0xffffe000, 0x1f }, /* FP_TOP_SSE_REG */ \
+{ 0x1fe00200,0xffffe000, 0x1f }, /* FP_SECOND_SSE_REG */ \
+{ 0x1fe0ff00,0xffffe000, 0x1f }, /* FLOAT_SSE_REGS */ \
+{ 0x11ffff, 0x1fe0, 0x0 }, /* FLOAT_INT_REGS */ \
+{ 0x1ff100ff,0xffffffe0, 0x1f }, /* INT_SSE_REGS */ \
+{ 0x1ff1ffff,0xffffffe0, 0x1f }, /* FLOAT_INT_SSE_REGS */ \
+ { 0x0, 0x0, 0x1fc0 }, /* MASK_EVEX_REGS */ \
+ { 0x0, 0x0, 0x1fe0 }, /* MASK_REGS */ \
+{ 0xffffffff,0xffffffff, 0x1fff } \
}
/* The same information, inverted:
@@ -1412,6 +1494,9 @@ enum reg_class
#define CC_REG_P(X) (REG_P (X) && CC_REGNO_P (REGNO (X)))
#define CC_REGNO_P(X) ((X) == FLAGS_REG || (X) == FPSR_REG)
+#define BND_REGNO_P(N) IN_RANGE ((N), FIRST_BND_REG, LAST_BND_REG)
+#define ANY_BND_REG_P(X) (REG_P (X) && BND_REGNO_P (REGNO (X)))
+
/* The class value for index registers, and the one for base regs. */
#define INDEX_REG_CLASS INDEX_REGS
@@ -1492,13 +1577,26 @@ enum reg_class
will be computed and placed into the variable `crtl->outgoing_args_size'.
No space will be pushed onto the stack for each call; instead, the
function prologue should increase the stack frame size by this amount.
+
+ In 32bit mode enabling argument accumulation results in about 5% code size
+ growth becuase move instructions are less compact than push. In 64bit
+ mode the difference is less drastic but visible.
+
+ FIXME: Unlike earlier implementations, the size of unwind info seems to
+ actually grouw with accumulation. Is that because accumulated args
+ unwind info became unnecesarily bloated?
64-bit MS ABI seem to require 16 byte alignment everywhere except for
- function prologue and apilogue. This is not possible without
- ACCUMULATE_OUTGOING_ARGS. */
+ function prologue and epilogue. This is not possible without
+ ACCUMULATE_OUTGOING_ARGS.
+
+ If stack probes are required, the space used for large function
+ arguments on the stack must also be probed, so enable
+ -maccumulate-outgoing-args so this happens in the prologue. */
#define ACCUMULATE_OUTGOING_ARGS \
- (TARGET_ACCUMULATE_OUTGOING_ARGS || TARGET_64BIT_MS_ABI)
+ ((TARGET_ACCUMULATE_OUTGOING_ARGS && optimize_function_for_speed_p (cfun)) \
+ || TARGET_STACK_PROBE || TARGET_64BIT_MS_ABI)
/* If defined, a C expression whose value is nonzero when we want to use PUSH
instructions to pass outgoing arguments. */
@@ -1843,6 +1941,9 @@ do { \
between pointers and any other objects of this machine mode. */
#define Pmode (ix86_pmode == PMODE_DI ? DImode : SImode)
+/* Specify the machine mode that bounds have. */
+#define BNDmode (ix86_pmode == PMODE_DI ? BND64mode : BND32mode)
+
/* A C expression whose value is zero if pointers that need to be extended
from being `POINTER_SIZE' bits wide to `Pmode' are sign-extended and
greater then zero if they are zero-extended and less then zero if the
@@ -1953,7 +2054,8 @@ do { \
"xmm20", "xmm21", "xmm22", "xmm23", \
"xmm24", "xmm25", "xmm26", "xmm27", \
"xmm28", "xmm29", "xmm30", "xmm31", \
- "k0", "k1", "k2", "k3", "k4", "k5", "k6", "k7" }
+ "k0", "k1", "k2", "k3", "k4", "k5", "k6", "k7", \
+ "bnd0", "bnd1", "bnd2", "bnd3" }
#define REGISTER_NAMES HI_REGISTER_NAMES
@@ -2138,6 +2240,7 @@ enum processor_type
PROCESSOR_NOCONA,
PROCESSOR_CORE2,
PROCESSOR_COREI7,
+ PROCESSOR_COREI7_AVX,
PROCESSOR_HASWELL,
PROCESSOR_GENERIC,
PROCESSOR_AMDFAM10,
diff --git a/gcc/config/i386/i386.md b/gcc/config/i386/i386.md
index e009bc96fc2..a37fa64a03e 100644
--- a/gcc/config/i386/i386.md
+++ b/gcc/config/i386/i386.md
@@ -63,6 +63,7 @@
;; ~ -- print "i" if TARGET_AVX2, "f" otherwise.
;; @ -- print a segment register of thread base pointer load
;; ^ -- print addr32 prefix if TARGET_64BIT and Pmode != word_mode
+;; ! -- print MPX prefix for jxx/call/ret instructions if required.
(define_c_enum "unspec" [
;; Relocation specifiers
@@ -178,6 +179,16 @@
;; For BMI2 support
UNSPEC_PDEP
UNSPEC_PEXT
+
+ UNSPEC_BNDMK
+ UNSPEC_BNDMK_ADDR
+ UNSPEC_BNDSTX
+ UNSPEC_BNDLDX
+ UNSPEC_BNDLDX_ADDR
+ UNSPEC_BNDCL
+ UNSPEC_BNDCU
+ UNSPEC_BNDCN
+ UNSPEC_MPX_FENCE
])
(define_c_enum "unspecv" [
@@ -211,6 +222,12 @@
UNSPECV_XSAVEOPT
UNSPECV_XSAVEOPT64
+ ;; For atomic compound assignments.
+ UNSPECV_FNSTENV
+ UNSPECV_FLDENV
+ UNSPECV_FNSTSW
+ UNSPECV_FNCLEX
+
;; For RDRAND support
UNSPECV_RDRAND
@@ -336,6 +353,8 @@
(MASK5_REG 74)
(MASK6_REG 75)
(MASK7_REG 76)
+ (BND0_REG 77)
+ (BND1_REG 78)
])
;; Insns whose names begin with "x86_" are emitted by gen_FOO calls
@@ -369,7 +388,8 @@
ssecvt,ssecvt1,sseicvt,sseins,
sseshuf,sseshuf1,ssemuladd,sse4arg,
lwp,mskmov,msklog,
- mmx,mmxmov,mmxadd,mmxmul,mmxcmp,mmxcvt,mmxshft"
+ mmx,mmxmov,mmxadd,mmxmul,mmxcmp,mmxcvt,mmxshft,
+ mpxmov,mpxmk,mpxchk,mpxld,mpxst"
(const_string "other"))
;; Main data type used by the insn
@@ -398,7 +418,8 @@
;; The (bounding maximum) length of an instruction immediate.
(define_attr "length_immediate" ""
(cond [(eq_attr "type" "incdec,setcc,icmov,str,lea,other,multi,idiv,leave,
- bitmanip,imulx,msklog,mskmov")
+ bitmanip,imulx,msklog,mskmov,mpxmk,mpxmov,mpxchk,
+ mpxld,mpxst")
(const_int 0)
(eq_attr "unit" "i387,sse,mmx")
(const_int 0)
@@ -453,13 +474,17 @@
(const_int 0)
(and (eq_attr "unit" "sse") (eq_attr "mode" "SF,DF"))
(const_int 1)
+ (and (eq_attr "type" "ibr,call,callv")
+ (match_test "ix86_bnd_prefixed_insn_p (insn)"))
+ (const_int 1)
]
(const_int 0)))
;; Set when 0f opcode prefix is used.
(define_attr "prefix_0f" ""
(if_then_else
- (ior (eq_attr "type" "imovx,setcc,icmov,bitmanip,msklog,mskmov")
+ (ior (eq_attr "type" "imovx,setcc,icmov,bitmanip,msklog,mskmov,
+ mpxmk,mpxmov,mpxchk,mpxld,mpxst")
(eq_attr "unit" "sse,mmx"))
(const_int 1)
(const_int 0)))
@@ -562,12 +587,19 @@
]
(const_int 1)))
+;; When this attribute is set, calculate total insn length from
+;; length_nobnd attribute, prefixed with eventual bnd prefix byte
+(define_attr "length_nobnd" "" (const_int 0))
+
;; The (bounding maximum) length of an instruction in bytes.
;; ??? fistp and frndint are in fact fldcw/{fistp,frndint}/fldcw sequences.
;; Later we may want to split them and compute proper length as for
;; other insns.
(define_attr "length" ""
- (cond [(eq_attr "type" "other,multi,fistp,frndint")
+ (cond [(eq_attr "length_nobnd" "!0")
+ (plus (symbol_ref ("ix86_bnd_prefixed_insn_p (insn)"))
+ (attr "length_nobnd"))
+ (eq_attr "type" "other,multi,fistp,frndint")
(const_int 16)
(eq_attr "type" "fcmp")
(const_int 4)
@@ -608,12 +640,16 @@
(define_attr "memory" "none,load,store,both,unknown"
(cond [(eq_attr "type" "other,multi,str,lwp")
(const_string "unknown")
- (eq_attr "type" "lea,fcmov,fpspc")
+ (eq_attr "type" "lea,fcmov,fpspc,mpxmk,mpxchk")
(const_string "none")
(eq_attr "type" "fistp,leave")
(const_string "both")
(eq_attr "type" "frndint")
(const_string "load")
+ (eq_attr "type" "mpxld")
+ (const_string "load")
+ (eq_attr "type" "mpxst")
+ (const_string "store")
(eq_attr "type" "push")
(if_then_else (match_operand 1 "memory_operand")
(const_string "both")
@@ -659,7 +695,7 @@
fmov,fcmp,fsgn,
sse,ssemov,ssecmp,ssecomi,ssecvt,ssecvt1,sseicvt,
sselog1,sseshuf1,sseadd1,sseiadd1,sseishft1,
- mmx,mmxmov,mmxcmp,mmxcvt,mskmov,msklog")
+ mmx,mmxmov,mmxcmp,mmxcvt,mskmov,msklog,mpxmov")
(match_operand 2 "memory_operand"))
(const_string "load")
(and (eq_attr "type" "icmov,ssemuladd,sse4arg")
@@ -746,6 +782,8 @@
(define_code_iterator sat_plusminus [ss_plus us_plus ss_minus us_minus])
+(define_code_iterator multdiv [mult div])
+
;; Base name for define_insn
(define_code_attr plusminus_insn
[(plus "add") (ss_plus "ssadd") (us_plus "usadd")
@@ -757,6 +795,8 @@
(minus "sub") (ss_minus "subs") (us_minus "subus")])
(define_code_attr plusminus_carry_mnemonic
[(plus "adc") (minus "sbb")])
+(define_code_attr multdiv_mnemonic
+ [(mult "mul") (div "div")])
;; Mark commutative operators as such in constraints.
(define_code_attr comm [(plus "%") (ss_plus "%") (us_plus "%")
@@ -779,6 +819,7 @@
;; Mapping of logic operators
(define_code_iterator any_logic [and ior xor])
(define_code_iterator any_or [ior xor])
+(define_code_iterator fpint_logic [and xor])
;; Base name for insn mnemonic.
(define_code_attr logic [(and "and") (ior "or") (xor "xor")])
@@ -826,6 +867,15 @@
(define_code_attr s [(sign_extend "s") (zero_extend "u")])
(define_code_attr u_bool [(sign_extend "false") (zero_extend "true")])
+;; Used in signed and unsigned truncations.
+(define_code_iterator any_truncate [ss_truncate truncate us_truncate])
+;; Instruction suffix for truncations.
+(define_code_attr trunsuffix [(ss_truncate "s") (truncate "") (us_truncate "us")])
+
+;; Used in signed and unsigned fix.
+(define_code_iterator any_fix [fix unsigned_fix])
+(define_code_attr fixsuffix [(fix "") (unsigned_fix "u")])
+
;; All integer modes.
(define_mode_iterator SWI1248x [QI HI SI DI])
@@ -884,6 +934,21 @@
(define_mode_iterator DWIH [(SI "!TARGET_64BIT")
(DI "TARGET_64BIT")])
+;; Bound modes.
+(define_mode_iterator BND [(BND32 "!TARGET_LP64")
+ (BND64 "TARGET_LP64")])
+
+;; Pointer mode corresponding to bound mode.
+(define_mode_attr bnd_ptr [(BND32 "SI") (BND64 "DI")])
+
+;; MPX check types
+(define_int_iterator BNDCHECK [UNSPEC_BNDCL UNSPEC_BNDCU UNSPEC_BNDCN])
+
+;; Check name
+(define_int_attr bndcheck [(UNSPEC_BNDCL "cl")
+ (UNSPEC_BNDCU "cu")
+ (UNSPEC_BNDCN "cn")])
+
;; Instruction suffix for integer modes.
(define_mode_attr imodesuffix [(QI "b") (HI "w") (SI "l") (DI "q")])
@@ -1610,7 +1675,7 @@
split_double_mode (DImode, &operands[1], 1, &operands[2], &operands[3]);
operands[1] = gen_lowpart (DImode, operands[2]);
- operands[2] = gen_rtx_MEM (SImode, gen_rtx_PLUS (DImode, stack_pointer_rtx,
+ operands[2] = gen_rtx_MEM (SImode, gen_rtx_PLUS (Pmode, stack_pointer_rtx,
GEN_INT (4)));
})
@@ -1627,7 +1692,7 @@
split_double_mode (DImode, &operands[1], 1, &operands[2], &operands[3]);
operands[1] = gen_lowpart (DImode, operands[2]);
- operands[2] = gen_rtx_MEM (SImode, gen_rtx_PLUS (DImode, stack_pointer_rtx,
+ operands[2] = gen_rtx_MEM (SImode, gen_rtx_PLUS (Pmode, stack_pointer_rtx,
GEN_INT (4)));
})
@@ -1948,11 +2013,10 @@
return "pxor\t%0, %0";
case TYPE_MMXMOV:
-#ifndef HAVE_AS_IX86_INTERUNIT_MOVQ
/* Handle broken assemblers that require movd instead of movq. */
- if (GENERAL_REG_P (operands[0]) || GENERAL_REG_P (operands[1]))
+ if (!HAVE_AS_IX86_INTERUNIT_MOVQ
+ && (GENERAL_REG_P (operands[0]) || GENERAL_REG_P (operands[1])))
return "movd\t{%1, %0|%0, %1}";
-#endif
return "movq\t{%1, %0|%0, %1}";
case TYPE_SSELOG1:
@@ -1965,11 +2029,10 @@
switch (get_attr_mode (insn))
{
case MODE_DI:
-#ifndef HAVE_AS_IX86_INTERUNIT_MOVQ
/* Handle broken assemblers that require movd instead of movq. */
- if (GENERAL_REG_P (operands[0]) || GENERAL_REG_P (operands[1]))
+ if (!HAVE_AS_IX86_INTERUNIT_MOVQ
+ && (GENERAL_REG_P (operands[0]) || GENERAL_REG_P (operands[1])))
return "%vmovd\t{%1, %0|%0, %1}";
-#endif
return "%vmovq\t{%1, %0|%0, %1}";
case MODE_TI:
return "%vmovdqa\t{%1, %0|%0, %1}";
@@ -2293,9 +2356,12 @@
;; For loads of Q_REG to NONQ_REG we use full sized moves except for partial
;; register stall machines with, where we use QImode instructions, since
;; partial register stall can be caused there. Then we use movzx.
+
(define_insn "*movqi_internal"
- [(set (match_operand:QI 0 "nonimmediate_operand" "=q,q ,q ,r,r ,?r,m ,Yk,Yk,r")
- (match_operand:QI 1 "general_operand" "q ,qn,qm,q,rn,qm,qn,r ,Yk,Yk"))]
+ [(set (match_operand:QI 0 "nonimmediate_operand"
+ "=q,q ,q ,r,r ,?r,m ,Yk,Yk,r")
+ (match_operand:QI 1 "general_operand"
+ "q ,qn,qm,q,rn,qm,qn,r ,Yk,Yk"))]
"!(MEM_P (operands[0]) && MEM_P (operands[1]))"
{
switch (get_attr_type (insn))
@@ -2882,12 +2948,11 @@
return "movlpd\t{%1, %0|%0, %1}";
case MODE_DI:
-#ifndef HAVE_AS_IX86_INTERUNIT_MOVQ
/* Handle broken assemblers that require movd instead of movq. */
- if (GENERAL_REG_P (operands[0]) || GENERAL_REG_P (operands[1]))
+ if (!HAVE_AS_IX86_INTERUNIT_MOVQ
+ && (GENERAL_REG_P (operands[0]) || GENERAL_REG_P (operands[1])))
return "%vmovd\t{%1, %0|%0, %1}";
-#endif
- return "%vmovq\t{%1, %0|%0, %1}";
+ return "%vmovq\t{%1, %0|%0, %1}";
default:
gcc_unreachable ();
@@ -5116,6 +5181,61 @@
emit_move_insn (operands[0], CONST0_RTX (<ssevecmode>mode));
})
+;; Break partial reg stall for cvtsd2ss.
+
+(define_peephole2
+ [(set (match_operand:SF 0 "register_operand")
+ (float_truncate:SF
+ (match_operand:DF 1 "nonimmediate_operand")))]
+ "TARGET_SSE2 && TARGET_SSE_MATH
+ && TARGET_SSE_PARTIAL_REG_DEPENDENCY
+ && optimize_function_for_speed_p (cfun)
+ && SSE_REG_P (operands[0])
+ && (!SSE_REG_P (operands[1])
+ || REGNO (operands[0]) != REGNO (operands[1]))"
+ [(set (match_dup 0)
+ (vec_merge:V4SF
+ (vec_duplicate:V4SF
+ (float_truncate:V2SF
+ (match_dup 1)))
+ (match_dup 0)
+ (const_int 1)))]
+{
+ operands[0] = simplify_gen_subreg (V4SFmode, operands[0],
+ SFmode, 0);
+ operands[1] = simplify_gen_subreg (V2DFmode, operands[1],
+ DFmode, 0);
+ emit_move_insn (operands[0], CONST0_RTX (V4SFmode));
+})
+
+;; Break partial reg stall for cvtss2sd.
+
+(define_peephole2
+ [(set (match_operand:DF 0 "register_operand")
+ (float_extend:DF
+ (match_operand:SF 1 "nonimmediate_operand")))]
+ "TARGET_SSE2 && TARGET_SSE_MATH
+ && TARGET_SSE_PARTIAL_REG_DEPENDENCY
+ && optimize_function_for_speed_p (cfun)
+ && SSE_REG_P (operands[0])
+ && (!SSE_REG_P (operands[1])
+ || REGNO (operands[0]) != REGNO (operands[1]))"
+ [(set (match_dup 0)
+ (vec_merge:V2DF
+ (float_extend:V2DF
+ (vec_select:V2SF
+ (match_dup 1)
+ (parallel [(const_int 0) (const_int 1)])))
+ (match_dup 0)
+ (const_int 1)))]
+{
+ operands[0] = simplify_gen_subreg (V2DFmode, operands[0],
+ DFmode, 0);
+ operands[1] = simplify_gen_subreg (V4SFmode, operands[1],
+ SFmode, 0);
+ emit_move_insn (operands[0], CONST0_RTX (V2DFmode));
+})
+
;; Avoid store forwarding (partial memory) stall penalty
;; by passing DImode value through XMM registers. */
@@ -5280,7 +5400,7 @@
(define_insn_and_split "*lea<mode>"
[(set (match_operand:SWI48 0 "register_operand" "=r")
- (match_operand:SWI48 1 "lea_address_operand" "p"))]
+ (match_operand:SWI48 1 "address_no_seg_operand" "Ts"))]
""
{
if (SImode_address_operand (operands[1], VOIDmode))
@@ -6422,7 +6542,7 @@
(set_attr "use_carry" "1")
(set_attr "mode" "<MODE>")])
-;; Overflow setting add and subtract instructions
+;; Overflow setting add instructions
(define_insn "*add<mode>3_cconly_overflow"
[(set (reg:CCC FLAGS_REG)
@@ -6437,43 +6557,31 @@
[(set_attr "type" "alu")
(set_attr "mode" "<MODE>")])
-(define_insn "*sub<mode>3_cconly_overflow"
- [(set (reg:CCC FLAGS_REG)
- (compare:CCC
- (minus:SWI
- (match_operand:SWI 0 "nonimmediate_operand" "<r>m,<r>")
- (match_operand:SWI 1 "<general_operand>" "<r><i>,<r>m"))
- (match_dup 0)))]
- ""
- "cmp{<imodesuffix>}\t{%1, %0|%0, %1}"
- [(set_attr "type" "icmp")
- (set_attr "mode" "<MODE>")])
-
-(define_insn "*<plusminus_insn><mode>3_cc_overflow"
+(define_insn "*add<mode>3_cc_overflow"
[(set (reg:CCC FLAGS_REG)
(compare:CCC
- (plusminus:SWI
- (match_operand:SWI 1 "nonimmediate_operand" "<comm>0,0")
+ (plus:SWI
+ (match_operand:SWI 1 "nonimmediate_operand" "%0,0")
(match_operand:SWI 2 "<general_operand>" "<r><i>,<r>m"))
(match_dup 1)))
(set (match_operand:SWI 0 "nonimmediate_operand" "=<r>m,<r>")
- (plusminus:SWI (match_dup 1) (match_dup 2)))]
- "ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)"
- "<plusminus_mnemonic>{<imodesuffix>}\t{%2, %0|%0, %2}"
+ (plus:SWI (match_dup 1) (match_dup 2)))]
+ "ix86_binary_operator_ok (PLUS, <MODE>mode, operands)"
+ "add{<imodesuffix>}\t{%2, %0|%0, %2}"
[(set_attr "type" "alu")
(set_attr "mode" "<MODE>")])
-(define_insn "*<plusminus_insn>si3_zext_cc_overflow"
+(define_insn "*addsi3_zext_cc_overflow"
[(set (reg:CCC FLAGS_REG)
(compare:CCC
- (plusminus:SI
- (match_operand:SI 1 "nonimmediate_operand" "<comm>0")
+ (plus:SI
+ (match_operand:SI 1 "nonimmediate_operand" "%0")
(match_operand:SI 2 "x86_64_general_operand" "rme"))
(match_dup 1)))
(set (match_operand:DI 0 "register_operand" "=r")
- (zero_extend:DI (plusminus:SI (match_dup 1) (match_dup 2))))]
- "TARGET_64BIT && ix86_binary_operator_ok (<CODE>, SImode, operands)"
- "<plusminus_mnemonic>{l}\t{%2, %k0|%k0, %2}"
+ (zero_extend:DI (plus:SI (match_dup 1) (match_dup 2))))]
+ "TARGET_64BIT && ix86_binary_operator_ok (PLUS, SImode, operands)"
+ "add{l}\t{%2, %k0|%k0, %2}"
[(set_attr "type" "alu")
(set_attr "mode" "SI")])
@@ -7764,8 +7872,7 @@
(parallel [(set (match_dup 0)
(and:HI (match_dup 0)
(match_dup 1)))
- (clobber (reg:CC FLAGS_REG))])]
- "")
+ (clobber (reg:CC FLAGS_REG))])])
;; Turn *anddi_1 into *andsi_1_zext if possible.
(define_split
@@ -8190,7 +8297,8 @@
(not:SWI12
(xor:SWI12
(match_operand:SWI12 1 "register_operand" "0,Yk")
- (match_operand:SWI12 2 "register_operand" "r,Yk"))))]
+ (match_operand:SWI12 2 "register_operand" "r,Yk"))))
+ (clobber (reg:CC FLAGS_REG))]
"TARGET_AVX512F"
"@
#
@@ -8204,15 +8312,15 @@
(not:SWI12
(xor:SWI12
(match_dup 0)
- (match_operand:SWI12 1 "general_reg_operand"))))]
+ (match_operand:SWI12 1 "general_reg_operand"))))
+ (clobber (reg:CC FLAGS_REG))]
"TARGET_AVX512F && reload_completed"
[(parallel [(set (match_dup 0)
(xor:HI (match_dup 0)
(match_dup 1)))
(clobber (reg:CC FLAGS_REG))])
(set (match_dup 0)
- (not:HI (match_dup 0)))]
- "")
+ (not:HI (match_dup 0)))])
(define_insn "kortestzhi"
[(set (reg:CCZ FLAGS_REG)
@@ -10679,10 +10787,10 @@
(label_ref (match_operand 0))
(pc)))]
""
- "%+j%C1\t%l0"
+ "%!%+j%C1\t%l0"
[(set_attr "type" "ibr")
(set_attr "modrm" "0")
- (set (attr "length")
+ (set (attr "length_nobnd")
(if_then_else (and (ge (minus (match_dup 0) (pc))
(const_int -126))
(lt (minus (match_dup 0) (pc))
@@ -10697,10 +10805,10 @@
(pc)
(label_ref (match_operand 0))))]
""
- "%+j%c1\t%l0"
+ "%!%+j%c1\t%l0"
[(set_attr "type" "ibr")
(set_attr "modrm" "0")
- (set (attr "length")
+ (set (attr "length_nobnd")
(if_then_else (and (ge (minus (match_dup 0) (pc))
(const_int -126))
(lt (minus (match_dup 0) (pc))
@@ -11163,9 +11271,9 @@
[(set (pc)
(label_ref (match_operand 0)))]
""
- "jmp\t%l0"
+ "%!jmp\t%l0"
[(set_attr "type" "ibr")
- (set (attr "length")
+ (set (attr "length_nobnd")
(if_then_else (and (ge (minus (match_dup 0) (pc))
(const_int -126))
(lt (minus (match_dup 0) (pc))
@@ -11185,7 +11293,7 @@
(define_insn "*indirect_jump"
[(set (pc) (match_operand:W 0 "indirect_branch_operand" "rw"))]
""
- "jmp\t%A0"
+ "%!jmp\t%A0"
[(set_attr "type" "ibr")
(set_attr "length_immediate" "0")])
@@ -11234,7 +11342,7 @@
[(set (pc) (match_operand:W 0 "indirect_branch_operand" "rw"))
(use (label_ref (match_operand 1)))]
""
- "jmp\t%A0"
+ "%!jmp\t%A0"
[(set_attr "type" "ibr")
(set_attr "length_immediate" "0")])
@@ -11621,8 +11729,8 @@
(define_insn "simple_return_internal"
[(simple_return)]
"reload_completed"
- "ret"
- [(set_attr "length" "1")
+ "%!ret"
+ [(set_attr "length_nobnd" "1")
(set_attr "atom_unit" "jeu")
(set_attr "length_immediate" "0")
(set_attr "modrm" "0")])
@@ -11634,7 +11742,12 @@
[(simple_return)
(unspec [(const_int 0)] UNSPEC_REP)]
"reload_completed"
- "rep%; ret"
+{
+ if (ix86_bnd_prefixed_insn_p (insn))
+ return "%!ret";
+
+ return "rep%; ret";
+}
[(set_attr "length" "2")
(set_attr "atom_unit" "jeu")
(set_attr "length_immediate" "0")
@@ -11645,8 +11758,8 @@
[(simple_return)
(use (match_operand:SI 0 "const_int_operand"))]
"reload_completed"
- "ret\t%0"
- [(set_attr "length" "3")
+ "%!ret\t%0"
+ [(set_attr "length_nobnd" "3")
(set_attr "atom_unit" "jeu")
(set_attr "length_immediate" "2")
(set_attr "modrm" "0")])
@@ -11655,7 +11768,7 @@
[(simple_return)
(use (match_operand:SI 0 "register_operand" "r"))]
"reload_completed"
- "jmp\t%A0"
+ "%!jmp\t%A0"
[(set_attr "type" "ibr")
(set_attr "length_immediate" "0")])
@@ -17517,7 +17630,7 @@
[(prefetch (match_operand 0 "address_operand")
(match_operand:SI 1 "const_int_operand")
(match_operand:SI 2 "const_int_operand"))]
- "TARGET_PREFETCH_SSE || TARGET_PRFCHW"
+ "TARGET_PREFETCH_SSE || TARGET_PRFCHW || TARGET_AVX512PF"
{
bool write = INTVAL (operands[1]) != 0;
int locality = INTVAL (operands[2]);
@@ -17528,7 +17641,9 @@
supported by SSE counterpart or the SSE prefetch is not available
(K6 machines). Otherwise use SSE prefetch as it allows specifying
of locality. */
- if (TARGET_PRFCHW && (write || !TARGET_PREFETCH_SSE))
+ if (TARGET_AVX512PF && write)
+ operands[2] = const1_rtx;
+ else if (TARGET_PRFCHW && (write || !TARGET_PREFETCH_SSE))
operands[2] = GEN_INT (3);
else
operands[1] = const0_rtx;
@@ -17571,6 +17686,18 @@
(symbol_ref "memory_address_length (operands[0], false)"))
(set_attr "memory" "none")])
+(define_insn "*prefetch_avx512pf_<mode>"
+ [(prefetch (match_operand:P 0 "address_operand" "p")
+ (const_int 1)
+ (const_int 1))]
+ "TARGET_AVX512PF"
+ "prefetchwt1\t%a0";
+ [(set_attr "type" "sse")
+ (set_attr "prefix" "evex")
+ (set (attr "length_address")
+ (symbol_ref "memory_address_length (operands[0], false)"))
+ (set_attr "memory" "none")])
+
(define_expand "stack_protect_set"
[(match_operand 0 "memory_operand")
(match_operand 1 "memory_operand")]
@@ -17893,6 +18020,71 @@
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;
+;; Floating-point instructions for atomic compound assignments
+;;
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+; Clobber all floating-point registers on environment save and restore
+; to ensure that the TOS value saved at fnstenv is valid after fldenv.
+(define_insn "fnstenv"
+ [(set (match_operand:BLK 0 "memory_operand" "=m")
+ (unspec_volatile:BLK [(const_int 0)] UNSPECV_FNSTENV))
+ (clobber (reg:HI FPCR_REG))
+ (clobber (reg:XF ST0_REG))
+ (clobber (reg:XF ST1_REG))
+ (clobber (reg:XF ST2_REG))
+ (clobber (reg:XF ST3_REG))
+ (clobber (reg:XF ST4_REG))
+ (clobber (reg:XF ST5_REG))
+ (clobber (reg:XF ST6_REG))
+ (clobber (reg:XF ST7_REG))]
+ "TARGET_80387"
+ "fnstenv\t%0"
+ [(set_attr "type" "other")
+ (set_attr "memory" "store")
+ (set (attr "length")
+ (symbol_ref "ix86_attr_length_address_default (insn) + 2"))])
+
+(define_insn "fldenv"
+ [(unspec_volatile [(match_operand:BLK 0 "memory_operand" "m")]
+ UNSPECV_FLDENV)
+ (clobber (reg:CCFP FPSR_REG))
+ (clobber (reg:HI FPCR_REG))
+ (clobber (reg:XF ST0_REG))
+ (clobber (reg:XF ST1_REG))
+ (clobber (reg:XF ST2_REG))
+ (clobber (reg:XF ST3_REG))
+ (clobber (reg:XF ST4_REG))
+ (clobber (reg:XF ST5_REG))
+ (clobber (reg:XF ST6_REG))
+ (clobber (reg:XF ST7_REG))]
+ "TARGET_80387"
+ "fldenv\t%0"
+ [(set_attr "type" "other")
+ (set_attr "memory" "load")
+ (set (attr "length")
+ (symbol_ref "ix86_attr_length_address_default (insn) + 2"))])
+
+(define_insn "fnstsw"
+ [(set (match_operand:HI 0 "memory_operand" "=m")
+ (unspec_volatile:HI [(const_int 0)] UNSPECV_FNSTSW))]
+ "TARGET_80387"
+ "fnstsw\t%0"
+ [(set_attr "type" "other")
+ (set_attr "memory" "store")
+ (set (attr "length")
+ (symbol_ref "ix86_attr_length_address_default (insn) + 2"))])
+
+(define_insn "fnclex"
+ [(unspec_volatile [(const_int 0)] UNSPECV_FNCLEX)]
+ "TARGET_80387"
+ "fnclex"
+ [(set_attr "type" "other")
+ (set_attr "memory" "none")
+ (set_attr "length" "2")])
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;;
;; LWP instructions
;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -18118,6 +18310,131 @@
[(set_attr "type" "other")
(set_attr "length" "3")])
+;; MPX instructions
+
+(define_expand "<mode>_mk"
+ [(set (match_operand:BND 0 "register_operand")
+ (unspec:BND
+ [(mem:<bnd_ptr>
+ (match_par_dup 3
+ [(match_operand:<bnd_ptr> 1 "register_operand")
+ (match_operand:<bnd_ptr> 2 "address_mpx_no_base_operand")]))]
+ UNSPEC_BNDMK))]
+ "TARGET_MPX"
+{
+ operands[3] = gen_rtx_UNSPEC (Pmode, gen_rtvec (2, operands[1],
+ operands[2]),
+ UNSPEC_BNDMK_ADDR);
+})
+
+(define_insn "*<mode>_mk"
+ [(set (match_operand:BND 0 "register_operand" "=B")
+ (unspec:BND
+ [(match_operator:<bnd_ptr> 3 "bnd_mem_operator"
+ [(unspec:<bnd_ptr>
+ [(match_operand:<bnd_ptr> 1 "register_operand" "r")
+ (match_operand:<bnd_ptr> 2 "address_mpx_no_base_operand" "Tb")]
+ UNSPEC_BNDMK_ADDR)])]
+ UNSPEC_BNDMK))]
+ "TARGET_MPX"
+ "bndmk\t{%3, %0|%0, %3}"
+ [(set_attr "type" "mpxmk")])
+
+(define_expand "mov<mode>"
+ [(set (match_operand:BND 0 "general_operand")
+ (match_operand:BND 1 "general_operand"))]
+ "TARGET_MPX"
+{
+ ix86_expand_move (<MODE>mode, operands);DONE;
+})
+
+(define_insn "*mov<mode>_internal_mpx"
+ [(set (match_operand:BND 0 "nonimmediate_operand" "=B,m")
+ (match_operand:BND 1 "general_operand" "Bm,B"))]
+ "TARGET_MPX"
+ "bndmov\t{%1, %0|%0, %1}"
+ [(set_attr "type" "mpxmov")])
+
+(define_expand "<mode>_<bndcheck>"
+ [(parallel [(unspec [(match_operand:BND 0 "register_operand")
+ (match_operand:<bnd_ptr> 1 "address_no_seg_operand")] BNDCHECK)
+ (set (match_dup 2)
+ (unspec:BLK [(match_dup 2)] UNSPEC_MPX_FENCE))])]
+ "TARGET_MPX"
+{
+ operands[2] = gen_rtx_MEM (BLKmode, operands[1]);
+ MEM_VOLATILE_P (operands[2]) = 1;
+})
+
+(define_insn "*<mode>_<bndcheck>"
+ [(parallel [(unspec [(match_operand:BND 0 "register_operand" "B")
+ (match_operand:<bnd_ptr> 1 "address_no_seg_operand" "Ts")] BNDCHECK)
+ (set (match_operand:BLK 2 "bnd_mem_operator")
+ (unspec:BLK [(match_dup 2)] UNSPEC_MPX_FENCE))])]
+ "TARGET_MPX"
+ "bnd<bndcheck>\t{%a1, %0|%0, %a1}"
+ [(set_attr "type" "mpxchk")])
+
+(define_expand "<mode>_ldx"
+ [(parallel [(set:BND (match_operand:BND 0 "register_operand")
+ (unspec:BND
+ [(mem:<bnd_ptr>
+ (match_par_dup 3
+ [(match_operand:<bnd_ptr> 1 "address_mpx_no_index_operand")
+ (match_operand:<bnd_ptr> 2 "register_operand")]))]
+ UNSPEC_BNDLDX))
+ (use (mem:BLK (match_dup 1)))])]
+ "TARGET_MPX"
+{
+ operands[3] = gen_rtx_UNSPEC (Pmode, gen_rtvec (2, operands[1],
+ operands[2]),
+ UNSPEC_BNDLDX_ADDR);
+})
+
+(define_insn "*<mode>_ldx"
+ [(parallel [(set:BND (match_operand:BND 0 "register_operand" "=B")
+ (unspec:BND
+ [(match_operator:<bnd_ptr> 3 "bnd_mem_operator"
+ [(unspec:<bnd_ptr>
+ [(match_operand:<bnd_ptr> 1 "address_mpx_no_index_operand" "Ti")
+ (match_operand:<bnd_ptr> 2 "register_operand" "l")]
+ UNSPEC_BNDLDX_ADDR)])]
+ UNSPEC_BNDLDX))
+ (use (mem:BLK (match_dup 1)))])]
+ "TARGET_MPX"
+ "bndldx\t{%3, %0|%0, %3}"
+ [(set_attr "type" "mpxld")])
+
+(define_expand "<mode>_stx"
+ [(parallel [(unspec [(mem:<bnd_ptr>
+ (match_par_dup 3
+ [(match_operand:<bnd_ptr> 0 "address_mpx_no_index_operand")
+ (match_operand:<bnd_ptr> 1 "register_operand")]))
+ (match_operand:BND 2 "register_operand")] UNSPEC_BNDSTX)
+ (set (match_dup 4)
+ (unspec:BLK [(match_dup 4)] UNSPEC_MPX_FENCE))])]
+ "TARGET_MPX"
+{
+ operands[3] = gen_rtx_UNSPEC (Pmode, gen_rtvec (2, operands[0],
+ operands[1]),
+ UNSPEC_BNDLDX_ADDR);
+ operands[4] = gen_rtx_MEM (BLKmode, operands[0]);
+ MEM_VOLATILE_P (operands[4]) = 1;
+})
+
+(define_insn "*<mode>_stx"
+ [(parallel [(unspec [(match_operator:<bnd_ptr> 3 "bnd_mem_operator"
+ [(unspec:<bnd_ptr>
+ [(match_operand:<bnd_ptr> 0 "address_mpx_no_index_operand" "Ti")
+ (match_operand:<bnd_ptr> 1 "register_operand" "l")]
+ UNSPEC_BNDLDX_ADDR)])
+ (match_operand:BND 2 "register_operand" "B")] UNSPEC_BNDSTX)
+ (set (match_operand:BLK 4 "bnd_mem_operator")
+ (unspec:BLK [(match_dup 4)] UNSPEC_MPX_FENCE))])]
+ "TARGET_MPX"
+ "bndstx\t{%2, %3|%3, %2}"
+ [(set_attr "type" "mpxst")])
+
(include "mmx.md")
(include "sse.md")
(include "sync.md")
diff --git a/gcc/config/i386/i386.opt b/gcc/config/i386/i386.opt
index 5495c295f57..1a1b8abefdc 100644
--- a/gcc/config/i386/i386.opt
+++ b/gcc/config/i386/i386.opt
@@ -61,9 +61,13 @@ TargetSave
HOST_WIDE_INT x_ix86_isa_flags_explicit
;; which flags were passed by the user
-TargetSave
+Variable
int ix86_target_flags_explicit
+;; which flags were passed by the user
+TargetSave
+HOST_WIDE_INT x_ix86_target_flags_explicit
+
;; whether -mtune was not specified
TargetSave
unsigned char tune_defaulted
@@ -665,6 +669,10 @@ mrtm
Target Report Mask(ISA_RTM) Var(ix86_isa_flags) Save
Support RTM built-in functions and code generation
+mmpx
+Target Report Mask(ISA_MPX) Var(ix86_isa_flags) Save
+Support MPX code generation
+
mstack-protector-guard=
Target RejectNegative Joined Enum(stack_protector_guard) Var(ix86_stack_protector_guard) Init(SSP_TLS)
Use given stack-protector guard
diff --git a/gcc/config/i386/mmx.md b/gcc/config/i386/mmx.md
index 17e24999258..cc0db3a9d06 100644
--- a/gcc/config/i386/mmx.md
+++ b/gcc/config/i386/mmx.md
@@ -99,11 +99,10 @@
return "pxor\t%0, %0";
case TYPE_MMXMOV:
-#ifndef HAVE_AS_IX86_INTERUNIT_MOVQ
/* Handle broken assemblers that require movd instead of movq. */
- if (GENERAL_REG_P (operands[0]) || GENERAL_REG_P (operands[1]))
+ if (!HAVE_AS_IX86_INTERUNIT_MOVQ
+ && (GENERAL_REG_P (operands[0]) || GENERAL_REG_P (operands[1])))
return "movd\t{%1, %0|%0, %1}";
-#endif
return "movq\t{%1, %0|%0, %1}";
case TYPE_SSECVT:
@@ -119,15 +118,13 @@
switch (get_attr_mode (insn))
{
case MODE_DI:
-#ifndef HAVE_AS_IX86_INTERUNIT_MOVQ
/* Handle broken assemblers that require movd instead of movq. */
- if (GENERAL_REG_P (operands[0]) || GENERAL_REG_P (operands[1]))
+ if (!HAVE_AS_IX86_INTERUNIT_MOVQ
+ && (GENERAL_REG_P (operands[0]) || GENERAL_REG_P (operands[1])))
return "%vmovd\t{%1, %0|%0, %1}";
-#endif
return "%vmovq\t{%1, %0|%0, %1}";
case MODE_TI:
return "%vmovdqa\t{%1, %0|%0, %1}";
-
case MODE_XI:
return "vmovdqa64\t{%g1, %g0|%g0, %g1}";
diff --git a/gcc/config/i386/predicates.md b/gcc/config/i386/predicates.md
index 18f425c4b87..e5dd90cfad2 100644
--- a/gcc/config/i386/predicates.md
+++ b/gcc/config/i386/predicates.md
@@ -752,16 +752,66 @@
(and (match_code "const_int")
(match_test "IN_RANGE (INTVAL (op), 6, 7)")))
+;; Match 8 to 9.
+(define_predicate "const_8_to_9_operand"
+ (and (match_code "const_int")
+ (match_test "IN_RANGE (INTVAL (op), 8, 9)")))
+
;; Match 8 to 11.
(define_predicate "const_8_to_11_operand"
(and (match_code "const_int")
(match_test "IN_RANGE (INTVAL (op), 8, 11)")))
+;; Match 8 to 15.
+(define_predicate "const_8_to_15_operand"
+ (and (match_code "const_int")
+ (match_test "IN_RANGE (INTVAL (op), 8, 15)")))
+
+;; Match 10 to 11.
+(define_predicate "const_10_to_11_operand"
+ (and (match_code "const_int")
+ (match_test "IN_RANGE (INTVAL (op), 10, 11)")))
+
+;; Match 12 to 13.
+(define_predicate "const_12_to_13_operand"
+ (and (match_code "const_int")
+ (match_test "IN_RANGE (INTVAL (op), 12, 13)")))
+
;; Match 12 to 15.
(define_predicate "const_12_to_15_operand"
(and (match_code "const_int")
(match_test "IN_RANGE (INTVAL (op), 12, 15)")))
+;; Match 14 to 15.
+(define_predicate "const_14_to_15_operand"
+ (and (match_code "const_int")
+ (match_test "IN_RANGE (INTVAL (op), 14, 15)")))
+
+;; Match 16 to 19.
+(define_predicate "const_16_to_19_operand"
+ (and (match_code "const_int")
+ (match_test "IN_RANGE (INTVAL (op), 16, 19)")))
+
+;; Match 16 to 31.
+(define_predicate "const_16_to_31_operand"
+ (and (match_code "const_int")
+ (match_test "IN_RANGE (INTVAL (op), 16, 31)")))
+
+;; Match 20 to 23.
+(define_predicate "const_20_to_23_operand"
+ (and (match_code "const_int")
+ (match_test "IN_RANGE (INTVAL (op), 20, 23)")))
+
+;; Match 24 to 27.
+(define_predicate "const_24_to_27_operand"
+ (and (match_code "const_int")
+ (match_test "IN_RANGE (INTVAL (op), 24, 27)")))
+
+;; Match 28 to 31.
+(define_predicate "const_28_to_31_operand"
+ (and (match_code "const_int")
+ (match_test "IN_RANGE (INTVAL (op), 28, 31)")))
+
;; True if this is a constant appropriate for an increment or decrement.
(define_predicate "incdec_operand"
(match_code "const_int")
@@ -862,10 +912,14 @@
(ior (match_operand 0 "register_operand")
(match_operand 0 "const0_operand")))
+;; Return true for RTX codes that force SImode address.
+(define_predicate "SImode_address_operand"
+ (match_code "subreg,zero_extend,and"))
+
;; Return true if op if a valid address for LEA, and does not contain
;; a segment override. Defined as a special predicate to allow
;; mode-less const_int operands pass to address_operand.
-(define_special_predicate "lea_address_operand"
+(define_special_predicate "address_no_seg_operand"
(match_operand 0 "address_operand")
{
struct ix86_address parts;
@@ -876,10 +930,6 @@
return parts.seg == SEG_DEFAULT;
})
-;; Return true for RTX codes that force SImode address.
-(define_predicate "SImode_address_operand"
- (match_code "subreg,zero_extend,and"))
-
;; Return true if op if a valid base register, displacement or
;; sum of base register and displacement for VSIB addressing.
(define_predicate "vsib_address_operand"
@@ -922,9 +972,74 @@
return true;
})
+;; Return true if op is valid MPX address operand without base
+(define_predicate "address_mpx_no_base_operand"
+ (match_operand 0 "address_operand")
+{
+ struct ix86_address parts;
+ int ok;
+
+ ok = ix86_decompose_address (op, &parts);
+ gcc_assert (ok);
+
+ if (parts.index && parts.base)
+ return false;
+
+ if (parts.seg != SEG_DEFAULT)
+ return false;
+
+ /* Do not support (%rip). */
+ if (parts.disp && flag_pic && TARGET_64BIT
+ && SYMBOLIC_CONST (parts.disp))
+ {
+ if (GET_CODE (parts.disp) != CONST
+ || GET_CODE (XEXP (parts.disp, 0)) != PLUS
+ || GET_CODE (XEXP (XEXP (parts.disp, 0), 0)) != UNSPEC
+ || !CONST_INT_P (XEXP (XEXP (parts.disp, 0), 1))
+ || (XINT (XEXP (XEXP (parts.disp, 0), 0), 1) != UNSPEC_DTPOFF
+ && XINT (XEXP (XEXP (parts.disp, 0), 0), 1) != UNSPEC_NTPOFF))
+ return false;
+ }
+
+ return true;
+})
+
+;; Return true if op is valid MPX address operand without index
+(define_predicate "address_mpx_no_index_operand"
+ (match_operand 0 "address_operand")
+{
+ struct ix86_address parts;
+ int ok;
+
+ ok = ix86_decompose_address (op, &parts);
+ gcc_assert (ok);
+
+ if (parts.index)
+ return false;
+
+ if (parts.seg != SEG_DEFAULT)
+ return false;
+
+ /* Do not support (%rip). */
+ if (parts.disp && flag_pic && TARGET_64BIT
+ && SYMBOLIC_CONST (parts.disp)
+ && (GET_CODE (parts.disp) != CONST
+ || GET_CODE (XEXP (parts.disp, 0)) != PLUS
+ || GET_CODE (XEXP (XEXP (parts.disp, 0), 0)) != UNSPEC
+ || !CONST_INT_P (XEXP (XEXP (parts.disp, 0), 1))
+ || (XINT (XEXP (XEXP (parts.disp, 0), 0), 1) != UNSPEC_DTPOFF
+ && XINT (XEXP (XEXP (parts.disp, 0), 0), 1) != UNSPEC_NTPOFF)))
+ return false;
+
+ return true;
+})
+
(define_predicate "vsib_mem_operator"
(match_code "mem"))
+(define_predicate "bnd_mem_operator"
+ (match_code "mem"))
+
;; Return true if the rtx is known to be at least 32 bits aligned.
(define_predicate "aligned_operand"
(match_operand 0 "general_operand")
@@ -1332,3 +1447,9 @@
(define_predicate "general_vector_operand"
(ior (match_operand 0 "nonimmediate_operand")
(match_code "const_vector")))
+
+;; Return true if OP is either -1 constant or stored in register.
+(define_predicate "register_or_constm1_operand"
+ (ior (match_operand 0 "register_operand")
+ (and (match_code "const_int")
+ (match_test "op == constm1_rtx"))))
diff --git a/gcc/config/i386/rtemself.h b/gcc/config/i386/rtemself.h
index 08ef18ccec5..087179191cb 100644
--- a/gcc/config/i386/rtemself.h
+++ b/gcc/config/i386/rtemself.h
@@ -26,7 +26,15 @@ along with GCC; see the file COPYING3. If not see
builtin_define ("__rtems__"); \
builtin_define ("__USE_INIT_FINI__"); \
builtin_assert ("system=rtems"); \
- if (!TARGET_80387) \
- builtin_define ("_SOFT_FLOAT"); \
} \
while (0)
+
+#undef LONG_DOUBLE_TYPE_SIZE
+#define LONG_DOUBLE_TYPE_SIZE (TARGET_80387 ? 80 : 64)
+
+#undef LIBGCC2_LONG_DOUBLE_TYPE_SIZE
+#ifdef _SOFT_FLOAT
+#define LIBGCC2_LONG_DOUBLE_TYPE_SIZE 64
+#else
+#define LIBGCC2_LONG_DOUBLE_TYPE_SIZE 80
+#endif
diff --git a/gcc/config/i386/sse.md b/gcc/config/i386/sse.md
index 9d9469e2c62..7bb2d7795f6 100644
--- a/gcc/config/i386/sse.md
+++ b/gcc/config/i386/sse.md
@@ -83,6 +83,35 @@
UNSPEC_VPERMTI
UNSPEC_GATHER
UNSPEC_VSIBADDR
+
+ ;; For AVX512F support
+ UNSPEC_VPERMI2
+ UNSPEC_VPERMT2
+ UNSPEC_UNSIGNED_FIX_NOTRUNC
+ UNSPEC_UNSIGNED_PCMP
+ UNSPEC_TESTM
+ UNSPEC_TESTNM
+ UNSPEC_SCATTER
+ UNSPEC_RCP14
+ UNSPEC_RSQRT14
+ UNSPEC_FIXUPIMM
+ UNSPEC_SCALEF
+ UNSPEC_VTERNLOG
+ UNSPEC_GETEXP
+ UNSPEC_GETMANT
+ UNSPEC_ALIGN
+ UNSPEC_CONFLICT
+ UNSPEC_MASKED_EQ
+ UNSPEC_MASKED_GT
+
+ ;; For AVX512PF support
+ UNSPEC_GATHER_PREFETCH
+ UNSPEC_SCATTER_PREFETCH
+
+ ;; For AVX512ER support
+ UNSPEC_EXP2
+ UNSPEC_RCP28
+ UNSPEC_RSQRT28
])
(define_c_enum "unspecv" [
@@ -97,22 +126,22 @@
;; All vector modes including V?TImode, used in move patterns.
(define_mode_iterator VMOVE
- [(V32QI "TARGET_AVX") V16QI
- (V16HI "TARGET_AVX") V8HI
- (V8SI "TARGET_AVX") V4SI
- (V4DI "TARGET_AVX") V2DI
+ [(V64QI "TARGET_AVX512F") (V32QI "TARGET_AVX") V16QI
+ (V32HI "TARGET_AVX512F") (V16HI "TARGET_AVX") V8HI
+ (V16SI "TARGET_AVX512F") (V8SI "TARGET_AVX") V4SI
+ (V8DI "TARGET_AVX512F") (V4DI "TARGET_AVX") V2DI
(V2TI "TARGET_AVX") V1TI
- (V8SF "TARGET_AVX") V4SF
- (V4DF "TARGET_AVX") V2DF])
+ (V16SF "TARGET_AVX512F") (V8SF "TARGET_AVX") V4SF
+ (V8DF "TARGET_AVX512F") (V4DF "TARGET_AVX") V2DF])
;; All vector modes
(define_mode_iterator V
[(V32QI "TARGET_AVX") V16QI
(V16HI "TARGET_AVX") V8HI
- (V8SI "TARGET_AVX") V4SI
- (V4DI "TARGET_AVX") V2DI
- (V8SF "TARGET_AVX") V4SF
- (V4DF "TARGET_AVX") (V2DF "TARGET_SSE2")])
+ (V16SI "TARGET_AVX512F") (V8SI "TARGET_AVX") V4SI
+ (V8DI "TARGET_AVX512F") (V4DI "TARGET_AVX") V2DI
+ (V16SF "TARGET_AVX512F") (V8SF "TARGET_AVX") V4SF
+ (V8DF "TARGET_AVX512F") (V4DF "TARGET_AVX") (V2DF "TARGET_SSE2")])
;; All 128bit vector modes
(define_mode_iterator V_128
@@ -122,19 +151,44 @@
(define_mode_iterator V_256
[V32QI V16HI V8SI V4DI V8SF V4DF])
+;; All 512bit vector modes
+(define_mode_iterator V_512 [V64QI V32HI V16SI V8DI V16SF V8DF])
+
+;; All 256bit and 512bit vector modes
+(define_mode_iterator V_256_512
+ [V32QI V16HI V8SI V4DI V8SF V4DF
+ (V64QI "TARGET_AVX512F") (V32HI "TARGET_AVX512F") (V16SI "TARGET_AVX512F")
+ (V8DI "TARGET_AVX512F") (V16SF "TARGET_AVX512F") (V8DF "TARGET_AVX512F")])
+
;; All vector float modes
(define_mode_iterator VF
+ [(V16SF "TARGET_AVX512F") (V8SF "TARGET_AVX") V4SF
+ (V8DF "TARGET_AVX512F") (V4DF "TARGET_AVX") (V2DF "TARGET_SSE2")])
+
+;; 128- and 256-bit float vector modes
+(define_mode_iterator VF_128_256
[(V8SF "TARGET_AVX") V4SF
(V4DF "TARGET_AVX") (V2DF "TARGET_SSE2")])
;; All SFmode vector float modes
(define_mode_iterator VF1
+ [(V16SF "TARGET_AVX512F") (V8SF "TARGET_AVX") V4SF])
+
+;; 128- and 256-bit SF vector modes
+(define_mode_iterator VF1_128_256
[(V8SF "TARGET_AVX") V4SF])
;; All DFmode vector float modes
(define_mode_iterator VF2
+ [(V8DF "TARGET_AVX512F") (V4DF "TARGET_AVX") V2DF])
+
+;; 128- and 256-bit DF vector modes
+(define_mode_iterator VF2_128_256
[(V4DF "TARGET_AVX") V2DF])
+(define_mode_iterator VF2_512_256
+ [(V8DF "TARGET_AVX512F") (V4DF "TARGET_AVX")])
+
;; All 128bit vector float modes
(define_mode_iterator VF_128
[V4SF (V2DF "TARGET_SSE2")])
@@ -143,9 +197,14 @@
(define_mode_iterator VF_256
[V8SF V4DF])
+;; All 512bit vector float modes
+(define_mode_iterator VF_512
+ [V16SF V8DF])
+
;; All vector integer modes
(define_mode_iterator VI
- [(V32QI "TARGET_AVX") V16QI
+ [(V16SI "TARGET_AVX512F") (V8DI "TARGET_AVX512F")
+ (V32QI "TARGET_AVX") V16QI
(V16HI "TARGET_AVX") V8HI
(V8SI "TARGET_AVX") V4SI
(V4DI "TARGET_AVX") V2DI])
@@ -153,16 +212,20 @@
(define_mode_iterator VI_AVX2
[(V32QI "TARGET_AVX2") V16QI
(V16HI "TARGET_AVX2") V8HI
- (V8SI "TARGET_AVX2") V4SI
- (V4DI "TARGET_AVX2") V2DI])
+ (V16SI "TARGET_AVX512F") (V8SI "TARGET_AVX2") V4SI
+ (V8DI "TARGET_AVX512F") (V4DI "TARGET_AVX2") V2DI])
;; All QImode vector integer modes
(define_mode_iterator VI1
[(V32QI "TARGET_AVX") V16QI])
+(define_mode_iterator VI_UNALIGNED_LOADSTORE
+ [(V32QI "TARGET_AVX") V16QI
+ (V16SI "TARGET_AVX512F") (V8DI "TARGET_AVX512F")])
+
;; All DImode vector integer modes
(define_mode_iterator VI8
- [(V4DI "TARGET_AVX") V2DI])
+ [(V8DI "TARGET_AVX512F") (V4DI "TARGET_AVX") V2DI])
(define_mode_iterator VI1_AVX2
[(V32QI "TARGET_AVX2") V16QI])
@@ -170,12 +233,36 @@
(define_mode_iterator VI2_AVX2
[(V16HI "TARGET_AVX2") V8HI])
+(define_mode_iterator VI2_AVX512F
+ [(V32HI "TARGET_AVX512F") (V16HI "TARGET_AVX2") V8HI])
+
+(define_mode_iterator VI4_AVX
+ [(V8SI "TARGET_AVX") V4SI])
+
(define_mode_iterator VI4_AVX2
[(V8SI "TARGET_AVX2") V4SI])
+(define_mode_iterator VI4_AVX512F
+ [(V16SI "TARGET_AVX512F") (V8SI "TARGET_AVX2") V4SI])
+
+(define_mode_iterator VI48_AVX512F
+ [(V16SI "TARGET_AVX512F") (V8SI "TARGET_AVX2") V4SI
+ (V8DI "TARGET_AVX512F")])
+
(define_mode_iterator VI8_AVX2
[(V4DI "TARGET_AVX2") V2DI])
+(define_mode_iterator VI8_AVX2_AVX512F
+ [(V8DI "TARGET_AVX512F") (V4DI "TARGET_AVX2") V2DI])
+
+;; All V8D* modes
+(define_mode_iterator V8FI
+ [V8DF V8DI])
+
+;; All V16S* modes
+(define_mode_iterator V16FI
+ [V16SF V16SI])
+
;; ??? We should probably use TImode instead.
(define_mode_iterator VIMAX_AVX2
[(V2TI "TARGET_AVX2") V1TI])
@@ -192,6 +279,17 @@
[(V16HI "TARGET_AVX2") V8HI
(V8SI "TARGET_AVX2") V4SI])
+(define_mode_iterator VI124_AVX2_48_AVX512F
+ [(V32QI "TARGET_AVX2") V16QI
+ (V16HI "TARGET_AVX2") V8HI
+ (V16SI "TARGET_AVX512F") (V8SI "TARGET_AVX2") V4SI
+ (V8DI "TARGET_AVX512F")])
+
+(define_mode_iterator VI124_AVX512F
+ [(V32QI "TARGET_AVX2") V16QI
+ (V32HI "TARGET_AVX512F") (V16HI "TARGET_AVX2") V8HI
+ (V16SI "TARGET_AVX512F") (V8SI "TARGET_AVX2") V4SI])
+
(define_mode_iterator VI124_AVX2
[(V32QI "TARGET_AVX2") V16QI
(V16HI "TARGET_AVX2") V8HI
@@ -202,9 +300,14 @@
(V8SI "TARGET_AVX2") V4SI
(V4DI "TARGET_AVX2") V2DI])
-(define_mode_iterator VI48_AVX2
- [(V8SI "TARGET_AVX2") V4SI
- (V4DI "TARGET_AVX2") V2DI])
+(define_mode_iterator VI248_AVX2_8_AVX512F
+ [(V16HI "TARGET_AVX2") V8HI
+ (V8SI "TARGET_AVX2") V4SI
+ (V8DI "TARGET_AVX512F") (V4DI "TARGET_AVX2") V2DI])
+
+(define_mode_iterator VI48_AVX2_48_AVX512F
+ [(V16SI "TARGET_AVX512F") (V8SI "TARGET_AVX2") V4SI
+ (V8DI "TARGET_AVX512F") (V4DI "TARGET_AVX2") V2DI])
(define_mode_iterator V48_AVX2
[V4SF V2DF
@@ -212,11 +315,18 @@
(V4SI "TARGET_AVX2") (V2DI "TARGET_AVX2")
(V8SI "TARGET_AVX2") (V4DI "TARGET_AVX2")])
+(define_mode_attr sse2_avx_avx512f
+ [(V16QI "sse2") (V32QI "avx") (V64QI "avx512f")
+ (V4SI "sse2") (V8SI "avx") (V16SI "avx512f")
+ (V8DI "avx512f")
+ (V16SF "avx512f") (V8SF "avx") (V4SF "avx")
+ (V8DF "avx512f") (V4DF "avx") (V2DF "avx")])
+
(define_mode_attr sse2_avx2
[(V16QI "sse2") (V32QI "avx2")
(V8HI "sse2") (V16HI "avx2")
- (V4SI "sse2") (V8SI "avx2")
- (V2DI "sse2") (V4DI "avx2")
+ (V4SI "sse2") (V8SI "avx2") (V16SI "avx512f")
+ (V2DI "sse2") (V4DI "avx2") (V8DI "avx512f")
(V1TI "sse2") (V2TI "avx2")])
(define_mode_attr ssse3_avx2
@@ -229,7 +339,7 @@
(define_mode_attr sse4_1_avx2
[(V16QI "sse4_1") (V32QI "avx2")
(V8HI "sse4_1") (V16HI "avx2")
- (V4SI "sse4_1") (V8SI "avx2")
+ (V4SI "sse4_1") (V8SI "avx2") (V16SI "avx512f")
(V2DI "sse4_1") (V4DI "avx2")])
(define_mode_attr avx_avx2
@@ -244,6 +354,12 @@
(V4SI "vec") (V8SI "avx2")
(V2DI "vec") (V4DI "avx2")])
+(define_mode_attr avx2_avx512f
+ [(V4SI "avx2") (V8SI "avx2") (V16SI "avx512f")
+ (V2DI "avx2") (V4DI "avx2") (V8DI "avx512f")
+ (V8SF "avx2") (V16SF "avx512f")
+ (V4DF "avx2") (V8DF "avx512f")])
+
(define_mode_attr shuffletype
[(V16SF "f") (V16SI "i") (V8DF "f") (V8DI "i")
(V8SF "f") (V8SI "i") (V4DF "f") (V4DI "i")
@@ -251,8 +367,12 @@
(V32QI "i") (V16HI "u") (V16QI "i") (V8HI "i")
(V64QI "i") (V1TI "i") (V2TI "i")])
+(define_mode_attr ssequartermode
+ [(V16SF "V4SF") (V8DF "V2DF") (V16SI "V4SI") (V8DI "V2DI")])
+
(define_mode_attr ssedoublemode
- [(V16HI "V16SI") (V8HI "V8SI") (V4HI "V4SI")
+ [(V16SF "V32SF") (V16SI "V32SI") (V8DI "V16DI") (V8DF "V16DF")
+ (V16HI "V16SI") (V8HI "V8SI") (V4HI "V4SI")
(V32QI "V32HI") (V16QI "V16HI")])
(define_mode_attr ssebytemode
@@ -264,7 +384,10 @@
;; All 256bit vector integer modes
(define_mode_iterator VI_256 [V32QI V16HI V8SI V4DI])
-;; Random 128bit vector integer mode combinations
+;; All 512bit vector integer modes
+(define_mode_iterator VI_512 [V64QI V32HI V16SI V8DI])
+
+;; Various 128bit vector integer mode combinations
(define_mode_iterator VI12_128 [V16QI V8HI])
(define_mode_iterator VI14_128 [V16QI V4SI])
(define_mode_iterator VI124_128 [V16QI V8HI V4SI])
@@ -273,36 +396,49 @@
(define_mode_iterator VI248_128 [V8HI V4SI V2DI])
(define_mode_iterator VI48_128 [V4SI V2DI])
-;; Random 256bit vector integer mode combinations
-(define_mode_iterator VI124_256 [V32QI V16HI V8SI])
+;; Various 256bit and 512 vector integer mode combinations
+(define_mode_iterator VI124_256_48_512
+ [V32QI V16HI V8SI (V8DI "TARGET_AVX512F") (V16SI "TARGET_AVX512F")])
(define_mode_iterator VI48_256 [V8SI V4DI])
+(define_mode_iterator VI48_512 [V16SI V8DI])
;; Int-float size matches
(define_mode_iterator VI4F_128 [V4SI V4SF])
(define_mode_iterator VI8F_128 [V2DI V2DF])
(define_mode_iterator VI4F_256 [V8SI V8SF])
(define_mode_iterator VI8F_256 [V4DI V4DF])
+(define_mode_iterator VI8F_256_512
+ [V4DI V4DF (V8DI "TARGET_AVX512F") (V8DF "TARGET_AVX512F")])
+(define_mode_iterator VI48F_256_512
+ [V8SI V8SF
+ (V16SI "TARGET_AVX512F") (V16SF "TARGET_AVX512F")
+ (V8DI "TARGET_AVX512F") (V8DF "TARGET_AVX512F")])
+(define_mode_iterator VI48F_512 [V16SI V16SF V8DI V8DF])
;; Mapping from float mode to required SSE level
(define_mode_attr sse
[(SF "sse") (DF "sse2")
(V4SF "sse") (V2DF "sse2")
- (V8SF "avx") (V4DF "avx")])
+ (V16SF "avx512f") (V8SF "avx")
+ (V8DF "avx512f") (V4DF "avx")])
(define_mode_attr sse2
- [(V16QI "sse2") (V32QI "avx")
- (V2DI "sse2") (V4DI "avx")])
+ [(V16QI "sse2") (V32QI "avx") (V64QI "avx512f")
+ (V2DI "sse2") (V4DI "avx") (V8DI "avx512f")])
(define_mode_attr sse3
[(V16QI "sse3") (V32QI "avx")])
(define_mode_attr sse4_1
[(V4SF "sse4_1") (V2DF "sse4_1")
- (V8SF "avx") (V4DF "avx")])
+ (V8SF "avx") (V4DF "avx")
+ (V8DF "avx512f")])
(define_mode_attr avxsizesuffix
- [(V32QI "256") (V16HI "256") (V8SI "256") (V4DI "256")
+ [(V64QI "512") (V32HI "512") (V16SI "512") (V8DI "512")
+ (V32QI "256") (V16HI "256") (V8SI "256") (V4DI "256")
(V16QI "") (V8HI "") (V4SI "") (V2DI "")
+ (V16SF "512") (V8DF "512")
(V8SF "256") (V4DF "256")
(V4SF "") (V2DF "")])
@@ -316,17 +452,29 @@
(V4SF "V4SF") (V2DF "V2DF")
(TI "TI")])
+;; Mapping of vector modes to corresponding mask size
+(define_mode_attr avx512fmaskmode
+ [(V16QI "HI")
+ (V16HI "HI") (V8HI "QI")
+ (V16SI "HI") (V8SI "QI") (V4SI "QI")
+ (V8DI "QI") (V4DI "QI") (V2DI "QI")
+ (V16SF "HI") (V8SF "QI") (V4SF "QI")
+ (V8DF "QI") (V4DF "QI") (V2DF "QI")])
+
;; Mapping of vector float modes to an integer mode of the same size
(define_mode_attr sseintvecmode
- [(V8SF "V8SI") (V4DF "V4DI")
- (V4SF "V4SI") (V2DF "V2DI")
- (V8SI "V8SI") (V4DI "V4DI")
- (V4SI "V4SI") (V2DI "V2DI")
- (V16HI "V16HI") (V8HI "V8HI")
+ [(V16SF "V16SI") (V8DF "V8DI")
+ (V8SF "V8SI") (V4DF "V4DI")
+ (V4SF "V4SI") (V2DF "V2DI")
+ (V16SI "V16SI") (V8DI "V8DI")
+ (V8SI "V8SI") (V4DI "V4DI")
+ (V4SI "V4SI") (V2DI "V2DI")
+ (V16HI "V16HI") (V8HI "V8HI")
(V32QI "V32QI") (V16QI "V16QI")])
(define_mode_attr sseintvecmodelower
- [(V8SF "v8si") (V4DF "v4di")
+ [(V16SF "v16si")
+ (V8SF "v8si") (V4DF "v4di")
(V4SF "v4si") (V2DF "v2di")
(V8SI "v8si") (V4DI "v4di")
(V4SI "v4si") (V2DI "v2di")
@@ -342,15 +490,19 @@
;; Mapping of vector modes to a vector mode of half size
(define_mode_attr ssehalfvecmode
- [(V32QI "V16QI") (V16HI "V8HI") (V8SI "V4SI") (V4DI "V2DI")
- (V16QI "V8QI") (V8HI "V4HI") (V4SI "V2SI")
- (V8SF "V4SF") (V4DF "V2DF")
- (V4SF "V2SF")])
+ [(V64QI "V32QI") (V32HI "V16HI") (V16SI "V8SI") (V8DI "V4DI")
+ (V32QI "V16QI") (V16HI "V8HI") (V8SI "V4SI") (V4DI "V2DI")
+ (V16QI "V8QI") (V8HI "V4HI") (V4SI "V2SI")
+ (V16SF "V8SF") (V8DF "V4DF")
+ (V8SF "V4SF") (V4DF "V2DF")
+ (V4SF "V2SF")])
;; Mapping of vector modes ti packed single mode of the same size
(define_mode_attr ssePSmode
- [(V32QI "V8SF") (V16QI "V4SF")
- (V16HI "V8SF") (V8HI "V4SF")
+ [(V16SI "V16SF") (V8DF "V16SF")
+ (V16SF "V16SF") (V8DI "V16SF")
+ (V64QI "V16SF") (V32QI "V8SF") (V16QI "V4SF")
+ (V32HI "V16SF") (V16HI "V8SF") (V8HI "V4SF")
(V8SI "V8SF") (V4SI "V4SF")
(V4DI "V8SF") (V2DI "V4SF")
(V2TI "V8SF") (V1TI "V4SF")
@@ -359,10 +511,21 @@
;; Mapping of vector modes back to the scalar modes
(define_mode_attr ssescalarmode
- [(V32QI "QI") (V16HI "HI") (V8SI "SI") (V4DI "DI")
- (V16QI "QI") (V8HI "HI") (V4SI "SI") (V2DI "DI")
- (V8SF "SF") (V4DF "DF")
- (V4SF "SF") (V2DF "DF")])
+ [(V64QI "QI") (V32QI "QI") (V16QI "QI")
+ (V32HI "HI") (V16HI "HI") (V8HI "HI")
+ (V16SI "SI") (V8SI "SI") (V4SI "SI")
+ (V8DI "DI") (V4DI "DI") (V2DI "DI")
+ (V16SF "SF") (V8SF "SF") (V4SF "SF")
+ (V8DF "DF") (V4DF "DF") (V2DF "DF")])
+
+;; Mapping of vector modes to the 128bit modes
+(define_mode_attr ssexmmmode
+ [(V64QI "V16QI") (V32QI "V16QI") (V16QI "V16QI")
+ (V32HI "V8HI") (V16HI "V8HI") (V8HI "V8HI")
+ (V16SI "V4SI") (V8SI "V4SI") (V4SI "V4SI")
+ (V8DI "V2DI") (V4DI "V2DI") (V2DI "V2DI")
+ (V16SF "V4SF") (V8SF "V4SF") (V4SF "V4SF")
+ (V8DF "V2DF") (V4DF "V2DF") (V2DF "V2DF")])
;; Pointer size override for scalar modes (Intel asm dialect)
(define_mode_attr iptr
@@ -374,8 +537,10 @@
;; Number of scalar elements in each vector type
(define_mode_attr ssescalarnum
- [(V32QI "32") (V16HI "16") (V8SI "8") (V4DI "4")
+ [(V64QI "64") (V16SI "16") (V8DI "8")
+ (V32QI "32") (V16HI "16") (V8SI "8") (V4DI "4")
(V16QI "16") (V8HI "8") (V4SI "4") (V2DI "2")
+ (V16SF "16") (V8DF "8")
(V8SF "8") (V4DF "4")
(V4SF "4") (V2DF "2")])
@@ -388,10 +553,12 @@
;; SSE prefix for integer vector modes
(define_mode_attr sseintprefix
- [(V2DI "p") (V2DF "")
- (V4DI "p") (V4DF "")
- (V4SI "p") (V4SF "")
- (V8SI "p") (V8SF "")])
+ [(V2DI "p") (V2DF "")
+ (V4DI "p") (V4DF "")
+ (V8DI "p") (V8DF "")
+ (V4SI "p") (V4SF "")
+ (V8SI "p") (V8SF "")
+ (V16SI "p") (V16SF "")])
;; SSE scalar suffix for vector modes
(define_mode_attr ssescalarmodesuffix
@@ -404,11 +571,13 @@
;; Pack/unpack vector modes
(define_mode_attr sseunpackmode
[(V16QI "V8HI") (V8HI "V4SI") (V4SI "V2DI")
- (V32QI "V16HI") (V16HI "V8SI") (V8SI "V4DI")])
+ (V32QI "V16HI") (V16HI "V8SI") (V8SI "V4DI")
+ (V32HI "V16SI") (V64QI "V32HI") (V16SI "V8DI")])
(define_mode_attr ssepackmode
[(V8HI "V16QI") (V4SI "V8HI") (V2DI "V4SI")
- (V16HI "V32QI") (V8SI "V16HI") (V4DI "V8SI")])
+ (V16HI "V32QI") (V8SI "V16HI") (V4DI "V8SI")
+ (V32HI "V64QI") (V16SI "V32HI") (V8DI "V16SI")])
;; Mapping of the max integer size for xop rotate immediate constraint
(define_mode_attr sserotatemax
@@ -421,9 +590,11 @@
(define_code_attr extsuffix [(sign_extend "sx") (zero_extend "zx")])
;; i128 for integer vectors and TARGET_AVX2, f128 otherwise.
+;; i64x4 or f64x4 for 512bit modes.
(define_mode_attr i128
- [(V8SF "f128") (V4DF "f128") (V32QI "%~128") (V16HI "%~128")
- (V8SI "%~128") (V4DI "%~128")])
+ [(V16SF "f64x4") (V8SF "f128") (V8DF "f64x4") (V4DF "f128")
+ (V64QI "i64x4") (V32QI "%~128") (V32HI "i64x4") (V16HI "%~128")
+ (V16SI "i64x4") (V8SI "%~128") (V8DI "i64x4") (V4DI "%~128")])
;; Mix-n-match
(define_mode_iterator AVX256MODE2P [V8SI V8SF V4DF])
@@ -432,6 +603,10 @@
(define_mode_attr blendbits
[(V8SF "255") (V4SF "15") (V4DF "15") (V2DF "3")])
+;; Mapping suffixes for broadcast
+(define_mode_attr bcstscalarsuff
+ [(V16SI "d") (V16SF "ss") (V8DI "q") (V8DF "sd")])
+
;; Patterns whose name begins with "sse{,2,3}_" are invoked by intrinsics.
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -571,6 +746,18 @@
]
(const_string "<sseinsnmode>")))])
+(define_insn "avx512f_blendm<mode>"
+ [(set (match_operand:VI48F_512 0 "register_operand" "=v")
+ (vec_merge:VI48F_512
+ (match_operand:VI48F_512 2 "nonimmediate_operand" "vm")
+ (match_operand:VI48F_512 1 "register_operand" "v")
+ (match_operand:<avx512fmaskmode> 3 "register_operand" "k")))]
+ "TARGET_AVX512F"
+ "v<sseintprefix>blendm<ssemodesuffix>\t{%2, %1, %0%{%3%}|%0%{%3%}, %1, %2}"
+ [(set_attr "type" "ssemov")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "<sseinsnmode>")])
+
(define_insn "sse2_movq128"
[(set (match_operand:V2DI 0 "register_operand" "=x")
(vec_concat:V2DI
@@ -613,10 +800,13 @@
gen_rtx_SUBREG (SImode, operands[1], 4)));
emit_insn (gen_vec_interleave_lowv4si (operands[0], operands[0],
operands[2]));
- }
+ }
else if (memory_operand (operands[1], DImode))
- emit_insn (gen_vec_concatv2di (gen_lowpart (V2DImode, operands[0]),
- operands[1], const0_rtx));
+ {
+ rtx tmp = gen_reg_rtx (V2DImode);
+ emit_insn (gen_vec_concatv2di (tmp, operands[1], const0_rtx));
+ emit_move_insn (operands[0], gen_lowpart (V4SImode, tmp));
+ }
else
gcc_unreachable ();
})
@@ -665,12 +855,13 @@
(define_insn "<sse>_loadu<ssemodesuffix><avxsizesuffix>"
[(set (match_operand:VF 0 "register_operand" "=v")
(unspec:VF
- [(match_operand:VF 1 "memory_operand" "m")]
+ [(match_operand:VF 1 "nonimmediate_operand" "vm")]
UNSPEC_LOADU))]
"TARGET_SSE"
{
switch (get_attr_mode (insn))
{
+ case MODE_V16SF:
case MODE_V8SF:
case MODE_V4SF:
return "%vmovups\t{%1, %0|%0, %1}";
@@ -694,12 +885,13 @@
(define_insn "<sse>_storeu<ssemodesuffix><avxsizesuffix>"
[(set (match_operand:VF 0 "memory_operand" "=m")
(unspec:VF
- [(match_operand:VF 1 "register_operand" "x")]
+ [(match_operand:VF 1 "register_operand" "v")]
UNSPEC_STOREU))]
"TARGET_SSE"
{
switch (get_attr_mode (insn))
{
+ case MODE_V16SF:
case MODE_V8SF:
case MODE_V4SF:
return "%vmovups\t{%1, %0|%0, %1}";
@@ -721,10 +913,11 @@
]
(const_string "<MODE>")))])
-(define_insn "<sse2>_loaddqu<avxsizesuffix>"
- [(set (match_operand:VI1 0 "register_operand" "=v")
- (unspec:VI1 [(match_operand:VI1 1 "memory_operand" "m")]
- UNSPEC_LOADU))]
+(define_insn "<sse2_avx_avx512f>_loaddqu<mode>"
+ [(set (match_operand:VI_UNALIGNED_LOADSTORE 0 "register_operand" "=v")
+ (unspec:VI_UNALIGNED_LOADSTORE
+ [(match_operand:VI_UNALIGNED_LOADSTORE 1 "nonimmediate_operand" "vm")]
+ UNSPEC_LOADU))]
"TARGET_SSE2"
{
switch (get_attr_mode (insn))
@@ -732,6 +925,11 @@
case MODE_V8SF:
case MODE_V4SF:
return "%vmovups\t{%1, %0|%0, %1}";
+ case MODE_XI:
+ if (<MODE>mode == V8DImode)
+ return "vmovdqu64\t{%1, %0|%0, %1}";
+ else
+ return "vmovdqu32\t{%1, %0|%0, %1}";
default:
return "%vmovdqu\t{%1, %0|%0, %1}";
}
@@ -754,10 +952,11 @@
]
(const_string "<sseinsnmode>")))])
-(define_insn "<sse2>_storedqu<avxsizesuffix>"
- [(set (match_operand:VI1 0 "memory_operand" "=m")
- (unspec:VI1 [(match_operand:VI1 1 "register_operand" "v")]
- UNSPEC_STOREU))]
+(define_insn "<sse2_avx_avx512f>_storedqu<mode>"
+ [(set (match_operand:VI_UNALIGNED_LOADSTORE 0 "memory_operand" "=m")
+ (unspec:VI_UNALIGNED_LOADSTORE
+ [(match_operand:VI_UNALIGNED_LOADSTORE 1 "register_operand" "v")]
+ UNSPEC_STOREU))]
"TARGET_SSE2"
{
switch (get_attr_mode (insn))
@@ -765,6 +964,11 @@
case MODE_V8SF:
case MODE_V4SF:
return "%vmovups\t{%1, %0|%0, %1}";
+ case MODE_XI:
+ if (<MODE>mode == V8DImode)
+ return "vmovdqu64\t{%1, %0|%0, %1}";
+ else
+ return "vmovdqu32\t{%1, %0|%0, %1}";
default:
return "%vmovdqu\t{%1, %0|%0, %1}";
}
@@ -821,8 +1025,9 @@
(define_insn "<sse>_movnt<mode>"
[(set (match_operand:VF 0 "memory_operand" "=m")
- (unspec:VF [(match_operand:VF 1 "register_operand" "x")]
- UNSPEC_MOVNT))]
+ (unspec:VF
+ [(match_operand:VF 1 "register_operand" "v")]
+ UNSPEC_MOVNT))]
"TARGET_SSE"
"%vmovnt<ssemodesuffix>\t{%1, %0|%0, %1}"
[(set_attr "type" "ssemov")
@@ -831,7 +1036,7 @@
(define_insn "<sse2>_movnt<mode>"
[(set (match_operand:VI8 0 "memory_operand" "=m")
- (unspec:VI8 [(match_operand:VI8 1 "register_operand" "x")]
+ (unspec:VI8 [(match_operand:VI8 1 "register_operand" "v")]
UNSPEC_MOVNT))]
"TARGET_SSE2"
"%vmovntdq\t{%1, %0|%0, %1}"
@@ -852,9 +1057,9 @@
(define_mode_iterator STORENT_MODE
[(DI "TARGET_SSE2 && TARGET_64BIT") (SI "TARGET_SSE2")
(SF "TARGET_SSE4A") (DF "TARGET_SSE4A")
- (V4DI "TARGET_AVX") (V2DI "TARGET_SSE2")
- (V8SF "TARGET_AVX") V4SF
- (V4DF "TARGET_AVX") (V2DF "TARGET_SSE2")])
+ (V8DI "TARGET_AVX512F") (V4DI "TARGET_AVX") (V2DI "TARGET_SSE2")
+ (V16SF "TARGET_AVX512F") (V8SF "TARGET_AVX") V4SF
+ (V8DF "TARGET_AVX512F") (V4DF "TARGET_AVX") (V2DF "TARGET_SSE2")])
(define_expand "storent<mode>"
[(set (match_operand:STORENT_MODE 0 "memory_operand")
@@ -877,10 +1082,10 @@
"ix86_expand_fp_absneg_operator (<CODE>, <MODE>mode, operands); DONE;")
(define_insn_and_split "*absneg<mode>2"
- [(set (match_operand:VF 0 "register_operand" "=x,x,x,x")
+ [(set (match_operand:VF 0 "register_operand" "=x,x,v,v")
(match_operator:VF 3 "absneg_operator"
- [(match_operand:VF 1 "nonimmediate_operand" "0, xm,x, m")]))
- (use (match_operand:VF 2 "nonimmediate_operand" "xm,0, xm,x"))]
+ [(match_operand:VF 1 "nonimmediate_operand" "0, xm, v, m")]))
+ (use (match_operand:VF 2 "nonimmediate_operand" "xm, 0, vm,v"))]
"TARGET_SSE"
"#"
"&& reload_completed"
@@ -962,10 +1167,10 @@
"ix86_fixup_binary_operands_no_copy (MULT, <MODE>mode, operands);")
(define_insn "*mul<mode>3"
- [(set (match_operand:VF 0 "register_operand" "=x,x")
+ [(set (match_operand:VF 0 "register_operand" "=x,v")
(mult:VF
- (match_operand:VF 1 "nonimmediate_operand" "%0,x")
- (match_operand:VF 2 "nonimmediate_operand" "xm,xm")))]
+ (match_operand:VF 1 "nonimmediate_operand" "%0,v")
+ (match_operand:VF 2 "nonimmediate_operand" "xm,vm")))]
"TARGET_SSE && ix86_binary_operator_ok (MULT, <MODE>mode, operands)"
"@
mul<ssemodesuffix>\t{%2, %0|%0, %2}
@@ -976,21 +1181,22 @@
(set_attr "btver2_decode" "direct,double")
(set_attr "mode" "<MODE>")])
-(define_insn "<sse>_vmmul<mode>3"
+(define_insn "<sse>_vm<multdiv_mnemonic><mode>3"
[(set (match_operand:VF_128 0 "register_operand" "=x,v")
(vec_merge:VF_128
- (mult:VF_128
+ (multdiv:VF_128
(match_operand:VF_128 1 "register_operand" "0,v")
(match_operand:VF_128 2 "nonimmediate_operand" "xm,vm"))
(match_dup 1)
(const_int 1)))]
"TARGET_SSE"
"@
- mul<ssescalarmodesuffix>\t{%2, %0|%0, %<iptr>2}
- vmul<ssescalarmodesuffix>\t{%2, %1, %0|%0, %1, %<iptr>2}"
+ <multdiv_mnemonic><ssescalarmodesuffix>\t{%2, %0|%0, %<iptr>2}
+ v<multdiv_mnemonic><ssescalarmodesuffix>\t{%2, %1, %0|%0, %1, %<iptr>2}"
[(set_attr "isa" "noavx,avx")
- (set_attr "type" "ssemul")
- (set_attr "prefix" "orig,vex")
+ (set_attr "type" "sse<multdiv_mnemonic>")
+ (set_attr "prefix" "orig,maybe_evex")
+ (set_attr "btver2_decode" "direct,double")
(set_attr "mode" "<ssescalarmode>")])
(define_expand "div<mode>3"
@@ -1033,28 +1239,10 @@
(set_attr "prefix" "orig,vex")
(set_attr "mode" "<MODE>")])
-(define_insn "<sse>_vmdiv<mode>3"
- [(set (match_operand:VF_128 0 "register_operand" "=x,v")
- (vec_merge:VF_128
- (div:VF_128
- (match_operand:VF_128 1 "register_operand" "0,v")
- (match_operand:VF_128 2 "nonimmediate_operand" "xm,vm"))
- (match_dup 1)
- (const_int 1)))]
- "TARGET_SSE"
- "@
- div<ssescalarmodesuffix>\t{%2, %0|%0, %<iptr>2}
- vdiv<ssescalarmodesuffix>\t{%2, %1, %0|%0, %1, %<iptr>2}"
- [(set_attr "isa" "noavx,avx")
- (set_attr "type" "ssediv")
- (set_attr "prefix" "orig,vex")
- (set_attr "btver2_decode" "direct,double")
- (set_attr "mode" "<ssescalarmode>")])
-
(define_insn "<sse>_rcp<mode>2"
- [(set (match_operand:VF1 0 "register_operand" "=x")
- (unspec:VF1
- [(match_operand:VF1 1 "nonimmediate_operand" "xm")] UNSPEC_RCP))]
+ [(set (match_operand:VF1_128_256 0 "register_operand" "=x")
+ (unspec:VF1_128_256
+ [(match_operand:VF1_128_256 1 "nonimmediate_operand" "xm")] UNSPEC_RCP))]
"TARGET_SSE"
"%vrcpps\t{%1, %0|%0, %1}"
[(set_attr "type" "sse")
@@ -1081,6 +1269,32 @@
(set_attr "prefix" "orig,vex")
(set_attr "mode" "SF")])
+(define_insn "rcp14<mode>"
+ [(set (match_operand:VF_512 0 "register_operand" "=v")
+ (unspec:VF_512
+ [(match_operand:VF_512 1 "nonimmediate_operand" "vm")]
+ UNSPEC_RCP14))]
+ "TARGET_AVX512F"
+ "vrcp14<ssemodesuffix>\t{%1, %0|%0, %1}"
+ [(set_attr "type" "sse")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "<MODE>")])
+
+(define_insn "srcp14<mode>"
+ [(set (match_operand:VF_128 0 "register_operand" "=v")
+ (vec_merge:VF_128
+ (unspec:VF_128
+ [(match_operand:VF_128 1 "register_operand" "v")
+ (match_operand:VF_128 2 "nonimmediate_operand" "vm")]
+ UNSPEC_RCP14)
+ (match_dup 1)
+ (const_int 1)))]
+ "TARGET_AVX512F"
+ "vrcp14<ssescalarmodesuffix>\t{%2, %1, %0|%0, %1, %2}"
+ [(set_attr "type" "sse")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "<MODE>")])
+
(define_expand "sqrt<mode>2"
[(set (match_operand:VF2 0 "register_operand")
(sqrt:VF2 (match_operand:VF2 1 "nonimmediate_operand")))]
@@ -1132,9 +1346,9 @@
(set_attr "mode" "<ssescalarmode>")])
(define_expand "rsqrt<mode>2"
- [(set (match_operand:VF1 0 "register_operand")
- (unspec:VF1
- [(match_operand:VF1 1 "nonimmediate_operand")] UNSPEC_RSQRT))]
+ [(set (match_operand:VF1_128_256 0 "register_operand")
+ (unspec:VF1_128_256
+ [(match_operand:VF1_128_256 1 "nonimmediate_operand")] UNSPEC_RSQRT))]
"TARGET_SSE_MATH"
{
ix86_emit_swsqrtsf (operands[0], operands[1], <MODE>mode, true);
@@ -1142,15 +1356,41 @@
})
(define_insn "<sse>_rsqrt<mode>2"
- [(set (match_operand:VF1 0 "register_operand" "=x")
- (unspec:VF1
- [(match_operand:VF1 1 "nonimmediate_operand" "xm")] UNSPEC_RSQRT))]
+ [(set (match_operand:VF1_128_256 0 "register_operand" "=x")
+ (unspec:VF1_128_256
+ [(match_operand:VF1_128_256 1 "nonimmediate_operand" "xm")] UNSPEC_RSQRT))]
"TARGET_SSE"
"%vrsqrtps\t{%1, %0|%0, %1}"
[(set_attr "type" "sse")
(set_attr "prefix" "maybe_vex")
(set_attr "mode" "<MODE>")])
+(define_insn "rsqrt14<mode>"
+ [(set (match_operand:VF_512 0 "register_operand" "=v")
+ (unspec:VF_512
+ [(match_operand:VF_512 1 "nonimmediate_operand" "vm")]
+ UNSPEC_RSQRT14))]
+ "TARGET_AVX512F"
+ "vrsqrt14<ssemodesuffix>\t{%1, %0|%0, %1}"
+ [(set_attr "type" "sse")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "<MODE>")])
+
+(define_insn "rsqrt14<mode>"
+ [(set (match_operand:VF_128 0 "register_operand" "=v")
+ (vec_merge:VF_128
+ (unspec:VF_128
+ [(match_operand:VF_128 1 "register_operand" "v")
+ (match_operand:VF_128 2 "nonimmediate_operand" "vm")]
+ UNSPEC_RSQRT14)
+ (match_dup 1)
+ (const_int 1)))]
+ "TARGET_AVX512F"
+ "vrsqrt14<ssescalarmodesuffix>\t{%2, %1, %0|%0, %1, %2}"
+ [(set_attr "type" "sse")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "<MODE>")])
+
(define_insn "sse_vmrsqrtv4sf2"
[(set (match_operand:V4SF 0 "register_operand" "=x,x")
(vec_merge:V4SF
@@ -1239,10 +1479,10 @@
;; presence of -0.0 and NaN.
(define_insn "*ieee_smin<mode>3"
- [(set (match_operand:VF 0 "register_operand" "=x,x")
+ [(set (match_operand:VF 0 "register_operand" "=v,v")
(unspec:VF
- [(match_operand:VF 1 "register_operand" "0,x")
- (match_operand:VF 2 "nonimmediate_operand" "xm,xm")]
+ [(match_operand:VF 1 "register_operand" "0,v")
+ (match_operand:VF 2 "nonimmediate_operand" "vm,vm")]
UNSPEC_IEEE_MIN))]
"TARGET_SSE"
"@
@@ -1254,10 +1494,10 @@
(set_attr "mode" "<MODE>")])
(define_insn "*ieee_smax<mode>3"
- [(set (match_operand:VF 0 "register_operand" "=x,x")
+ [(set (match_operand:VF 0 "register_operand" "=v,v")
(unspec:VF
- [(match_operand:VF 1 "register_operand" "0,x")
- (match_operand:VF 2 "nonimmediate_operand" "xm,xm")]
+ [(match_operand:VF 1 "register_operand" "0,v")
+ (match_operand:VF 2 "nonimmediate_operand" "vm,vm")]
UNSPEC_IEEE_MAX))]
"TARGET_SSE"
"@
@@ -1536,6 +1776,15 @@
(set_attr "prefix_rep" "1,*")
(set_attr "mode" "V4SF")])
+(define_expand "reduc_splus_v8df"
+ [(match_operand:V8DF 0 "register_operand")
+ (match_operand:V8DF 1 "register_operand")]
+ "TARGET_AVX512F"
+{
+ ix86_expand_reduc (gen_addv8df3, operands[0], operands[1]);
+ DONE;
+})
+
(define_expand "reduc_splus_v4df"
[(match_operand:V4DF 0 "register_operand")
(match_operand:V4DF 1 "register_operand")]
@@ -1558,6 +1807,15 @@
DONE;
})
+(define_expand "reduc_splus_v16sf"
+ [(match_operand:V16SF 0 "register_operand")
+ (match_operand:V16SF 1 "register_operand")]
+ "TARGET_AVX512F"
+{
+ ix86_expand_reduc (gen_addv16sf3, operands[0], operands[1]);
+ DONE;
+})
+
(define_expand "reduc_splus_v8sf"
[(match_operand:V8SF 0 "register_operand")
(match_operand:V8SF 1 "register_operand")]
@@ -1593,7 +1851,9 @@
[(V32QI "TARGET_AVX2") (V16HI "TARGET_AVX2")
(V8SI "TARGET_AVX2") (V4DI "TARGET_AVX2")
(V8SF "TARGET_AVX") (V4DF "TARGET_AVX")
- (V4SF "TARGET_SSE")])
+ (V4SF "TARGET_SSE") (V16SI "TARGET_AVX512F")
+ (V8DI "TARGET_AVX512F") (V16SF "TARGET_AVX512F")
+ (V8DF "TARGET_AVX512F")])
(define_expand "reduc_<code>_<mode>"
[(smaxmin:REDUC_SMINMAX_MODE
@@ -1606,6 +1866,16 @@
})
(define_expand "reduc_<code>_<mode>"
+ [(umaxmin:VI48_512
+ (match_operand:VI48_512 0 "register_operand")
+ (match_operand:VI48_512 1 "register_operand"))]
+ "TARGET_AVX512F"
+{
+ ix86_expand_reduc (gen_<code><mode>3, operands[0], operands[1]);
+ DONE;
+})
+
+(define_expand "reduc_<code>_<mode>"
[(umaxmin:VI_256
(match_operand:VI_256 0 "register_operand")
(match_operand:VI_256 1 "register_operand"))]
@@ -1632,10 +1902,10 @@
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(define_insn "avx_cmp<mode>3"
- [(set (match_operand:VF 0 "register_operand" "=x")
- (unspec:VF
- [(match_operand:VF 1 "register_operand" "x")
- (match_operand:VF 2 "nonimmediate_operand" "xm")
+ [(set (match_operand:VF_128_256 0 "register_operand" "=x")
+ (unspec:VF_128_256
+ [(match_operand:VF_128_256 1 "register_operand" "x")
+ (match_operand:VF_128_256 2 "nonimmediate_operand" "xm")
(match_operand:SI 3 "const_0_to_31_operand" "n")]
UNSPEC_PCMP))]
"TARGET_AVX"
@@ -1663,10 +1933,10 @@
(set_attr "mode" "<ssescalarmode>")])
(define_insn "*<sse>_maskcmp<mode>3_comm"
- [(set (match_operand:VF 0 "register_operand" "=x,x")
- (match_operator:VF 3 "sse_comparison_operator"
- [(match_operand:VF 1 "register_operand" "%0,x")
- (match_operand:VF 2 "nonimmediate_operand" "xm,xm")]))]
+ [(set (match_operand:VF_128_256 0 "register_operand" "=x,x")
+ (match_operator:VF_128_256 3 "sse_comparison_operator"
+ [(match_operand:VF_128_256 1 "register_operand" "%0,x")
+ (match_operand:VF_128_256 2 "nonimmediate_operand" "xm,xm")]))]
"TARGET_SSE
&& GET_RTX_CLASS (GET_CODE (operands[3])) == RTX_COMM_COMPARE"
"@
@@ -1679,10 +1949,10 @@
(set_attr "mode" "<MODE>")])
(define_insn "<sse>_maskcmp<mode>3"
- [(set (match_operand:VF 0 "register_operand" "=x,x")
- (match_operator:VF 3 "sse_comparison_operator"
- [(match_operand:VF 1 "register_operand" "0,x")
- (match_operand:VF 2 "nonimmediate_operand" "xm,xm")]))]
+ [(set (match_operand:VF_128_256 0 "register_operand" "=x,x")
+ (match_operator:VF_128_256 3 "sse_comparison_operator"
+ [(match_operand:VF_128_256 1 "register_operand" "0,x")
+ (match_operand:VF_128_256 2 "nonimmediate_operand" "xm,xm")]))]
"TARGET_SSE"
"@
cmp%D3<ssemodesuffix>\t{%2, %0|%0, %2}
@@ -1711,14 +1981,74 @@
(set_attr "prefix" "orig,vex")
(set_attr "mode" "<ssescalarmode>")])
+(define_mode_attr cmp_imm_predicate
+ [(V16SF "const_0_to_31_operand") (V8DF "const_0_to_31_operand")
+ (V16SI "const_0_to_7_operand") (V8DI "const_0_to_7_operand")])
+
+(define_insn "avx512f_cmp<mode>3"
+ [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=k")
+ (unspec:<avx512fmaskmode>
+ [(match_operand:VI48F_512 1 "register_operand" "v")
+ (match_operand:VI48F_512 2 "nonimmediate_operand" "vm")
+ (match_operand:SI 3 "<cmp_imm_predicate>" "n")]
+ UNSPEC_PCMP))]
+ "TARGET_AVX512F"
+ "v<sseintprefix>cmp<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
+ [(set_attr "type" "ssecmp")
+ (set_attr "length_immediate" "1")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "<sseinsnmode>")])
+
+(define_insn "avx512f_ucmp<mode>3"
+ [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=k")
+ (unspec:<avx512fmaskmode>
+ [(match_operand:VI48_512 1 "register_operand" "v")
+ (match_operand:VI48_512 2 "nonimmediate_operand" "vm")
+ (match_operand:SI 3 "const_0_to_7_operand" "n")]
+ UNSPEC_UNSIGNED_PCMP))]
+ "TARGET_AVX512F"
+ "vpcmpu<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
+ [(set_attr "type" "ssecmp")
+ (set_attr "length_immediate" "1")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "<sseinsnmode>")])
+
+(define_insn "avx512f_vmcmp<mode>3"
+ [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=k")
+ (and:<avx512fmaskmode>
+ (unspec:<avx512fmaskmode>
+ [(match_operand:VF_128 1 "register_operand" "v")
+ (match_operand:VF_128 2 "nonimmediate_operand" "vm")
+ (match_operand:SI 3 "const_0_to_31_operand" "n")]
+ UNSPEC_PCMP)
+ (const_int 1)))]
+ "TARGET_AVX512F"
+ "vcmp<ssescalarmodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
+ [(set_attr "type" "ssecmp")
+ (set_attr "length_immediate" "1")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "<ssescalarmode>")])
+
+(define_insn "avx512f_maskcmp<mode>3"
+ [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=k")
+ (match_operator:<avx512fmaskmode> 3 "sse_comparison_operator"
+ [(match_operand:VF 1 "register_operand" "v")
+ (match_operand:VF 2 "nonimmediate_operand" "vm")]))]
+ "TARGET_SSE"
+ "vcmp%D3<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
+ [(set_attr "type" "ssecmp")
+ (set_attr "length_immediate" "1")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "<sseinsnmode>")])
+
(define_insn "<sse>_comi"
[(set (reg:CCFP FLAGS_REG)
(compare:CCFP
(vec_select:MODEF
- (match_operand:<ssevecmode> 0 "register_operand" "x")
+ (match_operand:<ssevecmode> 0 "register_operand" "v")
(parallel [(const_int 0)]))
(vec_select:MODEF
- (match_operand:<ssevecmode> 1 "nonimmediate_operand" "xm")
+ (match_operand:<ssevecmode> 1 "nonimmediate_operand" "vm")
(parallel [(const_int 0)]))))]
"SSE_FLOAT_MODE_P (<MODE>mode)"
"%vcomi<ssemodesuffix>\t{%1, %0|%0, %<iptr>1}"
@@ -1735,10 +2065,10 @@
[(set (reg:CCFPU FLAGS_REG)
(compare:CCFPU
(vec_select:MODEF
- (match_operand:<ssevecmode> 0 "register_operand" "x")
+ (match_operand:<ssevecmode> 0 "register_operand" "v")
(parallel [(const_int 0)]))
(vec_select:MODEF
- (match_operand:<ssevecmode> 1 "nonimmediate_operand" "xm")
+ (match_operand:<ssevecmode> 1 "nonimmediate_operand" "vm")
(parallel [(const_int 0)]))))]
"SSE_FLOAT_MODE_P (<MODE>mode)"
"%vucomi<ssemodesuffix>\t{%1, %0|%0, %<iptr>1}"
@@ -1751,6 +2081,23 @@
(const_string "0")))
(set_attr "mode" "<MODE>")])
+(define_expand "vcond<V_512:mode><VF_512:mode>"
+ [(set (match_operand:V_512 0 "register_operand")
+ (if_then_else:V_512
+ (match_operator 3 ""
+ [(match_operand:VF_512 4 "nonimmediate_operand")
+ (match_operand:VF_512 5 "nonimmediate_operand")])
+ (match_operand:V_512 1 "general_operand")
+ (match_operand:V_512 2 "general_operand")))]
+ "TARGET_AVX512F
+ && (GET_MODE_NUNITS (<V_512:MODE>mode)
+ == GET_MODE_NUNITS (<VF_512:MODE>mode))"
+{
+ bool ok = ix86_expand_fp_vcond (operands);
+ gcc_assert (ok);
+ DONE;
+})
+
(define_expand "vcond<V_256:mode><VF_256:mode>"
[(set (match_operand:V_256 0 "register_operand")
(if_then_else:V_256
@@ -1792,11 +2139,11 @@
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(define_insn "<sse>_andnot<mode>3"
- [(set (match_operand:VF 0 "register_operand" "=x,x")
+ [(set (match_operand:VF 0 "register_operand" "=x,v")
(and:VF
(not:VF
- (match_operand:VF 1 "register_operand" "0,x"))
- (match_operand:VF 2 "nonimmediate_operand" "xm,xm")))]
+ (match_operand:VF 1 "register_operand" "0,v"))
+ (match_operand:VF 2 "nonimmediate_operand" "xm,vm")))]
"TARGET_SSE"
{
static char buf[32];
@@ -1825,12 +2172,19 @@
gcc_unreachable ();
}
+ /* There is no vandnp[sd]. Use vpandnq. */
+ if (GET_MODE_SIZE (<MODE>mode) == 64)
+ {
+ suffix = "q";
+ ops = "vpandn%s\t{%%2, %%1, %%0|%%0, %%1, %%2}";
+ }
+
snprintf (buf, sizeof (buf), ops, suffix);
return buf;
}
[(set_attr "isa" "noavx,avx")
(set_attr "type" "sselog")
- (set_attr "prefix" "orig,vex")
+ (set_attr "prefix" "orig,maybe_evex")
(set (attr "mode")
(cond [(match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL")
(const_string "<ssePSmode>")
@@ -1842,13 +2196,21 @@
(const_string "<MODE>")))])
(define_expand "<code><mode>3"
- [(set (match_operand:VF 0 "register_operand")
- (any_logic:VF
- (match_operand:VF 1 "nonimmediate_operand")
- (match_operand:VF 2 "nonimmediate_operand")))]
+ [(set (match_operand:VF_128_256 0 "register_operand")
+ (any_logic:VF_128_256
+ (match_operand:VF_128_256 1 "nonimmediate_operand")
+ (match_operand:VF_128_256 2 "nonimmediate_operand")))]
"TARGET_SSE"
"ix86_fixup_binary_operands_no_copy (<CODE>, <MODE>mode, operands);")
+(define_expand "<code><mode>3"
+ [(set (match_operand:VF_512 0 "register_operand")
+ (fpint_logic:VF_512
+ (match_operand:VF_512 1 "nonimmediate_operand")
+ (match_operand:VF_512 2 "nonimmediate_operand")))]
+ "TARGET_AVX512F"
+ "ix86_fixup_binary_operands_no_copy (<CODE>, <MODE>mode, operands);")
+
(define_insn "*<code><mode>3"
[(set (match_operand:VF 0 "register_operand" "=x,v")
(any_logic:VF
@@ -1882,12 +2244,19 @@
gcc_unreachable ();
}
+ /* There is no v<logic>p[sd]. Use vp<logic>q. */
+ if (GET_MODE_SIZE (<MODE>mode) == 64)
+ {
+ suffix = "q";
+ ops = "vp<logic>%s\t{%%2, %%1, %%0|%%0, %%1, %%2}";
+ }
+
snprintf (buf, sizeof (buf), ops, suffix);
return buf;
}
[(set_attr "isa" "noavx,avx")
(set_attr "type" "sselog")
- (set_attr "prefix" "orig,vex")
+ (set_attr "prefix" "orig,maybe_evex")
(set (attr "mode")
(cond [(match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL")
(const_string "<ssePSmode>")
@@ -2105,6 +2474,23 @@
]
(const_string "TI")))])
+;; There are no floating point xor for V16SF and V8DF in avx512f
+;; but we need them for negation. Instead we use int versions of
+;; xor. Maybe there could be a better way to do that.
+
+(define_mode_attr avx512flogicsuff
+ [(V16SF "d") (V8DF "q")])
+
+(define_insn "avx512f_<logic><mode>"
+ [(set (match_operand:VF_512 0 "register_operand" "=v")
+ (fpint_logic:VF_512
+ (match_operand:VF_512 1 "register_operand" "v")
+ (match_operand:VF_512 2 "nonimmediate_operand" "vm")))]
+ "TARGET_AVX512F"
+ "vp<logic><avx512flogicsuff>\t{%2, %1, %0|%0, %1, %2}"
+ [(set_attr "type" "sselog")
+ (set_attr "prefix" "evex")])
+
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;
;; FMA floating point multiply/accumulate instructions. These include
@@ -2113,9 +2499,22 @@
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; The standard names for scalar FMA are only available with SSE math enabled.
-(define_mode_iterator FMAMODEM [(SF "TARGET_SSE_MATH")
- (DF "TARGET_SSE_MATH")
- V4SF V2DF V8SF V4DF])
+;; CPUID bit AVX512F enables evex encoded scalar and 512-bit fma. It doesn't
+;; care about FMA bit, so we enable fma for TARGET_AVX512F even when TARGET_FMA
+;; and TARGET_FMA4 are both false.
+;; TODO: In theory AVX512F does not automatically imply FMA, and without FMA
+;; one must force the EVEX encoding of the fma insns. Ideally we'd improve
+;; GAS to allow proper prefix selection. However, for the moment all hardware
+;; that supports AVX512F also supports FMA so we can ignore this for now.
+(define_mode_iterator FMAMODEM
+ [(SF "TARGET_SSE_MATH && (TARGET_FMA || TARGET_FMA4 || TARGET_AVX512F)")
+ (DF "TARGET_SSE_MATH && (TARGET_FMA || TARGET_FMA4 || TARGET_AVX512F)")
+ (V4SF "TARGET_FMA || TARGET_FMA4")
+ (V2DF "TARGET_FMA || TARGET_FMA4")
+ (V8SF "TARGET_FMA || TARGET_FMA4")
+ (V4DF "TARGET_FMA || TARGET_FMA4")
+ (V16SF "TARGET_AVX512F")
+ (V8DF "TARGET_AVX512F")])
(define_expand "fma<mode>4"
[(set (match_operand:FMAMODEM 0 "register_operand")
@@ -2123,7 +2522,7 @@
(match_operand:FMAMODEM 1 "nonimmediate_operand")
(match_operand:FMAMODEM 2 "nonimmediate_operand")
(match_operand:FMAMODEM 3 "nonimmediate_operand")))]
- "TARGET_FMA || TARGET_FMA4")
+ "")
(define_expand "fms<mode>4"
[(set (match_operand:FMAMODEM 0 "register_operand")
@@ -2131,7 +2530,7 @@
(match_operand:FMAMODEM 1 "nonimmediate_operand")
(match_operand:FMAMODEM 2 "nonimmediate_operand")
(neg:FMAMODEM (match_operand:FMAMODEM 3 "nonimmediate_operand"))))]
- "TARGET_FMA || TARGET_FMA4")
+ "")
(define_expand "fnma<mode>4"
[(set (match_operand:FMAMODEM 0 "register_operand")
@@ -2139,7 +2538,7 @@
(neg:FMAMODEM (match_operand:FMAMODEM 1 "nonimmediate_operand"))
(match_operand:FMAMODEM 2 "nonimmediate_operand")
(match_operand:FMAMODEM 3 "nonimmediate_operand")))]
- "TARGET_FMA || TARGET_FMA4")
+ "")
(define_expand "fnms<mode>4"
[(set (match_operand:FMAMODEM 0 "register_operand")
@@ -2147,10 +2546,17 @@
(neg:FMAMODEM (match_operand:FMAMODEM 1 "nonimmediate_operand"))
(match_operand:FMAMODEM 2 "nonimmediate_operand")
(neg:FMAMODEM (match_operand:FMAMODEM 3 "nonimmediate_operand"))))]
- "TARGET_FMA || TARGET_FMA4")
+ "")
;; The builtins for intrinsics are not constrained by SSE math enabled.
-(define_mode_iterator FMAMODE [SF DF V4SF V2DF V8SF V4DF])
+(define_mode_iterator FMAMODE [(SF "TARGET_FMA || TARGET_FMA4 || TARGET_AVX512F")
+ (DF "TARGET_FMA || TARGET_FMA4 || TARGET_AVX512F")
+ (V4SF "TARGET_FMA || TARGET_FMA4")
+ (V2DF "TARGET_FMA || TARGET_FMA4")
+ (V8SF "TARGET_FMA || TARGET_FMA4")
+ (V4DF "TARGET_FMA || TARGET_FMA4")
+ (V16SF "TARGET_AVX512F")
+ (V8DF "TARGET_AVX512F")])
(define_expand "fma4i_fmadd_<mode>"
[(set (match_operand:FMAMODE 0 "register_operand")
@@ -2158,7 +2564,7 @@
(match_operand:FMAMODE 1 "nonimmediate_operand")
(match_operand:FMAMODE 2 "nonimmediate_operand")
(match_operand:FMAMODE 3 "nonimmediate_operand")))]
- "TARGET_FMA || TARGET_FMA4")
+ "")
(define_insn "*fma_fmadd_<mode>"
[(set (match_operand:FMAMODE 0 "register_operand" "=v,v,v,x,x")
@@ -2166,7 +2572,7 @@
(match_operand:FMAMODE 1 "nonimmediate_operand" "%0, 0, v, x,x")
(match_operand:FMAMODE 2 "nonimmediate_operand" "vm, v,vm, x,m")
(match_operand:FMAMODE 3 "nonimmediate_operand" " v,vm, 0,xm,x")))]
- "TARGET_FMA || TARGET_FMA4"
+ ""
"@
vfmadd132<ssemodesuffix>\t{%2, %3, %0|%0, %3, %2}
vfmadd213<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}
@@ -2177,14 +2583,14 @@
(set_attr "type" "ssemuladd")
(set_attr "mode" "<MODE>")])
-(define_insn "*fma_fmsub_<mode>"
+(define_insn "fma_fmsub_<mode>"
[(set (match_operand:FMAMODE 0 "register_operand" "=v,v,v,x,x")
(fma:FMAMODE
(match_operand:FMAMODE 1 "nonimmediate_operand" "%0, 0, v, x,x")
(match_operand:FMAMODE 2 "nonimmediate_operand" "vm, v,vm, x,m")
(neg:FMAMODE
(match_operand:FMAMODE 3 "nonimmediate_operand" " v,vm, 0,xm,x"))))]
- "TARGET_FMA || TARGET_FMA4"
+ ""
"@
vfmsub132<ssemodesuffix>\t{%2, %3, %0|%0, %3, %2}
vfmsub213<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}
@@ -2195,14 +2601,14 @@
(set_attr "type" "ssemuladd")
(set_attr "mode" "<MODE>")])
-(define_insn "*fma_fnmadd_<mode>"
+(define_insn "fma_fnmadd_<mode>"
[(set (match_operand:FMAMODE 0 "register_operand" "=v,v,v,x,x")
(fma:FMAMODE
(neg:FMAMODE
(match_operand:FMAMODE 1 "nonimmediate_operand" "%0, 0, v, x,x"))
(match_operand:FMAMODE 2 "nonimmediate_operand" "vm, v,vm, x,m")
(match_operand:FMAMODE 3 "nonimmediate_operand" " v,vm, 0,xm,x")))]
- "TARGET_FMA || TARGET_FMA4"
+ ""
"@
vfnmadd132<ssemodesuffix>\t{%2, %3, %0|%0, %3, %2}
vfnmadd213<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}
@@ -2221,7 +2627,7 @@
(match_operand:FMAMODE 2 "nonimmediate_operand" "vm, v,vm, x,m")
(neg:FMAMODE
(match_operand:FMAMODE 3 "nonimmediate_operand" " v,vm, 0,xm,x"))))]
- "TARGET_FMA || TARGET_FMA4"
+ ""
"@
vfnmsub132<ssemodesuffix>\t{%2, %3, %0|%0, %3, %2}
vfnmsub213<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}
@@ -2250,7 +2656,7 @@
(match_operand:VF 2 "nonimmediate_operand")
(match_operand:VF 3 "nonimmediate_operand")]
UNSPEC_FMADDSUB))]
- "TARGET_FMA || TARGET_FMA4")
+ "TARGET_FMA || TARGET_FMA4 || TARGET_AVX512F")
(define_insn "*fma_fmaddsub_<mode>"
[(set (match_operand:VF 0 "register_operand" "=v,v,v,x,x")
@@ -2259,7 +2665,7 @@
(match_operand:VF 2 "nonimmediate_operand" "vm, v,vm, x,m")
(match_operand:VF 3 "nonimmediate_operand" " v,vm, 0,xm,x")]
UNSPEC_FMADDSUB))]
- "TARGET_FMA || TARGET_FMA4"
+ "(TARGET_FMA || TARGET_FMA4 || TARGET_AVX512F)"
"@
vfmaddsub132<ssemodesuffix>\t{%2, %3, %0|%0, %3, %2}
vfmaddsub213<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}
@@ -2278,7 +2684,7 @@
(neg:VF
(match_operand:VF 3 "nonimmediate_operand" " v,vm, 0,xm,x"))]
UNSPEC_FMADDSUB))]
- "TARGET_FMA || TARGET_FMA4"
+ "(TARGET_FMA || TARGET_FMA4 || TARGET_AVX512F)"
"@
vfmsubadd132<ssemodesuffix>\t{%2, %3, %0|%0, %3, %2}
vfmsubadd213<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}
@@ -2312,7 +2718,7 @@
(match_operand:VF_128 3 "nonimmediate_operand" " v,vm"))
(match_dup 1)
(const_int 1)))]
- "TARGET_FMA"
+ "TARGET_FMA || TARGET_AVX512F"
"@
vfmadd132<ssescalarmodesuffix>\t{%2, %3, %0|%0, %<iptr>3, %<iptr>2}
vfmadd213<ssescalarmodesuffix>\t{%3, %2, %0|%0, %<iptr>2, %<iptr>3}"
@@ -2329,7 +2735,7 @@
(match_operand:VF_128 3 "nonimmediate_operand" " v,vm")))
(match_dup 1)
(const_int 1)))]
- "TARGET_FMA"
+ "TARGET_FMA || TARGET_AVX512F"
"@
vfmsub132<ssescalarmodesuffix>\t{%2, %3, %0|%0, %<iptr>3, %<iptr>2}
vfmsub213<ssescalarmodesuffix>\t{%3, %2, %0|%0, %<iptr>2, %<iptr>3}"
@@ -2346,7 +2752,7 @@
(match_operand:VF_128 3 "nonimmediate_operand" " v,vm"))
(match_dup 1)
(const_int 1)))]
- "TARGET_FMA"
+ "TARGET_FMA || TARGET_AVX512F"
"@
vfnmadd132<ssescalarmodesuffix>\t{%2, %3, %0|%0, %<iptr>3, %<iptr>2}
vfnmadd213<ssescalarmodesuffix>\t{%3, %2, %0|%0, %<iptr>2, %<iptr>3}"
@@ -2364,7 +2770,7 @@
(match_operand:VF_128 3 "nonimmediate_operand" " v,vm")))
(match_dup 1)
(const_int 1)))]
- "TARGET_FMA"
+ "TARGET_FMA || TARGET_AVX512F"
"@
vfnmsub132<ssescalarmodesuffix>\t{%2, %3, %0|%0, %<iptr>3, %<iptr>2}
vfnmsub213<ssescalarmodesuffix>\t{%3, %2, %0|%0, %<iptr>2, %<iptr>3}"
@@ -2506,7 +2912,7 @@
(set_attr "amdfam10_decode" "vector,double,*")
(set_attr "bdver1_decode" "double,direct,*")
(set_attr "btver2_decode" "double,double,double")
- (set_attr "prefix" "orig,orig,vex")
+ (set_attr "prefix" "orig,orig,maybe_evex")
(set_attr "mode" "SF")])
(define_insn "sse_cvtsi2ssq"
@@ -2529,7 +2935,7 @@
(set_attr "btver2_decode" "double,double,double")
(set_attr "length_vex" "*,*,4")
(set_attr "prefix_rex" "1,1,*")
- (set_attr "prefix" "orig,orig,vex")
+ (set_attr "prefix" "orig,orig,maybe_evex")
(set_attr "mode" "SF")])
(define_insn "sse_cvtss2si"
@@ -2580,7 +2986,7 @@
(define_insn "sse_cvtss2siq_2"
[(set (match_operand:DI 0 "register_operand" "=r,r")
- (unspec:DI [(match_operand:SF 1 "nonimmediate_operand" "x,m")]
+ (unspec:DI [(match_operand:SF 1 "nonimmediate_operand" "v,m")]
UNSPEC_FIX_NOTRUNC))]
"TARGET_SSE && TARGET_64BIT"
"%vcvtss2si{q}\t{%1, %0|%0, %k1}"
@@ -2624,6 +3030,34 @@
(set_attr "prefix" "maybe_vex")
(set_attr "mode" "DI")])
+(define_insn "cvtusi2<ssescalarmodesuffix>32"
+ [(set (match_operand:VF_128 0 "register_operand" "=v")
+ (vec_merge:VF_128
+ (vec_duplicate:VF_128
+ (unsigned_float:<ssescalarmode>
+ (match_operand:SI 2 "nonimmediate_operand" "rm")))
+ (match_operand:VF_128 1 "register_operand" "v")
+ (const_int 1)))]
+ "TARGET_AVX512F"
+ "vcvtusi2<ssescalarmodesuffix>\t{%2, %1, %0|%0, %1, %2}"
+ [(set_attr "type" "sseicvt")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "<ssescalarmode>")])
+
+(define_insn "cvtusi2<ssescalarmodesuffix>64"
+ [(set (match_operand:VF_128 0 "register_operand" "=v")
+ (vec_merge:VF_128
+ (vec_duplicate:VF_128
+ (unsigned_float:<ssescalarmode>
+ (match_operand:DI 2 "nonimmediate_operand" "rm")))
+ (match_operand:VF_128 1 "register_operand" "v")
+ (const_int 1)))]
+ "TARGET_AVX512F && TARGET_64BIT"
+ "vcvtusi2<ssescalarmodesuffix>\t{%2, %1, %0|%0, %1, %2}"
+ [(set_attr "type" "sseicvt")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "<ssescalarmode>")])
+
(define_insn "float<sseintvecmodelower><mode>2"
[(set (match_operand:VF1 0 "register_operand" "=v")
(float:VF1
@@ -2634,6 +3068,16 @@
(set_attr "prefix" "maybe_vex")
(set_attr "mode" "<sseinsnmode>")])
+(define_insn "ufloatv16siv16sf2"
+ [(set (match_operand:V16SF 0 "register_operand" "=v")
+ (unsigned_float:V16SF
+ (match_operand:V16SI 1 "nonimmediate_operand" "vm")))]
+ "TARGET_AVX512F"
+ "vcvtudq2ps\t{%1, %0|%0, %1}"
+ [(set_attr "type" "ssecvt")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "V16SF")])
+
(define_expand "floatuns<sseintvecmodelower><mode>2"
[(match_operand:VF1 0 "register_operand")
(match_operand:<sseintvecmode> 1 "register_operand")]
@@ -2643,20 +3087,16 @@
DONE;
})
-(define_insn "avx_cvtps2dq256"
- [(set (match_operand:V8SI 0 "register_operand" "=x")
- (unspec:V8SI [(match_operand:V8SF 1 "nonimmediate_operand" "xm")]
- UNSPEC_FIX_NOTRUNC))]
- "TARGET_AVX"
- "vcvtps2dq\t{%1, %0|%0, %1}"
- [(set_attr "type" "ssecvt")
- (set_attr "prefix" "vex")
- (set_attr "mode" "OI")])
-(define_insn "sse2_cvtps2dq"
- [(set (match_operand:V4SI 0 "register_operand" "=x")
- (unspec:V4SI [(match_operand:V4SF 1 "nonimmediate_operand" "xm")]
- UNSPEC_FIX_NOTRUNC))]
+;; For <sse2_avx_avx512f>_fix_notrunc<sf2simodelower><mode> insn pattern
+(define_mode_attr sf2simodelower
+ [(V16SI "v16sf") (V8SI "v8sf") (V4SI "v4sf")])
+
+(define_insn "<sse2_avx_avx512f>_fix_notrunc<sf2simodelower><mode>"
+ [(set (match_operand:VI4_AVX 0 "register_operand" "=v")
+ (unspec:VI4_AVX
+ [(match_operand:<ssePSmode> 1 "nonimmediate_operand" "vm")]
+ UNSPEC_FIX_NOTRUNC))]
"TARGET_SSE2"
"%vcvtps2dq\t{%1, %0|%0, %1}"
[(set_attr "type" "ssecvt")
@@ -2666,7 +3106,39 @@
(const_string "*")
(const_string "1")))
(set_attr "prefix" "maybe_vex")
- (set_attr "mode" "TI")])
+ (set_attr "mode" "<sseinsnmode>")])
+
+(define_insn "avx512f_fix_notruncv16sfv16si"
+ [(set (match_operand:V16SI 0 "register_operand" "=v")
+ (unspec:V16SI
+ [(match_operand:V16SF 1 "nonimmediate_operand" "vm")]
+ UNSPEC_FIX_NOTRUNC))]
+ "TARGET_AVX512F"
+ "vcvtps2dq\t{%1, %0|%0, %1}"
+ [(set_attr "type" "ssecvt")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "XI")])
+
+(define_insn "avx512f_ufix_notruncv16sfv16si"
+ [(set (match_operand:V16SI 0 "register_operand" "=v")
+ (unspec:V16SI
+ [(match_operand:V16SF 1 "nonimmediate_operand" "vm")]
+ UNSPEC_UNSIGNED_FIX_NOTRUNC))]
+ "TARGET_AVX512F"
+ "vcvtps2udq\t{%1, %0|%0, %1}"
+ [(set_attr "type" "ssecvt")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "XI")])
+
+(define_insn "<fixsuffix>fix_truncv16sfv16si2"
+ [(set (match_operand:V16SI 0 "register_operand" "=v")
+ (any_fix:V16SI
+ (match_operand:V16SF 1 "nonimmediate_operand" "vm")))]
+ "TARGET_AVX512F"
+ "vcvttps2<fixsuffix>dq\t{%1, %0|%0, %1}"
+ [(set_attr "type" "ssecvt")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "XI")])
(define_insn "fix_truncv8sfv8si2"
[(set (match_operand:V8SI 0 "register_operand" "=x")
@@ -2772,11 +3244,11 @@
(set_attr "mode" "DF")])
(define_insn "sse2_cvtsi2sdq"
- [(set (match_operand:V2DF 0 "register_operand" "=x,x,x")
+ [(set (match_operand:V2DF 0 "register_operand" "=x,x,v")
(vec_merge:V2DF
(vec_duplicate:V2DF
(float:DF (match_operand:DI 2 "nonimmediate_operand" "r,m,rm")))
- (match_operand:V2DF 1 "register_operand" "0,0,x")
+ (match_operand:V2DF 1 "register_operand" "0,0,v")
(const_int 1)))]
"TARGET_SSE2 && TARGET_64BIT"
"@
@@ -2790,14 +3262,114 @@
(set_attr "bdver1_decode" "double,direct,*")
(set_attr "length_vex" "*,*,4")
(set_attr "prefix_rex" "1,1,*")
- (set_attr "prefix" "orig,orig,vex")
+ (set_attr "prefix" "orig,orig,maybe_evex")
(set_attr "mode" "DF")])
+(define_insn "avx512f_vcvtss2usi"
+ [(set (match_operand:SI 0 "register_operand" "=r")
+ (unspec:SI
+ [(vec_select:SF
+ (match_operand:V4SF 1 "nonimmediate_operand" "vm")
+ (parallel [(const_int 0)]))]
+ UNSPEC_UNSIGNED_FIX_NOTRUNC))]
+ "TARGET_AVX512F"
+ "vcvtss2usi\t{%1, %0|%0, %1}"
+ [(set_attr "type" "sseicvt")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "SI")])
+
+(define_insn "avx512f_vcvtss2usiq"
+ [(set (match_operand:DI 0 "register_operand" "=r")
+ (unspec:DI
+ [(vec_select:SF
+ (match_operand:V4SF 1 "nonimmediate_operand" "vm")
+ (parallel [(const_int 0)]))]
+ UNSPEC_UNSIGNED_FIX_NOTRUNC))]
+ "TARGET_AVX512F && TARGET_64BIT"
+ "vcvtss2usi\t{%1, %0|%0, %1}"
+ [(set_attr "type" "sseicvt")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "DI")])
+
+(define_insn "avx512f_vcvttss2usi"
+ [(set (match_operand:SI 0 "register_operand" "=r")
+ (unsigned_fix:SI
+ (vec_select:SF
+ (match_operand:V4SF 1 "nonimmediate_operand" "vm")
+ (parallel [(const_int 0)]))))]
+ "TARGET_AVX512F"
+ "vcvttss2usi\t{%1, %0|%0, %1}"
+ [(set_attr "type" "sseicvt")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "SI")])
+
+(define_insn "avx512f_vcvttss2usiq"
+ [(set (match_operand:DI 0 "register_operand" "=r")
+ (unsigned_fix:DI
+ (vec_select:SF
+ (match_operand:V4SF 1 "nonimmediate_operand" "vm")
+ (parallel [(const_int 0)]))))]
+ "TARGET_AVX512F && TARGET_64BIT"
+ "vcvttss2usi\t{%1, %0|%0, %1}"
+ [(set_attr "type" "sseicvt")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "DI")])
+
+(define_insn "avx512f_vcvtsd2usi"
+ [(set (match_operand:SI 0 "register_operand" "=r")
+ (unspec:SI
+ [(vec_select:DF
+ (match_operand:V2DF 1 "nonimmediate_operand" "vm")
+ (parallel [(const_int 0)]))]
+ UNSPEC_UNSIGNED_FIX_NOTRUNC))]
+ "TARGET_AVX512F"
+ "vcvtsd2usi\t{%1, %0|%0, %1}"
+ [(set_attr "type" "sseicvt")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "SI")])
+
+(define_insn "avx512f_vcvtsd2usiq"
+ [(set (match_operand:DI 0 "register_operand" "=r")
+ (unspec:DI
+ [(vec_select:DF
+ (match_operand:V2DF 1 "nonimmediate_operand" "vm")
+ (parallel [(const_int 0)]))]
+ UNSPEC_UNSIGNED_FIX_NOTRUNC))]
+ "TARGET_AVX512F && TARGET_64BIT"
+ "vcvtsd2usi\t{%1, %0|%0, %1}"
+ [(set_attr "type" "sseicvt")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "DI")])
+
+(define_insn "avx512f_vcvttsd2usi"
+ [(set (match_operand:SI 0 "register_operand" "=r")
+ (unsigned_fix:SI
+ (vec_select:DF
+ (match_operand:V2DF 1 "nonimmediate_operand" "vm")
+ (parallel [(const_int 0)]))))]
+ "TARGET_AVX512F"
+ "vcvttsd2usi\t{%1, %0|%0, %1}"
+ [(set_attr "type" "sseicvt")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "SI")])
+
+(define_insn "avx512f_vcvttsd2usiq"
+ [(set (match_operand:DI 0 "register_operand" "=r")
+ (unsigned_fix:DI
+ (vec_select:DF
+ (match_operand:V2DF 1 "nonimmediate_operand" "vm")
+ (parallel [(const_int 0)]))))]
+ "TARGET_AVX512F && TARGET_64BIT"
+ "vcvttsd2usi\t{%1, %0|%0, %1}"
+ [(set_attr "type" "sseicvt")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "DI")])
+
(define_insn "sse2_cvtsd2si"
[(set (match_operand:SI 0 "register_operand" "=r,r")
(unspec:SI
[(vec_select:DF
- (match_operand:V2DF 1 "nonimmediate_operand" "x,m")
+ (match_operand:V2DF 1 "nonimmediate_operand" "v,m")
(parallel [(const_int 0)]))]
UNSPEC_FIX_NOTRUNC))]
"TARGET_SSE2"
@@ -2828,7 +3400,7 @@
[(set (match_operand:DI 0 "register_operand" "=r,r")
(unspec:DI
[(vec_select:DF
- (match_operand:V2DF 1 "nonimmediate_operand" "x,m")
+ (match_operand:V2DF 1 "nonimmediate_operand" "v,m")
(parallel [(const_int 0)]))]
UNSPEC_FIX_NOTRUNC))]
"TARGET_SSE2 && TARGET_64BIT"
@@ -2858,7 +3430,7 @@
[(set (match_operand:SI 0 "register_operand" "=r,r")
(fix:SI
(vec_select:DF
- (match_operand:V2DF 1 "nonimmediate_operand" "x,m")
+ (match_operand:V2DF 1 "nonimmediate_operand" "v,m")
(parallel [(const_int 0)]))))]
"TARGET_SSE2"
"%vcvttsd2si\t{%1, %0|%0, %q1}"
@@ -2875,7 +3447,7 @@
[(set (match_operand:DI 0 "register_operand" "=r,r")
(fix:DI
(vec_select:DF
- (match_operand:V2DF 1 "nonimmediate_operand" "x,m")
+ (match_operand:V2DF 1 "nonimmediate_operand" "v,m")
(parallel [(const_int 0)]))))]
"TARGET_SSE2 && TARGET_64BIT"
"%vcvttsd2si{q}\t{%1, %0|%0, %q1}"
@@ -2887,14 +3459,44 @@
(set_attr "prefix" "maybe_vex")
(set_attr "mode" "DI")])
-(define_insn "floatv4siv4df2"
- [(set (match_operand:V4DF 0 "register_operand" "=x")
- (float:V4DF (match_operand:V4SI 1 "nonimmediate_operand" "xm")))]
+;; For float<si2dfmode><mode>2 insn pattern
+(define_mode_attr si2dfmode
+ [(V8DF "V8SI") (V4DF "V4SI")])
+(define_mode_attr si2dfmodelower
+ [(V8DF "v8si") (V4DF "v4si")])
+
+(define_insn "float<si2dfmodelower><mode>2"
+ [(set (match_operand:VF2_512_256 0 "register_operand" "=v")
+ (float:VF2_512_256 (match_operand:<si2dfmode> 1 "nonimmediate_operand" "vm")))]
"TARGET_AVX"
"vcvtdq2pd\t{%1, %0|%0, %1}"
[(set_attr "type" "ssecvt")
- (set_attr "prefix" "vex")
- (set_attr "mode" "V4DF")])
+ (set_attr "prefix" "maybe_vex")
+ (set_attr "mode" "<MODE>")])
+
+(define_insn "ufloatv8siv8df"
+ [(set (match_operand:V8DF 0 "register_operand" "=v")
+ (unsigned_float:V8DF (match_operand:V8SI 1 "nonimmediate_operand" "vm")))]
+ "TARGET_AVX512F"
+ "vcvtudq2pd\t{%1, %0|%0, %1}"
+ [(set_attr "type" "ssecvt")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "V8DF")])
+
+(define_insn "avx512f_cvtdq2pd512_2"
+ [(set (match_operand:V8DF 0 "register_operand" "=v")
+ (float:V8DF
+ (vec_select:V8SI
+ (match_operand:V16SI 1 "nonimmediate_operand" "vm")
+ (parallel [(const_int 0) (const_int 1)
+ (const_int 2) (const_int 3)
+ (const_int 4) (const_int 5)
+ (const_int 6) (const_int 7)]))))]
+ "TARGET_AVX"
+ "vcvtdq2pd\t{%t1, %0|%0, %t1}"
+ [(set_attr "type" "ssecvt")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "V8DF")])
(define_insn "avx_cvtdq2pd256_2"
[(set (match_operand:V4DF 0 "register_operand" "=x")
@@ -2921,6 +3523,16 @@
(set_attr "prefix" "maybe_vex")
(set_attr "mode" "V2DF")])
+(define_insn "avx512f_cvtpd2dq512"
+ [(set (match_operand:V8SI 0 "register_operand" "=v")
+ (unspec:V8SI [(match_operand:V8DF 1 "nonimmediate_operand" "vm")]
+ UNSPEC_FIX_NOTRUNC))]
+ "TARGET_AVX512F"
+ "vcvtpd2dq\t{%1, %0|%0, %1}"
+ [(set_attr "type" "ssecvt")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "OI")])
+
(define_insn "avx_cvtpd2dq256"
[(set (match_operand:V4SI 0 "register_operand" "=x")
(unspec:V4SI [(match_operand:V4DF 1 "nonimmediate_operand" "xm")]
@@ -2984,6 +3596,26 @@
(set_attr "athlon_decode" "vector")
(set_attr "bdver1_decode" "double")])
+(define_insn "avx512f_ufix_notruncv8dfv8si"
+ [(set (match_operand:V8SI 0 "register_operand" "=v")
+ (unspec:V8SI
+ [(match_operand:V8DF 1 "nonimmediate_operand" "vm")]
+ UNSPEC_UNSIGNED_FIX_NOTRUNC))]
+ "TARGET_AVX512F"
+ "vcvtpd2udq\t{%1, %0|%0, %1}"
+ [(set_attr "type" "ssecvt")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "OI")])
+
+(define_insn "<fixsuffix>fix_truncv8dfv8si2"
+ [(set (match_operand:V8SI 0 "register_operand" "=v")
+ (any_fix:V8SI (match_operand:V8DF 1 "nonimmediate_operand" "vm")))]
+ "TARGET_AVX512F"
+ "vcvttpd2<fixsuffix>dq\t{%1, %0|%0, %1}"
+ [(set_attr "type" "ssecvt")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "OI")])
+
(define_insn "fix_truncv4dfv4si2"
[(set (match_operand:V4SI 0 "register_operand" "=x")
(fix:V4SI (match_operand:V4DF 1 "nonimmediate_operand" "xm")))]
@@ -3085,6 +3717,16 @@
(set_attr "prefix" "orig,orig,vex")
(set_attr "mode" "DF")])
+(define_insn "avx512f_cvtpd2ps512"
+ [(set (match_operand:V8SF 0 "register_operand" "=v")
+ (float_truncate:V8SF
+ (match_operand:V8DF 1 "nonimmediate_operand" "vm")))]
+ "TARGET_AVX512F"
+ "vcvtpd2ps\t{%1, %0|%0, %1}"
+ [(set_attr "type" "ssecvt")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "V8SF")])
+
(define_insn "avx_cvtpd2ps256"
[(set (match_operand:V4SF 0 "register_operand" "=x")
(float_truncate:V4SF
@@ -3126,15 +3768,19 @@
(set_attr "prefix" "maybe_vex")
(set_attr "mode" "V4SF")])
-(define_insn "avx_cvtps2pd256"
- [(set (match_operand:V4DF 0 "register_operand" "=x")
- (float_extend:V4DF
- (match_operand:V4SF 1 "nonimmediate_operand" "xm")))]
+;; For <sse2_avx_avx512f>_cvtps2pd<avxsizesuffix> insn pattern
+(define_mode_attr sf2dfmode
+ [(V8DF "V8SF") (V4DF "V4SF")])
+
+(define_insn "<sse2_avx_avx512f>_cvtps2pd<avxsizesuffix>"
+ [(set (match_operand:VF2_512_256 0 "register_operand" "=v")
+ (float_extend:VF2_512_256
+ (match_operand:<sf2dfmode> 1 "nonimmediate_operand" "vm")))]
"TARGET_AVX"
"vcvtps2pd\t{%1, %0|%0, %1}"
[(set_attr "type" "ssecvt")
- (set_attr "prefix" "vex")
- (set_attr "mode" "V4DF")])
+ (set_attr "prefix" "maybe_vex")
+ (set_attr "mode" "<MODE>")])
(define_insn "*avx_cvtps2pd256_2"
[(set (match_operand:V4DF 0 "register_operand" "=x")
@@ -3149,6 +3795,21 @@
(set_attr "prefix" "vex")
(set_attr "mode" "V4DF")])
+(define_insn "vec_unpacks_lo_v16sf"
+ [(set (match_operand:V8DF 0 "register_operand" "=v")
+ (float_extend:V8DF
+ (vec_select:V8SF
+ (match_operand:V16SF 1 "nonimmediate_operand" "vm")
+ (parallel [(const_int 0) (const_int 1)
+ (const_int 2) (const_int 3)
+ (const_int 4) (const_int 5)
+ (const_int 6) (const_int 7)]))))]
+ "TARGET_AVX512F"
+ "vcvtps2pd\t{%t1, %0|%0, %t1}"
+ [(set_attr "type" "ssecvt")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "V8DF")])
+
(define_insn "sse2_cvtps2pd"
[(set (match_operand:V2DF 0 "register_operand" "=x")
(float_extend:V2DF
@@ -3193,6 +3854,20 @@
"TARGET_AVX"
"operands[2] = gen_reg_rtx (V4SFmode);")
+(define_expand "vec_unpacks_hi_v16sf"
+ [(set (match_dup 2)
+ (vec_select:V8SF
+ (match_operand:V16SF 1 "nonimmediate_operand")
+ (parallel [(const_int 8) (const_int 9)
+ (const_int 10) (const_int 11)
+ (const_int 12) (const_int 13)
+ (const_int 14) (const_int 15)])))
+ (set (match_operand:V8DF 0 "register_operand")
+ (float_extend:V8DF
+ (match_dup 2)))]
+"TARGET_AVX512F"
+"operands[2] = gen_reg_rtx (V8SFmode);")
+
(define_expand "vec_unpacks_lo_v4sf"
[(set (match_operand:V2DF 0 "register_operand")
(float_extend:V2DF
@@ -3211,11 +3886,12 @@
"TARGET_AVX")
(define_mode_attr sseunpackfltmode
- [(V8HI "V4SF") (V4SI "V2DF") (V16HI "V8SF") (V8SI "V4DF")])
+ [(V8HI "V4SF") (V4SI "V2DF") (V16HI "V8SF")
+ (V8SI "V4DF") (V32HI "V16SF") (V16SI "V8DF")])
(define_expand "vec_unpacks_float_hi_<mode>"
[(match_operand:<sseunpackfltmode> 0 "register_operand")
- (match_operand:VI2_AVX2 1 "register_operand")]
+ (match_operand:VI2_AVX512F 1 "register_operand")]
"TARGET_SSE2"
{
rtx tmp = gen_reg_rtx (<sseunpackmode>mode);
@@ -3228,7 +3904,7 @@
(define_expand "vec_unpacks_float_lo_<mode>"
[(match_operand:<sseunpackfltmode> 0 "register_operand")
- (match_operand:VI2_AVX2 1 "register_operand")]
+ (match_operand:VI2_AVX512F 1 "register_operand")]
"TARGET_SSE2"
{
rtx tmp = gen_reg_rtx (<sseunpackmode>mode);
@@ -3241,7 +3917,7 @@
(define_expand "vec_unpacku_float_hi_<mode>"
[(match_operand:<sseunpackfltmode> 0 "register_operand")
- (match_operand:VI2_AVX2 1 "register_operand")]
+ (match_operand:VI2_AVX512F 1 "register_operand")]
"TARGET_SSE2"
{
rtx tmp = gen_reg_rtx (<sseunpackmode>mode);
@@ -3254,7 +3930,7 @@
(define_expand "vec_unpacku_float_lo_<mode>"
[(match_operand:<sseunpackfltmode> 0 "register_operand")
- (match_operand:VI2_AVX2 1 "register_operand")]
+ (match_operand:VI2_AVX512F 1 "register_operand")]
"TARGET_SSE2"
{
rtx tmp = gen_reg_rtx (<sseunpackmode>mode);
@@ -3308,6 +3984,31 @@
(const_int 2) (const_int 3)]))))]
"TARGET_AVX")
+(define_expand "vec_unpacks_float_hi_v16si"
+ [(set (match_dup 2)
+ (vec_select:V8SI
+ (match_operand:V16SI 1 "nonimmediate_operand")
+ (parallel [(const_int 8) (const_int 9)
+ (const_int 10) (const_int 11)
+ (const_int 12) (const_int 13)
+ (const_int 14) (const_int 15)])))
+ (set (match_operand:V8DF 0 "register_operand")
+ (float:V8DF
+ (match_dup 2)))]
+ "TARGET_AVX512F"
+ "operands[2] = gen_reg_rtx (V8SImode);")
+
+(define_expand "vec_unpacks_float_lo_v16si"
+ [(set (match_operand:V8DF 0 "register_operand")
+ (float:V8DF
+ (vec_select:V8SI
+ (match_operand:V16SI 1 "nonimmediate_operand")
+ (parallel [(const_int 0) (const_int 1)
+ (const_int 2) (const_int 3)
+ (const_int 4) (const_int 5)
+ (const_int 6) (const_int 7)]))))]
+ "TARGET_AVX512F")
+
(define_expand "vec_unpacku_float_hi_v4si"
[(set (match_dup 5)
(vec_select:V4SI
@@ -3425,21 +4126,21 @@
DONE;
})
-(define_expand "vec_pack_trunc_v4df"
+(define_expand "vec_pack_trunc_<mode>"
[(set (match_dup 3)
- (float_truncate:V4SF
- (match_operand:V4DF 1 "nonimmediate_operand")))
+ (float_truncate:<sf2dfmode>
+ (match_operand:VF2_512_256 1 "nonimmediate_operand")))
(set (match_dup 4)
- (float_truncate:V4SF
- (match_operand:V4DF 2 "nonimmediate_operand")))
- (set (match_operand:V8SF 0 "register_operand")
- (vec_concat:V8SF
+ (float_truncate:<sf2dfmode>
+ (match_operand:VF2_512_256 2 "nonimmediate_operand")))
+ (set (match_operand:<ssePSmode> 0 "register_operand")
+ (vec_concat:<ssePSmode>
(match_dup 3)
(match_dup 4)))]
"TARGET_AVX"
{
- operands[3] = gen_reg_rtx (V4SFmode);
- operands[4] = gen_reg_rtx (V4SFmode);
+ operands[3] = gen_reg_rtx (<sf2dfmode>mode);
+ operands[4] = gen_reg_rtx (<sf2dfmode>mode);
})
(define_expand "vec_pack_trunc_v2df"
@@ -3470,6 +4171,23 @@
DONE;
})
+(define_expand "vec_pack_sfix_trunc_v8df"
+ [(match_operand:V16SI 0 "register_operand")
+ (match_operand:V8DF 1 "nonimmediate_operand")
+ (match_operand:V8DF 2 "nonimmediate_operand")]
+ "TARGET_AVX512F"
+{
+ rtx r1, r2;
+
+ r1 = gen_reg_rtx (V8SImode);
+ r2 = gen_reg_rtx (V8SImode);
+
+ emit_insn (gen_fix_truncv8dfv8si2 (r1, operands[1]));
+ emit_insn (gen_fix_truncv8dfv8si2 (r2, operands[2]));
+ emit_insn (gen_avx_vec_concatv16si (operands[0], r1, r2));
+ DONE;
+})
+
(define_expand "vec_pack_sfix_trunc_v4df"
[(match_operand:V8SI 0 "register_operand")
(match_operand:V4DF 1 "nonimmediate_operand")
@@ -3493,7 +4211,7 @@
(match_operand:V2DF 2 "nonimmediate_operand")]
"TARGET_SSE2"
{
- rtx tmp0, tmp1;
+ rtx tmp0, tmp1, tmp2;
if (TARGET_AVX && !TARGET_PREFER_AVX128)
{
@@ -3507,24 +4225,25 @@
{
tmp0 = gen_reg_rtx (V4SImode);
tmp1 = gen_reg_rtx (V4SImode);
+ tmp2 = gen_reg_rtx (V2DImode);
emit_insn (gen_sse2_cvttpd2dq (tmp0, operands[1]));
emit_insn (gen_sse2_cvttpd2dq (tmp1, operands[2]));
- emit_insn
- (gen_vec_interleave_lowv2di (gen_lowpart (V2DImode, operands[0]),
- gen_lowpart (V2DImode, tmp0),
- gen_lowpart (V2DImode, tmp1)));
+ emit_insn (gen_vec_interleave_lowv2di (tmp2,
+ gen_lowpart (V2DImode, tmp0),
+ gen_lowpart (V2DImode, tmp1)));
+ emit_move_insn (operands[0], gen_lowpart (V4SImode, tmp2));
}
DONE;
})
(define_mode_attr ssepackfltmode
- [(V4DF "V8SI") (V2DF "V4SI")])
+ [(V8DF "V16SI") (V4DF "V8SI") (V2DF "V4SI")])
(define_expand "vec_pack_ufix_trunc_<mode>"
[(match_operand:<ssepackfltmode> 0 "register_operand")
- (match_operand:VF2 1 "register_operand")
- (match_operand:VF2 2 "register_operand")]
+ (match_operand:VF2_128_256 1 "register_operand")
+ (match_operand:VF2_128_256 2 "register_operand")]
"TARGET_SSE2"
{
rtx tmp[7];
@@ -3574,7 +4293,7 @@
(match_operand:V2DF 2 "nonimmediate_operand")]
"TARGET_SSE2"
{
- rtx tmp0, tmp1;
+ rtx tmp0, tmp1, tmp2;
if (TARGET_AVX && !TARGET_PREFER_AVX128)
{
@@ -3588,13 +4307,14 @@
{
tmp0 = gen_reg_rtx (V4SImode);
tmp1 = gen_reg_rtx (V4SImode);
+ tmp2 = gen_reg_rtx (V2DImode);
emit_insn (gen_sse2_cvtpd2dq (tmp0, operands[1]));
emit_insn (gen_sse2_cvtpd2dq (tmp1, operands[2]));
- emit_insn
- (gen_vec_interleave_lowv2di (gen_lowpart (V2DImode, operands[0]),
- gen_lowpart (V2DImode, tmp0),
- gen_lowpart (V2DImode, tmp1)));
+ emit_insn (gen_vec_interleave_lowv2di (tmp2,
+ gen_lowpart (V2DImode, tmp0),
+ gen_lowpart (V2DImode, tmp1)));
+ emit_move_insn (operands[0], gen_lowpart (V4SImode, tmp2));
}
DONE;
})
@@ -3695,6 +4415,26 @@
(set_attr "prefix" "orig,vex,orig,vex,maybe_vex")
(set_attr "mode" "V4SF,V4SF,V2SF,V2SF,V2SF")])
+(define_insn "avx512f_unpckhps512"
+ [(set (match_operand:V16SF 0 "register_operand" "=v")
+ (vec_select:V16SF
+ (vec_concat:V32SF
+ (match_operand:V16SF 1 "register_operand" "v")
+ (match_operand:V16SF 2 "nonimmediate_operand" "vm"))
+ (parallel [(const_int 2) (const_int 18)
+ (const_int 3) (const_int 19)
+ (const_int 6) (const_int 22)
+ (const_int 7) (const_int 23)
+ (const_int 10) (const_int 26)
+ (const_int 11) (const_int 27)
+ (const_int 14) (const_int 30)
+ (const_int 15) (const_int 31)])))]
+ "TARGET_AVX512F"
+ "vunpckhps\t{%2, %1, %0|%0, %1, %2}"
+ [(set_attr "type" "sselog")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "V16SF")])
+
;; Recall that the 256-bit unpck insns only shuffle within their lanes.
(define_insn "avx_unpckhps256"
[(set (match_operand:V8SF 0 "register_operand" "=x")
@@ -3763,6 +4503,26 @@
(set_attr "prefix" "orig,vex")
(set_attr "mode" "V4SF")])
+(define_insn "avx512f_unpcklps512"
+ [(set (match_operand:V16SF 0 "register_operand" "=v")
+ (vec_select:V16SF
+ (vec_concat:V32SF
+ (match_operand:V16SF 1 "register_operand" "v")
+ (match_operand:V16SF 2 "nonimmediate_operand" "vm"))
+ (parallel [(const_int 0) (const_int 16)
+ (const_int 1) (const_int 17)
+ (const_int 4) (const_int 20)
+ (const_int 5) (const_int 21)
+ (const_int 8) (const_int 24)
+ (const_int 9) (const_int 25)
+ (const_int 12) (const_int 28)
+ (const_int 13) (const_int 29)])))]
+ "TARGET_AVX512F"
+ "vunpcklps\t{%2, %1, %0|%0, %1, %2}"
+ [(set_attr "type" "sselog")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "V16SF")])
+
;; Recall that the 256-bit unpck insns only shuffle within their lanes.
(define_insn "avx_unpcklps256"
[(set (match_operand:V8SF 0 "register_operand" "=x")
@@ -3866,6 +4626,26 @@
(set_attr "prefix" "maybe_vex")
(set_attr "mode" "V4SF")])
+(define_insn "avx512f_movshdup512"
+ [(set (match_operand:V16SF 0 "register_operand" "=v")
+ (vec_select:V16SF
+ (vec_concat:V32SF
+ (match_operand:V16SF 1 "nonimmediate_operand" "vm")
+ (match_dup 1))
+ (parallel [(const_int 1) (const_int 1)
+ (const_int 3) (const_int 3)
+ (const_int 5) (const_int 5)
+ (const_int 7) (const_int 7)
+ (const_int 9) (const_int 9)
+ (const_int 11) (const_int 11)
+ (const_int 13) (const_int 13)
+ (const_int 15) (const_int 15)])))]
+ "TARGET_AVX512F"
+ "vmovshdup\t{%1, %0|%0, %1}"
+ [(set_attr "type" "sse")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "V16SF")])
+
(define_insn "avx_movsldup256"
[(set (match_operand:V8SF 0 "register_operand" "=x")
(vec_select:V8SF
@@ -3899,6 +4679,26 @@
(set_attr "prefix" "maybe_vex")
(set_attr "mode" "V4SF")])
+(define_insn "avx512f_movsldup512"
+ [(set (match_operand:V16SF 0 "register_operand" "=v")
+ (vec_select:V16SF
+ (vec_concat:V32SF
+ (match_operand:V16SF 1 "nonimmediate_operand" "vm")
+ (match_dup 1))
+ (parallel [(const_int 0) (const_int 0)
+ (const_int 2) (const_int 2)
+ (const_int 4) (const_int 4)
+ (const_int 6) (const_int 6)
+ (const_int 8) (const_int 8)
+ (const_int 10) (const_int 10)
+ (const_int 12) (const_int 12)
+ (const_int 14) (const_int 14)])))]
+ "TARGET_AVX512F"
+ "vmovsldup\t{%1, %0|%0, %1}"
+ [(set_attr "type" "sse")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "V16SF")])
+
(define_expand "avx_shufps256"
[(match_operand:V8SF 0 "register_operand")
(match_operand:V8SF 1 "register_operand")
@@ -4128,8 +4928,8 @@
(set_attr "mode" "SF")])
(define_insn "avx2_vec_dup<mode>"
- [(set (match_operand:VF1 0 "register_operand" "=x")
- (vec_duplicate:VF1
+ [(set (match_operand:VF1_128_256 0 "register_operand" "=x")
+ (vec_duplicate:VF1_128_256
(vec_select:SF
(match_operand:V4SF 1 "register_operand" "x")
(parallel [(const_int 0)]))))]
@@ -4428,6 +5228,86 @@
operands[1] = adjust_address (operands[1], SFmode, INTVAL (operands[2]) * 4);
})
+(define_insn "avx512f_vextract<shuffletype>32x4_1"
+ [(set (match_operand:<ssequartermode> 0 "nonimmediate_operand" "=vm")
+ (vec_select:<ssequartermode>
+ (match_operand:V16FI 1 "register_operand" "v")
+ (parallel [(match_operand 2 "const_0_to_15_operand")
+ (match_operand 3 "const_0_to_15_operand")
+ (match_operand 4 "const_0_to_15_operand")
+ (match_operand 5 "const_0_to_15_operand")])))]
+ "TARGET_AVX512F && (INTVAL (operands[2]) = INTVAL (operands[3]) - 1)
+ && (INTVAL (operands[3]) = INTVAL (operands[4]) - 1)
+ && (INTVAL (operands[4]) = INTVAL (operands[5]) - 1)"
+{
+ operands[2] = GEN_INT ((INTVAL (operands[2])) >> 2);
+ return "vextract<shuffletype>32x4\t{%2, %1, %0|%0, %1, %2}";
+}
+ [(set_attr "type" "sselog")
+ (set_attr "prefix_extra" "1")
+ (set_attr "length_immediate" "1")
+ (set (attr "memory")
+ (if_then_else (match_test "MEM_P (operands[0])")
+ (const_string "store")
+ (const_string "none")))
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "<sseinsnmode>")])
+
+(define_split
+ [(set (match_operand:<ssehalfvecmode> 0 "nonimmediate_operand")
+ (vec_select:<ssehalfvecmode>
+ (match_operand:V8FI 1 "nonimmediate_operand")
+ (parallel [(const_int 0) (const_int 1)
+ (const_int 2) (const_int 3)])))]
+ "TARGET_AVX512F && !(MEM_P (operands[0]) && MEM_P (operands[1]))
+ && reload_completed"
+ [(const_int 0)]
+{
+ rtx op1 = operands[1];
+ if (REG_P (op1))
+ op1 = gen_rtx_REG (<ssehalfvecmode>mode, REGNO (op1));
+ else
+ op1 = gen_lowpart (<ssehalfvecmode>mode, op1);
+ emit_move_insn (operands[0], op1);
+ DONE;
+})
+
+(define_insn "vec_extract_lo_<mode>"
+ [(set (match_operand:<ssehalfvecmode> 0 "nonimmediate_operand" "=vm")
+ (vec_select:<ssehalfvecmode>
+ (match_operand:V8FI 1 "nonimmediate_operand" "vm")
+ (parallel [(const_int 0) (const_int 1)
+ (const_int 2) (const_int 3)])))]
+ "TARGET_AVX512F && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
+ "#"
+ [(set_attr "type" "sselog")
+ (set_attr "prefix_extra" "1")
+ (set_attr "length_immediate" "1")
+ (set (attr "memory")
+ (if_then_else (match_test "MEM_P (operands[0])")
+ (const_string "store")
+ (const_string "none")))
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "<sseinsnmode>")])
+
+(define_insn "vec_extract_hi_<mode>"
+ [(set (match_operand:<ssehalfvecmode> 0 "nonimmediate_operand" "=vm")
+ (vec_select:<ssehalfvecmode>
+ (match_operand:V8FI 1 "register_operand" "v")
+ (parallel [(const_int 4) (const_int 5)
+ (const_int 6) (const_int 7)])))]
+ "TARGET_AVX512F"
+ "vextract<shuffletype>64x4\t{$0x1, %1, %0|%0, %1, 0x1}"
+ [(set_attr "type" "sselog")
+ (set_attr "prefix_extra" "1")
+ (set_attr "length_immediate" "1")
+ (set (attr "memory")
+ (if_then_else (match_test "MEM_P (operands[0])")
+ (const_string "store")
+ (const_string "none")))
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "<sseinsnmode>")])
+
(define_expand "avx_vextractf128<mode>"
[(match_operand:<ssehalfvecmode> 0 "nonimmediate_operand")
(match_operand:V_256 1 "register_operand")
@@ -4453,6 +5333,45 @@
})
(define_insn_and_split "vec_extract_lo_<mode>"
+ [(set (match_operand:<ssehalfvecmode> 0 "nonimmediate_operand" "=v,m")
+ (vec_select:<ssehalfvecmode>
+ (match_operand:V16FI 1 "nonimmediate_operand" "vm,v")
+ (parallel [(const_int 0) (const_int 1)
+ (const_int 2) (const_int 3)
+ (const_int 4) (const_int 5)
+ (const_int 6) (const_int 7)])))]
+ "TARGET_AVX512F && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
+ "#"
+ "&& reload_completed"
+ [(const_int 0)]
+{
+ rtx op1 = operands[1];
+ if (REG_P (op1))
+ op1 = gen_rtx_REG (<ssehalfvecmode>mode, REGNO (op1));
+ else
+ op1 = gen_lowpart (<ssehalfvecmode>mode, op1);
+ emit_move_insn (operands[0], op1);
+ DONE;
+})
+
+(define_insn "vec_extract_hi_<mode>"
+ [(set (match_operand:<ssehalfvecmode> 0 "nonimmediate_operand" "=v,m")
+ (vec_select:<ssehalfvecmode>
+ (match_operand:V16FI 1 "nonimmediate_operand" "v,v")
+ (parallel [(const_int 8) (const_int 9)
+ (const_int 10) (const_int 11)
+ (const_int 12) (const_int 13)
+ (const_int 14) (const_int 15)])))]
+ "TARGET_AVX512F"
+ "vextracti64x4\t{$0x1, %1, %0|%0, %1, 0x1}"
+ [(set_attr "type" "sselog")
+ (set_attr "prefix_extra" "1")
+ (set_attr "length_immediate" "1")
+ (set_attr "memory" "none,store")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "XI")])
+
+(define_insn_and_split "vec_extract_lo_<mode>"
[(set (match_operand:<ssehalfvecmode> 0 "nonimmediate_operand" "=x,m")
(vec_select:<ssehalfvecmode>
(match_operand:VI8F_256 1 "nonimmediate_operand" "xm,x")
@@ -4514,6 +5433,50 @@
(set_attr "prefix" "vex")
(set_attr "mode" "<sseinsnmode>")])
+(define_insn_and_split "vec_extract_lo_v32hi"
+ [(set (match_operand:V16HI 0 "nonimmediate_operand" "=v,m")
+ (vec_select:V16HI
+ (match_operand:V32HI 1 "nonimmediate_operand" "vm,v")
+ (parallel [(const_int 0) (const_int 1)
+ (const_int 2) (const_int 3)
+ (const_int 4) (const_int 5)
+ (const_int 6) (const_int 7)
+ (const_int 8) (const_int 9)
+ (const_int 10) (const_int 11)
+ (const_int 12) (const_int 13)
+ (const_int 14) (const_int 15)])))]
+ "TARGET_AVX512F && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
+ "#"
+ "&& reload_completed"
+ [(set (match_dup 0) (match_dup 1))]
+{
+ if (REG_P (operands[1]))
+ operands[1] = gen_rtx_REG (V16HImode, REGNO (operands[1]));
+ else
+ operands[1] = adjust_address (operands[1], V16HImode, 0);
+})
+
+(define_insn "vec_extract_hi_v32hi"
+ [(set (match_operand:V16HI 0 "nonimmediate_operand" "=v,m")
+ (vec_select:V16HI
+ (match_operand:V32HI 1 "nonimmediate_operand" "v,v")
+ (parallel [(const_int 16) (const_int 17)
+ (const_int 18) (const_int 19)
+ (const_int 20) (const_int 21)
+ (const_int 22) (const_int 23)
+ (const_int 24) (const_int 25)
+ (const_int 26) (const_int 27)
+ (const_int 28) (const_int 29)
+ (const_int 30) (const_int 31)])))]
+ "TARGET_AVX512F"
+ "vextracti64x4\t{$0x1, %1, %0|%0, %1, 0x1}"
+ [(set_attr "type" "sselog")
+ (set_attr "prefix_extra" "1")
+ (set_attr "length_immediate" "1")
+ (set_attr "memory" "none,store")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "XI")])
+
(define_insn_and_split "vec_extract_lo_v16hi"
[(set (match_operand:V8HI 0 "nonimmediate_operand" "=x,m")
(vec_select:V8HI
@@ -4550,6 +5513,66 @@
(set_attr "prefix" "vex")
(set_attr "mode" "OI")])
+(define_insn_and_split "vec_extract_lo_v64qi"
+ [(set (match_operand:V32QI 0 "nonimmediate_operand" "=v,m")
+ (vec_select:V32QI
+ (match_operand:V64QI 1 "nonimmediate_operand" "vm,v")
+ (parallel [(const_int 0) (const_int 1)
+ (const_int 2) (const_int 3)
+ (const_int 4) (const_int 5)
+ (const_int 6) (const_int 7)
+ (const_int 8) (const_int 9)
+ (const_int 10) (const_int 11)
+ (const_int 12) (const_int 13)
+ (const_int 14) (const_int 15)
+ (const_int 16) (const_int 17)
+ (const_int 18) (const_int 19)
+ (const_int 20) (const_int 21)
+ (const_int 22) (const_int 23)
+ (const_int 24) (const_int 25)
+ (const_int 26) (const_int 27)
+ (const_int 28) (const_int 29)
+ (const_int 30) (const_int 31)])))]
+ "TARGET_AVX512F && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
+ "#"
+ "&& reload_completed"
+ [(set (match_dup 0) (match_dup 1))]
+{
+ if (REG_P (operands[1]))
+ operands[1] = gen_rtx_REG (V32QImode, REGNO (operands[1]));
+ else
+ operands[1] = adjust_address (operands[1], V32QImode, 0);
+})
+
+(define_insn "vec_extract_hi_v64qi"
+ [(set (match_operand:V32QI 0 "nonimmediate_operand" "=v,m")
+ (vec_select:V32QI
+ (match_operand:V64QI 1 "nonimmediate_operand" "v,v")
+ (parallel [(const_int 32) (const_int 33)
+ (const_int 34) (const_int 35)
+ (const_int 36) (const_int 37)
+ (const_int 38) (const_int 39)
+ (const_int 40) (const_int 41)
+ (const_int 42) (const_int 43)
+ (const_int 44) (const_int 45)
+ (const_int 46) (const_int 47)
+ (const_int 48) (const_int 49)
+ (const_int 50) (const_int 51)
+ (const_int 52) (const_int 53)
+ (const_int 54) (const_int 55)
+ (const_int 56) (const_int 57)
+ (const_int 58) (const_int 59)
+ (const_int 60) (const_int 61)
+ (const_int 62) (const_int 63)])))]
+ "TARGET_AVX512F"
+ "vextracti64x4\t{$0x1, %1, %0|%0, %1, 0x1}"
+ [(set_attr "type" "sselog")
+ (set_attr "prefix_extra" "1")
+ (set_attr "length_immediate" "1")
+ (set_attr "memory" "none,store")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "XI")])
+
(define_insn_and_split "vec_extract_lo_v32qi"
[(set (match_operand:V16QI 0 "nonimmediate_operand" "=x,m")
(vec_select:V16QI
@@ -4598,10 +5621,10 @@
(define_mode_iterator VEC_EXTRACT_MODE
[(V32QI "TARGET_AVX") V16QI
(V16HI "TARGET_AVX") V8HI
- (V8SI "TARGET_AVX") V4SI
- (V4DI "TARGET_AVX") V2DI
- (V8SF "TARGET_AVX") V4SF
- (V4DF "TARGET_AVX") V2DF])
+ (V16SI "TARGET_AVX512F") (V8SI "TARGET_AVX") V4SI
+ (V8DI "TARGET_AVX512F") (V4DI "TARGET_AVX") V2DI
+ (V16SF "TARGET_AVX512F") (V8SF "TARGET_AVX") V4SF
+ (V8DF "TARGET_AVX512F") (V4DF "TARGET_AVX") V2DF])
(define_expand "vec_extract<mode>"
[(match_operand:<ssescalarmode> 0 "register_operand")
@@ -4620,6 +5643,22 @@
;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+(define_insn "avx512f_unpckhpd512"
+ [(set (match_operand:V8DF 0 "register_operand" "=v")
+ (vec_select:V8DF
+ (vec_concat:V16DF
+ (match_operand:V8DF 1 "nonimmediate_operand" "v")
+ (match_operand:V8DF 2 "nonimmediate_operand" "vm"))
+ (parallel [(const_int 1) (const_int 9)
+ (const_int 3) (const_int 11)
+ (const_int 5) (const_int 13)
+ (const_int 7) (const_int 15)])))]
+ "TARGET_AVX512F"
+ "vunpckhpd\t{%2, %1, %0|%0, %1, %2}"
+ [(set_attr "type" "sselog")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "V8DF")])
+
;; Recall that the 256-bit unpck insns only shuffle within their lanes.
(define_insn "avx_unpckhpd256"
[(set (match_operand:V4DF 0 "register_operand" "=x")
@@ -4700,6 +5739,48 @@
(set_attr "prefix" "orig,vex,maybe_vex,orig,vex,maybe_vex")
(set_attr "mode" "V2DF,V2DF,DF,V1DF,V1DF,V1DF")])
+(define_expand "avx512f_movddup512"
+ [(set (match_operand:V8DF 0 "register_operand")
+ (vec_select:V8DF
+ (vec_concat:V16DF
+ (match_operand:V8DF 1 "nonimmediate_operand")
+ (match_dup 1))
+ (parallel [(const_int 0) (const_int 8)
+ (const_int 2) (const_int 10)
+ (const_int 4) (const_int 12)
+ (const_int 6) (const_int 14)])))]
+ "TARGET_AVX512F")
+
+(define_expand "avx512f_unpcklpd512"
+ [(set (match_operand:V8DF 0 "register_operand")
+ (vec_select:V8DF
+ (vec_concat:V16DF
+ (match_operand:V8DF 1 "register_operand")
+ (match_operand:V8DF 2 "nonimmediate_operand"))
+ (parallel [(const_int 0) (const_int 8)
+ (const_int 2) (const_int 10)
+ (const_int 4) (const_int 12)
+ (const_int 6) (const_int 14)])))]
+ "TARGET_AVX512F")
+
+(define_insn "*avx512f_unpcklpd512"
+ [(set (match_operand:V8DF 0 "register_operand" "=v,v")
+ (vec_select:V8DF
+ (vec_concat:V16DF
+ (match_operand:V8DF 1 "nonimmediate_operand" "vm, v")
+ (match_operand:V8DF 2 "nonimmediate_operand" "1 ,vm"))
+ (parallel [(const_int 0) (const_int 8)
+ (const_int 2) (const_int 10)
+ (const_int 4) (const_int 12)
+ (const_int 6) (const_int 14)])))]
+ "TARGET_AVX512F"
+ "@
+ vmovddup\t{%1, %0|%0, %1}
+ vunpcklpd\t{%2, %1, %0|%0, %1, %2}"
+ [(set_attr "type" "sselog")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "V8DF")])
+
;; Recall that the 256-bit unpck insns only shuffle within their lanes.
(define_expand "avx_movddup256"
[(set (match_operand:V4DF 0 "register_operand")
@@ -4832,6 +5913,218 @@
operands[1] = adjust_address (operands[1], DFmode, INTVAL (operands[2]) * 8);
})
+(define_insn "avx512f_vmscalef<mode>"
+ [(set (match_operand:VF_128 0 "register_operand" "=v")
+ (vec_merge:VF_128
+ (unspec:VF_128 [(match_operand:VF_128 1 "register_operand" "v")
+ (match_operand:VF_128 2 "nonimmediate_operand" "vm")]
+ UNSPEC_SCALEF)
+ (match_dup 1)
+ (const_int 1)))]
+ "TARGET_AVX512F"
+ "%vscalef<ssescalarmodesuffix>\t{%2, %1, %0|%0, %1, %2}"
+ [(set_attr "prefix" "evex")
+ (set_attr "mode" "<ssescalarmode>")])
+
+(define_insn "avx512f_scalef<mode>"
+ [(set (match_operand:VF_512 0 "register_operand" "=v")
+ (unspec:VF_512 [(match_operand:VF_512 1 "register_operand" "v")
+ (match_operand:VF_512 2 "nonimmediate_operand" "vm")]
+ UNSPEC_SCALEF))]
+ "TARGET_AVX512F"
+ "%vscalef<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
+ [(set_attr "prefix" "evex")
+ (set_attr "mode" "<MODE>")])
+
+(define_insn "avx512f_vternlog<mode>"
+ [(set (match_operand:VI48_512 0 "register_operand" "=v")
+ (unspec:VI48_512
+ [(match_operand:VI48_512 1 "register_operand" "0")
+ (match_operand:VI48_512 2 "register_operand" "v")
+ (match_operand:VI48_512 3 "nonimmediate_operand" "vm")
+ (match_operand:SI 4 "const_0_to_255_operand")]
+ UNSPEC_VTERNLOG))]
+ "TARGET_AVX512F"
+ "vpternlog<ssemodesuffix>\t{%4, %3, %2, %0|%0, %2, %3, %4}"
+ [(set_attr "type" "sselog")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "<sseinsnmode>")])
+
+(define_insn "avx512f_getexp<mode>"
+ [(set (match_operand:VF_512 0 "register_operand" "=v")
+ (unspec:VF_512 [(match_operand:VF_512 1 "nonimmediate_operand" "vm")]
+ UNSPEC_GETEXP))]
+ "TARGET_AVX512F"
+ "vgetexp<ssemodesuffix>\t{%1, %0|%0, %1}";
+ [(set_attr "prefix" "evex")
+ (set_attr "mode" "<MODE>")])
+
+(define_insn "avx512f_sgetexp<mode>"
+ [(set (match_operand:VF_128 0 "register_operand" "=v")
+ (vec_merge:VF_128
+ (unspec:VF_128 [(match_operand:VF_128 1 "register_operand" "v")
+ (match_operand:VF_128 2 "nonimmediate_operand" "vm")]
+ UNSPEC_GETEXP)
+ (match_dup 1)
+ (const_int 1)))]
+ "TARGET_AVX512F"
+ "vgetexp<ssescalarmodesuffix>\t{%2, %1, %0|%0, %1, %2}";
+ [(set_attr "prefix" "evex")
+ (set_attr "mode" "<ssescalarmode>")])
+
+(define_insn "avx512f_align<mode>"
+ [(set (match_operand:VI48_512 0 "register_operand" "=v")
+ (unspec:VI48_512 [(match_operand:VI48_512 1 "register_operand" "v")
+ (match_operand:VI48_512 2 "nonimmediate_operand" "vm")
+ (match_operand:SI 3 "const_0_to_255_operand")]
+ UNSPEC_ALIGN))]
+ "TARGET_AVX512F"
+ "valign<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}";
+ [(set_attr "prefix" "evex")
+ (set_attr "mode" "<sseinsnmode>")])
+
+(define_insn "avx512f_fixupimm<mode>"
+ [(set (match_operand:VF_512 0 "register_operand" "=v")
+ (unspec:VF_512
+ [(match_operand:VF_512 1 "register_operand" "0")
+ (match_operand:VF_512 2 "register_operand" "v")
+ (match_operand:<sseintvecmode> 3 "nonimmediate_operand" "vm")
+ (match_operand:SI 4 "const_0_to_255_operand")]
+ UNSPEC_FIXUPIMM))]
+ "TARGET_AVX512F"
+ "vfixupimm<ssemodesuffix>\t{%4, %3, %2, %0|%0, %2, %3, %4}";
+ [(set_attr "prefix" "evex")
+ (set_attr "mode" "<MODE>")])
+
+(define_insn "avx512f_sfixupimm<mode>"
+ [(set (match_operand:VF_128 0 "register_operand" "=v")
+ (vec_merge:VF_128
+ (unspec:VF_128
+ [(match_operand:VF_128 1 "register_operand" "0")
+ (match_operand:VF_128 2 "register_operand" "v")
+ (match_operand:<sseintvecmode> 3 "nonimmediate_operand" "vm")
+ (match_operand:SI 4 "const_0_to_255_operand")]
+ UNSPEC_FIXUPIMM)
+ (match_dup 1)
+ (const_int 1)))]
+ "TARGET_AVX512F"
+ "vfixupimm<ssescalarmodesuffix>\t{%4, %3, %2, %0|%0, %2, %3, %4}";
+ [(set_attr "prefix" "evex")
+ (set_attr "mode" "<ssescalarmode>")])
+
+(define_insn "avx512f_rndscale<mode>"
+ [(set (match_operand:VF_512 0 "register_operand" "=v")
+ (unspec:VF_512
+ [(match_operand:VF_512 1 "nonimmediate_operand" "vm")
+ (match_operand:SI 2 "const_0_to_255_operand")]
+ UNSPEC_ROUND))]
+ "TARGET_AVX512F"
+ "vrndscale<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
+ [(set_attr "length_immediate" "1")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "<MODE>")])
+
+(define_insn "avx512f_rndscale<mode>"
+ [(set (match_operand:VF_128 0 "register_operand" "=v")
+ (vec_merge:VF_128
+ (unspec:VF_128
+ [(match_operand:VF_128 1 "register_operand" "v")
+ (match_operand:VF_128 2 "nonimmediate_operand" "vm")
+ (match_operand:SI 3 "const_0_to_255_operand")]
+ UNSPEC_ROUND)
+ (match_dup 1)
+ (const_int 1)))]
+ "TARGET_AVX512F"
+ "vrndscale<ssescalarmodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
+ [(set_attr "length_immediate" "1")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "<MODE>")])
+
+;; One bit in mask selects 2 elements.
+(define_insn "avx512f_shufps512_1"
+ [(set (match_operand:V16SF 0 "register_operand" "=v")
+ (vec_select:V16SF
+ (vec_concat:V32SF
+ (match_operand:V16SF 1 "register_operand" "v")
+ (match_operand:V16SF 2 "nonimmediate_operand" "vm"))
+ (parallel [(match_operand 3 "const_0_to_3_operand")
+ (match_operand 4 "const_0_to_3_operand")
+ (match_operand 5 "const_16_to_19_operand")
+ (match_operand 6 "const_16_to_19_operand")
+ (match_operand 7 "const_4_to_7_operand")
+ (match_operand 8 "const_4_to_7_operand")
+ (match_operand 9 "const_20_to_23_operand")
+ (match_operand 10 "const_20_to_23_operand")
+ (match_operand 11 "const_8_to_11_operand")
+ (match_operand 12 "const_8_to_11_operand")
+ (match_operand 13 "const_24_to_27_operand")
+ (match_operand 14 "const_24_to_27_operand")
+ (match_operand 15 "const_12_to_15_operand")
+ (match_operand 16 "const_12_to_15_operand")
+ (match_operand 17 "const_28_to_31_operand")
+ (match_operand 18 "const_28_to_31_operand")])))]
+ "TARGET_AVX512F
+ && (INTVAL (operands[3]) == (INTVAL (operands[7]) - 4)
+ && INTVAL (operands[4]) == (INTVAL (operands[8]) - 4)
+ && INTVAL (operands[5]) == (INTVAL (operands[9]) - 4)
+ && INTVAL (operands[6]) == (INTVAL (operands[10]) - 4)
+ && INTVAL (operands[3]) == (INTVAL (operands[11]) - 8)
+ && INTVAL (operands[4]) == (INTVAL (operands[12]) - 8)
+ && INTVAL (operands[5]) == (INTVAL (operands[13]) - 8)
+ && INTVAL (operands[6]) == (INTVAL (operands[14]) - 8)
+ && INTVAL (operands[3]) == (INTVAL (operands[15]) - 12)
+ && INTVAL (operands[4]) == (INTVAL (operands[16]) - 12)
+ && INTVAL (operands[5]) == (INTVAL (operands[17]) - 12)
+ && INTVAL (operands[6]) == (INTVAL (operands[18]) - 12))"
+{
+ int mask;
+ mask = INTVAL (operands[3]);
+ mask |= INTVAL (operands[4]) << 2;
+ mask |= (INTVAL (operands[5]) - 16) << 4;
+ mask |= (INTVAL (operands[6]) - 16) << 6;
+ operands[3] = GEN_INT (mask);
+
+ return "vshufps\t{%3, %2, %1, %0|%0, %1, %2, %3}";
+}
+ [(set_attr "type" "sselog")
+ (set_attr "length_immediate" "1")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "V16SF")])
+
+(define_insn "avx512f_shufpd512_1"
+ [(set (match_operand:V8DF 0 "register_operand" "=v")
+ (vec_select:V8DF
+ (vec_concat:V16DF
+ (match_operand:V8DF 1 "register_operand" "v")
+ (match_operand:V8DF 2 "nonimmediate_operand" "vm"))
+ (parallel [(match_operand 3 "const_0_to_1_operand")
+ (match_operand 4 "const_8_to_9_operand")
+ (match_operand 5 "const_2_to_3_operand")
+ (match_operand 6 "const_10_to_11_operand")
+ (match_operand 7 "const_4_to_5_operand")
+ (match_operand 8 "const_12_to_13_operand")
+ (match_operand 9 "const_6_to_7_operand")
+ (match_operand 10 "const_14_to_15_operand")])))]
+ "TARGET_AVX512F"
+{
+ int mask;
+ mask = INTVAL (operands[3]);
+ mask |= (INTVAL (operands[4]) - 8) << 1;
+ mask |= (INTVAL (operands[5]) - 2) << 2;
+ mask |= (INTVAL (operands[6]) - 10) << 3;
+ mask |= (INTVAL (operands[7]) - 4) << 4;
+ mask |= (INTVAL (operands[8]) - 12) << 5;
+ mask |= (INTVAL (operands[9]) - 6) << 6;
+ mask |= (INTVAL (operands[10]) - 14) << 7;
+ operands[3] = GEN_INT (mask);
+
+ return "vshufpd\t{%3, %2, %1, %0|%0, %1, %2, %3}";
+}
+ [(set_attr "type" "sselog")
+ (set_attr "length_immediate" "1")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "V8DF")])
+
(define_expand "avx_shufpd256"
[(match_operand:V4DF 0 "register_operand")
(match_operand:V4DF 1 "register_operand")
@@ -4905,6 +6198,22 @@
(set_attr "prefix" "vex")
(set_attr "mode" "OI")])
+(define_insn "avx512f_interleave_highv8di"
+ [(set (match_operand:V8DI 0 "register_operand" "=v")
+ (vec_select:V8DI
+ (vec_concat:V16DI
+ (match_operand:V8DI 1 "register_operand" "v")
+ (match_operand:V8DI 2 "nonimmediate_operand" "vm"))
+ (parallel [(const_int 1) (const_int 9)
+ (const_int 3) (const_int 11)
+ (const_int 5) (const_int 13)
+ (const_int 7) (const_int 15)])))]
+ "TARGET_AVX512F"
+ "vpunpckhqdq\t{%2, %1, %0|%0, %1, %2}"
+ [(set_attr "type" "sselog")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "XI")])
+
(define_insn "vec_interleave_highv2di"
[(set (match_operand:V2DI 0 "register_operand" "=x,x")
(vec_select:V2DI
@@ -4939,6 +6248,22 @@
(set_attr "prefix" "vex")
(set_attr "mode" "OI")])
+(define_insn "avx512f_interleave_lowv8di"
+ [(set (match_operand:V8DI 0 "register_operand" "=v")
+ (vec_select:V8DI
+ (vec_concat:V16DI
+ (match_operand:V8DI 1 "register_operand" "v")
+ (match_operand:V8DI 2 "nonimmediate_operand" "vm"))
+ (parallel [(const_int 0) (const_int 8)
+ (const_int 2) (const_int 10)
+ (const_int 4) (const_int 12)
+ (const_int 6) (const_int 14)])))]
+ "TARGET_AVX512F"
+ "vpunpcklqdq\t{%2, %1, %0|%0, %1, %2}"
+ [(set_attr "type" "sselog")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "XI")])
+
(define_insn "vec_interleave_lowv2di"
[(set (match_operand:V2DI 0 "register_operand" "=x,x")
(vec_select:V2DI
@@ -5282,6 +6607,64 @@
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;
+;; Parallel integer down-conversion operations
+;;
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(define_mode_iterator PMOV_DST_MODE [V16QI V16HI V8SI V8HI])
+(define_mode_attr pmov_src_mode
+ [(V16QI "V16SI") (V16HI "V16SI") (V8SI "V8DI") (V8HI "V8DI")])
+(define_mode_attr pmov_src_lower
+ [(V16QI "v16si") (V16HI "v16si") (V8SI "v8di") (V8HI "v8di")])
+(define_mode_attr pmov_suff
+ [(V16QI "db") (V16HI "dw") (V8SI "qd") (V8HI "qw")])
+
+(define_insn "*avx512f_<code><pmov_src_lower><mode>2"
+ [(set (match_operand:PMOV_DST_MODE 0 "nonimmediate_operand" "=v,m")
+ (any_truncate:PMOV_DST_MODE
+ (match_operand:<pmov_src_mode> 1 "register_operand" "v,v")))]
+ "TARGET_AVX512F"
+ "vpmov<trunsuffix><pmov_suff>\t{%1, %0|%0, %1}"
+ [(set_attr "type" "ssemov")
+ (set_attr "memory" "none,store")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "<sseinsnmode>")])
+
+(define_insn "*avx512f_<code>v8div16qi2"
+ [(set (match_operand:V16QI 0 "register_operand" "=v")
+ (vec_concat:V16QI
+ (any_truncate:V8QI
+ (match_operand:V8DI 1 "register_operand" "v"))
+ (const_vector:V8QI [(const_int 0) (const_int 0)
+ (const_int 0) (const_int 0)
+ (const_int 0) (const_int 0)
+ (const_int 0) (const_int 0)])))]
+ "TARGET_AVX512F"
+ "vpmov<trunsuffix>qb\t{%1, %0|%0, %1}"
+ [(set_attr "type" "ssemov")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "TI")])
+
+(define_insn "*avx512f_<code>v8div16qi2_store"
+ [(set (match_operand:V16QI 0 "memory_operand" "=m")
+ (vec_concat:V16QI
+ (any_truncate:V8QI
+ (match_operand:V8DI 1 "register_operand" "v"))
+ (vec_select:V8QI
+ (match_dup 0)
+ (parallel [(const_int 8) (const_int 9)
+ (const_int 10) (const_int 11)
+ (const_int 12) (const_int 13)
+ (const_int 14) (const_int 15)]))))]
+ "TARGET_AVX512F"
+ "vpmov<trunsuffix>qb\t{%1, %0|%0, %1}"
+ [(set_attr "type" "ssemov")
+ (set_attr "memory" "store")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "TI")])
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;;
;; Parallel integral arithmetic
;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -5404,6 +6787,51 @@
(set_attr "prefix" "orig,vex")
(set_attr "mode" "<sseinsnmode>")])
+(define_expand "vec_widen_umult_even_v16si"
+ [(set (match_operand:V8DI 0 "register_operand")
+ (mult:V8DI
+ (zero_extend:V8DI
+ (vec_select:V8SI
+ (match_operand:V16SI 1 "nonimmediate_operand")
+ (parallel [(const_int 0) (const_int 2)
+ (const_int 4) (const_int 6)
+ (const_int 8) (const_int 10)
+ (const_int 12) (const_int 14)])))
+ (zero_extend:V8DI
+ (vec_select:V8SI
+ (match_operand:V16SI 2 "nonimmediate_operand")
+ (parallel [(const_int 0) (const_int 2)
+ (const_int 4) (const_int 6)
+ (const_int 8) (const_int 10)
+ (const_int 12) (const_int 14)])))))]
+ "TARGET_AVX512F"
+ "ix86_fixup_binary_operands_no_copy (MULT, V16SImode, operands);")
+
+(define_insn "*vec_widen_umult_even_v16si"
+ [(set (match_operand:V8DI 0 "register_operand" "=v")
+ (mult:V8DI
+ (zero_extend:V8DI
+ (vec_select:V8SI
+ (match_operand:V16SI 1 "nonimmediate_operand" "%v")
+ (parallel [(const_int 0) (const_int 2)
+ (const_int 4) (const_int 6)
+ (const_int 8) (const_int 10)
+ (const_int 12) (const_int 14)])))
+ (zero_extend:V8DI
+ (vec_select:V8SI
+ (match_operand:V16SI 2 "nonimmediate_operand" "vm")
+ (parallel [(const_int 0) (const_int 2)
+ (const_int 4) (const_int 6)
+ (const_int 8) (const_int 10)
+ (const_int 12) (const_int 14)])))))]
+ "TARGET_AVX512F && ix86_binary_operator_ok (MULT, V16SImode, operands)"
+ "vpmuludq\t{%2, %1, %0|%0, %1, %2}"
+ [(set_attr "isa" "avx512f")
+ (set_attr "type" "sseimul")
+ (set_attr "prefix_extra" "1")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "XI")])
+
(define_expand "vec_widen_umult_even_v8si"
[(set (match_operand:V4DI 0 "register_operand")
(mult:V4DI
@@ -5474,6 +6902,51 @@
(set_attr "prefix" "orig,vex")
(set_attr "mode" "TI")])
+(define_expand "vec_widen_smult_even_v16si"
+ [(set (match_operand:V8DI 0 "register_operand")
+ (mult:V8DI
+ (sign_extend:V8DI
+ (vec_select:V8SI
+ (match_operand:V16SI 1 "nonimmediate_operand")
+ (parallel [(const_int 0) (const_int 2)
+ (const_int 4) (const_int 6)
+ (const_int 8) (const_int 10)
+ (const_int 12) (const_int 14)])))
+ (sign_extend:V8DI
+ (vec_select:V8SI
+ (match_operand:V16SI 2 "nonimmediate_operand")
+ (parallel [(const_int 0) (const_int 2)
+ (const_int 4) (const_int 6)
+ (const_int 8) (const_int 10)
+ (const_int 12) (const_int 14)])))))]
+ "TARGET_AVX512F"
+ "ix86_fixup_binary_operands_no_copy (MULT, V16SImode, operands);")
+
+(define_insn "*vec_widen_smult_even_v16si"
+ [(set (match_operand:V8DI 0 "register_operand" "=v")
+ (mult:V8DI
+ (sign_extend:V8DI
+ (vec_select:V8SI
+ (match_operand:V16SI 1 "nonimmediate_operand" "%v")
+ (parallel [(const_int 0) (const_int 2)
+ (const_int 4) (const_int 6)
+ (const_int 8) (const_int 10)
+ (const_int 12) (const_int 14)])))
+ (sign_extend:V8DI
+ (vec_select:V8SI
+ (match_operand:V16SI 2 "nonimmediate_operand" "vm")
+ (parallel [(const_int 0) (const_int 2)
+ (const_int 4) (const_int 6)
+ (const_int 8) (const_int 10)
+ (const_int 12) (const_int 14)])))))]
+ "TARGET_AVX512F && ix86_binary_operator_ok (MULT, V16SImode, operands)"
+ "vpmuldq\t{%2, %1, %0|%0, %1, %2}"
+ [(set_attr "isa" "avx512f")
+ (set_attr "type" "sseimul")
+ (set_attr "prefix_extra" "1")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "XI")])
+
(define_expand "vec_widen_smult_even_v8si"
[(set (match_operand:V4DI 0 "register_operand")
(mult:V4DI
@@ -5505,8 +6978,7 @@
(const_int 4) (const_int 6)])))))]
"TARGET_AVX2 && ix86_binary_operator_ok (MULT, V8SImode, operands)"
"vpmuldq\t{%2, %1, %0|%0, %1, %2}"
- [(set_attr "isa" "avx")
- (set_attr "type" "sseimul")
+ [(set_attr "type" "sseimul")
(set_attr "prefix_extra" "1")
(set_attr "prefix" "vex")
(set_attr "mode" "OI")])
@@ -5679,10 +7151,10 @@
(set_attr "mode" "TI")])
(define_expand "mul<mode>3"
- [(set (match_operand:VI4_AVX2 0 "register_operand")
- (mult:VI4_AVX2
- (match_operand:VI4_AVX2 1 "general_vector_operand")
- (match_operand:VI4_AVX2 2 "general_vector_operand")))]
+ [(set (match_operand:VI4_AVX512F 0 "register_operand")
+ (mult:VI4_AVX512F
+ (match_operand:VI4_AVX512F 1 "general_vector_operand")
+ (match_operand:VI4_AVX512F 2 "general_vector_operand")))]
"TARGET_SSE2"
{
if (TARGET_SSE4_1)
@@ -5701,10 +7173,10 @@
})
(define_insn "*<sse4_1_avx2>_mul<mode>3"
- [(set (match_operand:VI4_AVX2 0 "register_operand" "=x,v")
- (mult:VI4_AVX2
- (match_operand:VI4_AVX2 1 "nonimmediate_operand" "%0,v")
- (match_operand:VI4_AVX2 2 "nonimmediate_operand" "xm,vm")))]
+ [(set (match_operand:VI4_AVX512F 0 "register_operand" "=x,v")
+ (mult:VI4_AVX512F
+ (match_operand:VI4_AVX512F 1 "nonimmediate_operand" "%0,v")
+ (match_operand:VI4_AVX512F 2 "nonimmediate_operand" "xm,vm")))]
"TARGET_SSE4_1 && ix86_binary_operator_ok (MULT, <MODE>mode, operands)"
"@
pmulld\t{%2, %0|%0, %2}
@@ -5717,9 +7189,10 @@
(set_attr "mode" "<sseinsnmode>")])
(define_expand "mul<mode>3"
- [(set (match_operand:VI8_AVX2 0 "register_operand")
- (mult:VI8_AVX2 (match_operand:VI8_AVX2 1 "register_operand")
- (match_operand:VI8_AVX2 2 "register_operand")))]
+ [(set (match_operand:VI8_AVX2_AVX512F 0 "register_operand")
+ (mult:VI8_AVX2_AVX512F
+ (match_operand:VI8_AVX2_AVX512F 1 "register_operand")
+ (match_operand:VI8_AVX2_AVX512F 2 "register_operand")))]
"TARGET_SSE2"
{
ix86_expand_sse2_mulvxdi3 (operands[0], operands[1], operands[2]);
@@ -5766,8 +7239,8 @@
(define_expand "vec_widen_<s>mult_odd_<mode>"
[(match_operand:<sseunpackmode> 0 "register_operand")
(any_extend:<sseunpackmode>
- (match_operand:VI4_AVX2 1 "general_vector_operand"))
- (match_operand:VI4_AVX2 2 "general_vector_operand")]
+ (match_operand:VI4_AVX512F 1 "general_vector_operand"))
+ (match_operand:VI4_AVX512F 2 "general_vector_operand")]
"TARGET_SSE2"
{
ix86_expand_mul_widen_evenodd (operands[0], operands[1], operands[2],
@@ -5825,9 +7298,9 @@
(set_attr "mode" "<sseinsnmode>")])
(define_insn "<shift_insn><mode>3"
- [(set (match_operand:VI248_AVX2 0 "register_operand" "=x,v")
+ [(set (match_operand:VI248_AVX2 0 "register_operand" "=x,x")
(any_lshift:VI248_AVX2
- (match_operand:VI248_AVX2 1 "register_operand" "0,v")
+ (match_operand:VI248_AVX2 1 "register_operand" "0,x")
(match_operand:SI 2 "nonmemory_operand" "xN,xN")))]
"TARGET_SSE2"
"@
@@ -5843,15 +7316,33 @@
(set_attr "prefix" "orig,vex")
(set_attr "mode" "<sseinsnmode>")])
+(define_insn "<shift_insn><mode>3"
+ [(set (match_operand:VI48_512 0 "register_operand" "=v,v")
+ (any_lshift:VI48_512
+ (match_operand:VI48_512 1 "register_operand" "v,m")
+ (match_operand:SI 2 "nonmemory_operand" "vN,N")))]
+ "TARGET_AVX512F"
+ "vp<vshift><ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
+ [(set_attr "isa" "avx512f")
+ (set_attr "type" "sseishft")
+ (set (attr "length_immediate")
+ (if_then_else (match_operand 2 "const_int_operand")
+ (const_string "1")
+ (const_string "0")))
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "<sseinsnmode>")])
+
(define_expand "vec_shl_<mode>"
- [(set (match_operand:VI_128 0 "register_operand")
+ [(set (match_dup 3)
(ashift:V1TI
(match_operand:VI_128 1 "register_operand")
- (match_operand:SI 2 "const_0_to_255_mul_8_operand")))]
+ (match_operand:SI 2 "const_0_to_255_mul_8_operand")))
+ (set (match_operand:VI_128 0 "register_operand") (match_dup 4))]
"TARGET_SSE2"
{
- operands[0] = gen_lowpart (V1TImode, operands[0]);
operands[1] = gen_lowpart (V1TImode, operands[1]);
+ operands[3] = gen_reg_rtx (V1TImode);
+ operands[4] = gen_lowpart (<MODE>mode, operands[3]);
})
(define_insn "<sse2_avx2>_ashl<mode>3"
@@ -5881,14 +7372,16 @@
(set_attr "mode" "<sseinsnmode>")])
(define_expand "vec_shr_<mode>"
- [(set (match_operand:VI_128 0 "register_operand")
+ [(set (match_dup 3)
(lshiftrt:V1TI
(match_operand:VI_128 1 "register_operand")
- (match_operand:SI 2 "const_0_to_255_mul_8_operand")))]
+ (match_operand:SI 2 "const_0_to_255_mul_8_operand")))
+ (set (match_operand:VI_128 0 "register_operand") (match_dup 4))]
"TARGET_SSE2"
{
- operands[0] = gen_lowpart (V1TImode, operands[0]);
operands[1] = gen_lowpart (V1TImode, operands[1]);
+ operands[3] = gen_reg_rtx (V1TImode);
+ operands[4] = gen_lowpart (<MODE>mode, operands[3]);
})
(define_insn "<sse2_avx2>_lshr<mode>3"
@@ -5918,25 +7411,44 @@
(set_attr "prefix" "orig,vex")
(set_attr "mode" "<sseinsnmode>")])
+(define_insn "avx512f_<rotate>v<mode>"
+ [(set (match_operand:VI48_512 0 "register_operand" "=v")
+ (any_rotate:VI48_512
+ (match_operand:VI48_512 1 "register_operand" "v")
+ (match_operand:VI48_512 2 "nonimmediate_operand" "vm")))]
+ "TARGET_AVX512F"
+ "vp<rotate>v<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
+ [(set_attr "prefix" "evex")
+ (set_attr "mode" "<sseinsnmode>")])
+
+(define_insn "avx512f_<rotate><mode>"
+ [(set (match_operand:VI48_512 0 "register_operand" "=v")
+ (any_rotate:VI48_512
+ (match_operand:VI48_512 1 "nonimmediate_operand" "vm")
+ (match_operand:SI 2 "const_0_to_255_operand")))]
+ "TARGET_AVX512F"
+ "vp<rotate><ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
+ [(set_attr "prefix" "evex")
+ (set_attr "mode" "<sseinsnmode>")])
(define_expand "<code><mode>3"
- [(set (match_operand:VI124_256 0 "register_operand")
- (maxmin:VI124_256
- (match_operand:VI124_256 1 "nonimmediate_operand")
- (match_operand:VI124_256 2 "nonimmediate_operand")))]
+ [(set (match_operand:VI124_256_48_512 0 "register_operand")
+ (maxmin:VI124_256_48_512
+ (match_operand:VI124_256_48_512 1 "nonimmediate_operand")
+ (match_operand:VI124_256_48_512 2 "nonimmediate_operand")))]
"TARGET_AVX2"
"ix86_fixup_binary_operands_no_copy (<CODE>, <MODE>mode, operands);")
(define_insn "*avx2_<code><mode>3"
- [(set (match_operand:VI124_256 0 "register_operand" "=v")
- (maxmin:VI124_256
- (match_operand:VI124_256 1 "nonimmediate_operand" "%v")
- (match_operand:VI124_256 2 "nonimmediate_operand" "vm")))]
+ [(set (match_operand:VI124_256_48_512 0 "register_operand" "=v")
+ (maxmin:VI124_256_48_512
+ (match_operand:VI124_256_48_512 1 "nonimmediate_operand" "%v")
+ (match_operand:VI124_256_48_512 2 "nonimmediate_operand" "vm")))]
"TARGET_AVX2 && ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)"
"vp<maxmin_int><ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
[(set_attr "type" "sseiadd")
(set_attr "prefix_extra" "1")
- (set_attr "prefix" "vex")
+ (set_attr "prefix" "maybe_evex")
(set_attr "mode" "OI")])
(define_expand "<code><mode>3"
@@ -6151,6 +7663,28 @@
(set_attr "prefix" "vex")
(set_attr "mode" "OI")])
+(define_expand "avx512f_eq<mode>3"
+ [(set (match_operand:<avx512fmaskmode> 0 "register_operand")
+ (unspec:<avx512fmaskmode>
+ [(match_operand:VI48_512 1 "register_operand")
+ (match_operand:VI48_512 2 "nonimmediate_operand")]
+ UNSPEC_MASKED_EQ))]
+ "TARGET_AVX512F"
+ "ix86_fixup_binary_operands_no_copy (EQ, <MODE>mode, operands);")
+
+(define_insn "avx512f_eq<mode>3_1"
+ [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=k")
+ (unspec:<avx512fmaskmode>
+ [(match_operand:VI48_512 1 "register_operand" "%v")
+ (match_operand:VI48_512 2 "nonimmediate_operand" "vm")]
+ UNSPEC_MASKED_EQ))]
+ "TARGET_AVX512F && ix86_binary_operator_ok (EQ, <MODE>mode, operands)"
+ "vpcmpeq<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
+ [(set_attr "type" "ssecmp")
+ (set_attr "prefix_extra" "1")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "<sseinsnmode>")])
+
(define_insn "*sse4_1_eqv2di3"
[(set (match_operand:V2DI 0 "register_operand" "=x,x")
(eq:V2DI
@@ -6225,6 +7759,18 @@
(set_attr "prefix" "vex")
(set_attr "mode" "OI")])
+(define_insn "avx512f_gt<mode>3"
+ [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=k")
+ (unspec:<avx512fmaskmode>
+ [(match_operand:VI48_512 1 "register_operand" "v")
+ (match_operand:VI48_512 2 "nonimmediate_operand" "vm")] UNSPEC_MASKED_GT))]
+ "TARGET_AVX512F"
+ "vpcmpgt<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
+ [(set_attr "type" "ssecmp")
+ (set_attr "prefix_extra" "1")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "<sseinsnmode>")])
+
(define_insn "sse2_gt<mode>3"
[(set (match_operand:VI124_128 0 "register_operand" "=x,x")
(gt:VI124_128
@@ -6240,6 +7786,23 @@
(set_attr "prefix" "orig,vex")
(set_attr "mode" "TI")])
+(define_expand "vcond<V_512:mode><VI_512:mode>"
+ [(set (match_operand:V_512 0 "register_operand")
+ (if_then_else:V_512
+ (match_operator 3 ""
+ [(match_operand:VI_512 4 "nonimmediate_operand")
+ (match_operand:VI_512 5 "general_operand")])
+ (match_operand:V_512 1)
+ (match_operand:V_512 2)))]
+ "TARGET_AVX512F
+ && (GET_MODE_NUNITS (<V_512:MODE>mode)
+ == GET_MODE_NUNITS (<VI_512:MODE>mode))"
+{
+ bool ok = ix86_expand_int_vcond (operands);
+ gcc_assert (ok);
+ DONE;
+})
+
(define_expand "vcond<V_256:mode><VI_256:mode>"
[(set (match_operand:V_256 0 "register_operand")
(if_then_else:V_256
@@ -6289,6 +7852,23 @@
DONE;
})
+(define_expand "vcondu<V_512:mode><VI_512:mode>"
+ [(set (match_operand:V_512 0 "register_operand")
+ (if_then_else:V_512
+ (match_operator 3 ""
+ [(match_operand:VI_512 4 "nonimmediate_operand")
+ (match_operand:VI_512 5 "nonimmediate_operand")])
+ (match_operand:V_512 1 "general_operand")
+ (match_operand:V_512 2 "general_operand")))]
+ "TARGET_AVX512F
+ && (GET_MODE_NUNITS (<V_512:MODE>mode)
+ == GET_MODE_NUNITS (<VI_512:MODE>mode))"
+{
+ bool ok = ix86_expand_int_vcond (operands);
+ gcc_assert (ok);
+ DONE;
+})
+
(define_expand "vcondu<V_256:mode><VI_256:mode>"
[(set (match_operand:V_256 0 "register_operand")
(if_then_else:V_256
@@ -6342,7 +7922,9 @@
[V16QI V8HI V4SI V2DI V4SF V2DF
(V32QI "TARGET_AVX2") (V16HI "TARGET_AVX2")
(V8SI "TARGET_AVX2") (V4DI "TARGET_AVX2")
- (V8SF "TARGET_AVX2") (V4DF "TARGET_AVX2")])
+ (V8SF "TARGET_AVX2") (V4DF "TARGET_AVX2")
+ (V16SF "TARGET_AVX512F") (V8DF "TARGET_AVX512F")
+ (V16SI "TARGET_AVX512F") (V8DI "TARGET_AVX512F")])
(define_expand "vec_perm<mode>"
[(match_operand:VEC_PERM_AVX2 0 "register_operand")
@@ -6361,7 +7943,9 @@
(V16QI "TARGET_SSE2") (V8HI "TARGET_SSE2")
(V8SF "TARGET_AVX") (V4DF "TARGET_AVX")
(V8SI "TARGET_AVX") (V4DI "TARGET_AVX")
- (V32QI "TARGET_AVX2") (V16HI "TARGET_AVX2")])
+ (V32QI "TARGET_AVX2") (V16HI "TARGET_AVX2")
+ (V16SI "TARGET_AVX512F") (V8DI "TARGET_AVX512F")
+ (V16SF "TARGET_AVX512F") (V8DF "TARGET_AVX512F")])
(define_expand "vec_perm_const<mode>"
[(match_operand:VEC_PERM_CONST 0 "register_operand")
@@ -6411,12 +7995,18 @@
(match_operand:VI 2 "nonimmediate_operand" "xm,vm")))]
"TARGET_SSE"
{
- static char buf[32];
+ static char buf[64];
const char *ops;
const char *tmp;
switch (get_attr_mode (insn))
{
+ case MODE_XI:
+ gcc_assert (TARGET_AVX512F);
+
+ tmp = "pandn<ssemodesuffix>";
+ break;
+
case MODE_OI:
gcc_assert (TARGET_AVX2);
case MODE_TI:
@@ -6496,12 +8086,17 @@
"TARGET_SSE
&& ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)"
{
- static char buf[32];
+ static char buf[64];
const char *ops;
const char *tmp;
switch (get_attr_mode (insn))
{
+ case MODE_XI:
+ gcc_assert (TARGET_AVX512F);
+ tmp = "p<logic><ssemodesuffix>";
+ break;
+
case MODE_OI:
gcc_assert (TARGET_AVX2);
case MODE_TI:
@@ -6510,6 +8105,8 @@
tmp = "p<logic>";
break;
+ case MODE_V16SF:
+ gcc_assert (TARGET_AVX512F);
case MODE_V8SF:
gcc_assert (TARGET_AVX);
case MODE_V4SF:
@@ -6562,6 +8159,28 @@
]
(const_string "<sseinsnmode>")))])
+(define_insn "avx512f_testm<mode>3"
+ [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=k")
+ (unspec:<avx512fmaskmode>
+ [(match_operand:VI48_512 1 "register_operand" "v")
+ (match_operand:VI48_512 2 "nonimmediate_operand" "vm")]
+ UNSPEC_TESTM))]
+ "TARGET_AVX512F"
+ "vptestm<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
+ [(set_attr "prefix" "evex")
+ (set_attr "mode" "<sseinsnmode>")])
+
+(define_insn "avx512f_testnm<mode>3"
+ [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=k")
+ (unspec:<avx512fmaskmode>
+ [(match_operand:VI48_512 1 "register_operand" "v")
+ (match_operand:VI48_512 2 "nonimmediate_operand" "vm")]
+ UNSPEC_TESTNM))]
+ "TARGET_AVX512CD"
+ "%vptestnm<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
+ [(set_attr "prefix" "evex")
+ (set_attr "mode" "<sseinsnmode>")])
+
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;
;; Parallel integral element swizzling
@@ -6570,8 +8189,8 @@
(define_expand "vec_pack_trunc_<mode>"
[(match_operand:<ssepackmode> 0 "register_operand")
- (match_operand:VI248_AVX2 1 "register_operand")
- (match_operand:VI248_AVX2 2 "register_operand")]
+ (match_operand:VI248_AVX2_8_AVX512F 1 "register_operand")
+ (match_operand:VI248_AVX2_8_AVX512F 2 "register_operand")]
"TARGET_SSE2"
{
rtx op1 = gen_lowpart (<ssepackmode>mode, operands[1]);
@@ -6831,6 +8450,27 @@
(set_attr "prefix" "vex")
(set_attr "mode" "OI")])
+(define_insn "avx512f_interleave_highv16si"
+ [(set (match_operand:V16SI 0 "register_operand" "=v")
+ (vec_select:V16SI
+ (vec_concat:V32SI
+ (match_operand:V16SI 1 "register_operand" "v")
+ (match_operand:V16SI 2 "nonimmediate_operand" "vm"))
+ (parallel [(const_int 2) (const_int 18)
+ (const_int 3) (const_int 19)
+ (const_int 6) (const_int 22)
+ (const_int 7) (const_int 23)
+ (const_int 10) (const_int 26)
+ (const_int 11) (const_int 27)
+ (const_int 14) (const_int 30)
+ (const_int 15) (const_int 31)])))]
+ "TARGET_AVX512F"
+ "vpunpckhdq\t{%2, %1, %0|%0, %1, %2}"
+ [(set_attr "type" "sselog")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "XI")])
+
+
(define_insn "vec_interleave_highv4si"
[(set (match_operand:V4SI 0 "register_operand" "=x,x")
(vec_select:V4SI
@@ -6865,6 +8505,26 @@
(set_attr "prefix" "vex")
(set_attr "mode" "OI")])
+(define_insn "avx512f_interleave_lowv16si"
+ [(set (match_operand:V16SI 0 "register_operand" "=v")
+ (vec_select:V16SI
+ (vec_concat:V32SI
+ (match_operand:V16SI 1 "register_operand" "v")
+ (match_operand:V16SI 2 "nonimmediate_operand" "vm"))
+ (parallel [(const_int 0) (const_int 16)
+ (const_int 1) (const_int 17)
+ (const_int 4) (const_int 20)
+ (const_int 5) (const_int 21)
+ (const_int 8) (const_int 24)
+ (const_int 9) (const_int 25)
+ (const_int 12) (const_int 28)
+ (const_int 13) (const_int 29)])))]
+ "TARGET_AVX512F"
+ "vpunpckldq\t{%2, %1, %0|%0, %1, %2}"
+ [(set_attr "type" "sselog")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "XI")])
+
(define_insn "vec_interleave_lowv4si"
[(set (match_operand:V4SI 0 "register_operand" "=x,x")
(vec_select:V4SI
@@ -6891,12 +8551,13 @@
{
rtx t1 = gen_reg_rtx (<MODE>mode);
rtx t2 = gen_reg_rtx (<MODE>mode);
+ rtx t3 = gen_reg_rtx (V4DImode);
emit_insn (gen_avx2_interleave_low<mode> (t1, operands[1], operands[2]));
emit_insn (gen_avx2_interleave_high<mode> (t2, operands[1], operands[2]));
- emit_insn (gen_avx2_permv2ti
- (gen_lowpart (V4DImode, operands[0]),
- gen_lowpart (V4DImode, t1),
- gen_lowpart (V4DImode, t2), GEN_INT (1 + (3 << 4))));
+ emit_insn (gen_avx2_permv2ti (t3, gen_lowpart (V4DImode, t1),
+ gen_lowpart (V4DImode, t2),
+ GEN_INT (1 + (3 << 4))));
+ emit_move_insn (operands[0], gen_lowpart (<MODE>mode, t3));
DONE;
})
@@ -6908,12 +8569,13 @@
{
rtx t1 = gen_reg_rtx (<MODE>mode);
rtx t2 = gen_reg_rtx (<MODE>mode);
+ rtx t3 = gen_reg_rtx (V4DImode);
emit_insn (gen_avx2_interleave_low<mode> (t1, operands[1], operands[2]));
emit_insn (gen_avx2_interleave_high<mode> (t2, operands[1], operands[2]));
- emit_insn (gen_avx2_permv2ti
- (gen_lowpart (V4DImode, operands[0]),
- gen_lowpart (V4DImode, t1),
- gen_lowpart (V4DImode, t2), GEN_INT (0 + (2 << 4))));
+ emit_insn (gen_avx2_permv2ti (t3, gen_lowpart (V4DImode, t1),
+ gen_lowpart (V4DImode, t2),
+ GEN_INT (0 + (2 << 4))));
+ emit_move_insn (operands[0], gen_lowpart (<MODE>mode, t3));
DONE;
})
@@ -6983,6 +8645,198 @@
(set_attr "prefix" "orig,orig,vex,vex")
(set_attr "mode" "TI")])
+(define_insn "avx512f_vinsert<shuffletype>32x4_1"
+ [(set (match_operand:V16FI 0 "register_operand" "=v")
+ (vec_merge:V16FI
+ (match_operand:V16FI 1 "register_operand" "v")
+ (vec_duplicate:V16FI
+ (match_operand:<ssequartermode> 2 "nonimmediate_operand" "vm"))
+ (match_operand:SI 3 "const_int_operand" "n")))]
+ "TARGET_AVX512F"
+{
+ int mask;
+ if (INTVAL (operands[3]) == 0xFFF)
+ mask = 0;
+ else if ( INTVAL (operands[3]) == 0xF0FF)
+ mask = 1;
+ else if ( INTVAL (operands[3]) == 0xFF0F)
+ mask = 2;
+ else if ( INTVAL (operands[3]) == 0xFFF0)
+ mask = 3;
+ else
+ gcc_unreachable ();
+
+ operands[3] = GEN_INT (mask);
+
+ return "vinsert<shuffletype>32x4\t{%3, %2, %1, %0|%0, %1, %2, %3}";
+}
+ [(set_attr "type" "sselog")
+ (set_attr "length_immediate" "1")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "<sseinsnmode>")])
+
+(define_insn "vec_set_lo_<mode>"
+ [(set (match_operand:V8FI 0 "register_operand" "=v")
+ (vec_concat:V8FI
+ (match_operand:<ssehalfvecmode> 2 "nonimmediate_operand" "vm")
+ (vec_select:<ssehalfvecmode>
+ (match_operand:V8FI 1 "register_operand" "v")
+ (parallel [(const_int 4) (const_int 5)
+ (const_int 6) (const_int 7)]))))]
+ "TARGET_AVX512F"
+ "vinsert<shuffletype>64x4\t{$0x0, %2, %1, %0|%0, %1, %2, $0x0}"
+ [(set_attr "type" "sselog")
+ (set_attr "length_immediate" "1")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "XI")])
+
+(define_insn "vec_set_hi_<mode>"
+ [(set (match_operand:V8FI 0 "register_operand" "=v")
+ (vec_concat:V8FI
+ (match_operand:<ssehalfvecmode> 2 "nonimmediate_operand" "vm")
+ (vec_select:<ssehalfvecmode>
+ (match_operand:V8FI 1 "register_operand" "v")
+ (parallel [(const_int 0) (const_int 1)
+ (const_int 2) (const_int 3)]))))]
+ "TARGET_AVX512F"
+ "vinsert<shuffletype>64x4\t{$0x1, %2, %1, %0|%0, %1, %2, $0x1}"
+ [(set_attr "type" "sselog")
+ (set_attr "length_immediate" "1")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "XI")])
+
+(define_insn "avx512f_shuf_<shuffletype>64x2_1"
+ [(set (match_operand:V8FI 0 "register_operand" "=v")
+ (vec_select:V8FI
+ (vec_concat:<ssedoublemode>
+ (match_operand:V8FI 1 "register_operand" "v")
+ (match_operand:V8FI 2 "nonimmediate_operand" "vm"))
+ (parallel [(match_operand 3 "const_0_to_7_operand")
+ (match_operand 4 "const_0_to_7_operand")
+ (match_operand 5 "const_0_to_7_operand")
+ (match_operand 6 "const_0_to_7_operand")
+ (match_operand 7 "const_8_to_15_operand")
+ (match_operand 8 "const_8_to_15_operand")
+ (match_operand 9 "const_8_to_15_operand")
+ (match_operand 10 "const_8_to_15_operand")])))]
+ "TARGET_AVX512F
+ && (INTVAL (operands[3]) == (INTVAL (operands[4]) - 1)
+ && INTVAL (operands[5]) == (INTVAL (operands[6]) - 1)
+ && INTVAL (operands[7]) == (INTVAL (operands[8]) - 1)
+ && INTVAL (operands[9]) == (INTVAL (operands[10]) - 1))"
+{
+ int mask;
+ mask = INTVAL (operands[3]) / 2;
+ mask |= INTVAL (operands[5]) / 2 << 2;
+ mask |= (INTVAL (operands[7]) - 8) / 2 << 4;
+ mask |= (INTVAL (operands[9]) - 8) / 2 << 6;
+ operands[3] = GEN_INT (mask);
+
+ return "vshuf<shuffletype>64x2\t{%3, %2, %1, %0|%0, %1, %2, %3}";
+}
+ [(set_attr "type" "sselog")
+ (set_attr "length_immediate" "1")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "<sseinsnmode>")])
+
+(define_insn "avx512f_shuf_<shuffletype>32x4_1"
+ [(set (match_operand:V16FI 0 "register_operand" "=v")
+ (vec_select:V16FI
+ (vec_concat:<ssedoublemode>
+ (match_operand:V16FI 1 "register_operand" "v")
+ (match_operand:V16FI 2 "nonimmediate_operand" "vm"))
+ (parallel [(match_operand 3 "const_0_to_15_operand")
+ (match_operand 4 "const_0_to_15_operand")
+ (match_operand 5 "const_0_to_15_operand")
+ (match_operand 6 "const_0_to_15_operand")
+ (match_operand 7 "const_0_to_15_operand")
+ (match_operand 8 "const_0_to_15_operand")
+ (match_operand 9 "const_0_to_15_operand")
+ (match_operand 10 "const_0_to_15_operand")
+ (match_operand 11 "const_16_to_31_operand")
+ (match_operand 12 "const_16_to_31_operand")
+ (match_operand 13 "const_16_to_31_operand")
+ (match_operand 14 "const_16_to_31_operand")
+ (match_operand 15 "const_16_to_31_operand")
+ (match_operand 16 "const_16_to_31_operand")
+ (match_operand 17 "const_16_to_31_operand")
+ (match_operand 18 "const_16_to_31_operand")])))]
+ "TARGET_AVX512F
+ && (INTVAL (operands[3]) == (INTVAL (operands[4]) - 1)
+ && INTVAL (operands[3]) == (INTVAL (operands[5]) - 2)
+ && INTVAL (operands[3]) == (INTVAL (operands[6]) - 3)
+ && INTVAL (operands[7]) == (INTVAL (operands[8]) - 1)
+ && INTVAL (operands[7]) == (INTVAL (operands[9]) - 2)
+ && INTVAL (operands[7]) == (INTVAL (operands[10]) - 3)
+ && INTVAL (operands[11]) == (INTVAL (operands[12]) - 1)
+ && INTVAL (operands[11]) == (INTVAL (operands[13]) - 2)
+ && INTVAL (operands[11]) == (INTVAL (operands[14]) - 3)
+ && INTVAL (operands[15]) == (INTVAL (operands[16]) - 1)
+ && INTVAL (operands[15]) == (INTVAL (operands[17]) - 2)
+ && INTVAL (operands[15]) == (INTVAL (operands[18]) - 3))"
+{
+ int mask;
+ mask = INTVAL (operands[3]) / 4;
+ mask |= INTVAL (operands[7]) / 4 << 2;
+ mask |= (INTVAL (operands[11]) - 16) / 4 << 4;
+ mask |= (INTVAL (operands[15]) - 16) / 4 << 6;
+ operands[3] = GEN_INT (mask);
+
+ return "vshuf<shuffletype>32x4\t{%3, %2, %1, %0|%0, %1, %2, %3}";
+}
+ [(set_attr "type" "sselog")
+ (set_attr "length_immediate" "1")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "<sseinsnmode>")])
+
+(define_insn "avx512f_pshufd_1"
+ [(set (match_operand:V16SI 0 "register_operand" "=v")
+ (vec_select:V16SI
+ (match_operand:V16SI 1 "nonimmediate_operand" "vm")
+ (parallel [(match_operand 2 "const_0_to_3_operand")
+ (match_operand 3 "const_0_to_3_operand")
+ (match_operand 4 "const_0_to_3_operand")
+ (match_operand 5 "const_0_to_3_operand")
+ (match_operand 6 "const_4_to_7_operand")
+ (match_operand 7 "const_4_to_7_operand")
+ (match_operand 8 "const_4_to_7_operand")
+ (match_operand 9 "const_4_to_7_operand")
+ (match_operand 10 "const_8_to_11_operand")
+ (match_operand 11 "const_8_to_11_operand")
+ (match_operand 12 "const_8_to_11_operand")
+ (match_operand 13 "const_8_to_11_operand")
+ (match_operand 14 "const_12_to_15_operand")
+ (match_operand 15 "const_12_to_15_operand")
+ (match_operand 16 "const_12_to_15_operand")
+ (match_operand 17 "const_12_to_15_operand")])))]
+ "TARGET_AVX512F
+ && INTVAL (operands[2]) + 4 == INTVAL (operands[6])
+ && INTVAL (operands[3]) + 4 == INTVAL (operands[7])
+ && INTVAL (operands[4]) + 4 == INTVAL (operands[8])
+ && INTVAL (operands[5]) + 4 == INTVAL (operands[9])
+ && INTVAL (operands[2]) + 8 == INTVAL (operands[10])
+ && INTVAL (operands[3]) + 8 == INTVAL (operands[11])
+ && INTVAL (operands[4]) + 8 == INTVAL (operands[12])
+ && INTVAL (operands[5]) + 8 == INTVAL (operands[13])
+ && INTVAL (operands[2]) + 12 == INTVAL (operands[14])
+ && INTVAL (operands[3]) + 12 == INTVAL (operands[15])
+ && INTVAL (operands[4]) + 12 == INTVAL (operands[16])
+ && INTVAL (operands[5]) + 12 == INTVAL (operands[17])"
+{
+ int mask = 0;
+ mask |= INTVAL (operands[2]) << 0;
+ mask |= INTVAL (operands[3]) << 2;
+ mask |= INTVAL (operands[4]) << 4;
+ mask |= INTVAL (operands[5]) << 6;
+ operands[2] = GEN_INT (mask);
+
+ return "vpshufd\t{%2, %1, %0|%0, %1, %2}";
+}
+ [(set_attr "type" "sselog1")
+ (set_attr "prefix" "evex")
+ (set_attr "length_immediate" "1")
+ (set_attr "mode" "XI")])
+
(define_expand "avx2_pshufdv3"
[(match_operand:V8SI 0 "register_operand")
(match_operand:V8SI 1 "nonimmediate_operand")
@@ -7635,7 +9489,7 @@
"@
pinsrq\t{$1, %2, %0|%0, %2, 1}
vpinsrq\t{$1, %2, %1, %0|%0, %1, %2, 1}
- %vmovd\t{%1, %0|%0, %1}
+ * return HAVE_AS_IX86_INTERUNIT_MOVQ ? \"%vmovq\t{%1, %0|%0, %1}\" : \"%vmovd\t{%1, %0|%0, %1}\";
%vmovq\t{%1, %0|%0, %1}
movq2dq\t{%1, %0|%0, %1}
punpcklqdq\t{%2, %0|%0, %2}
@@ -7657,25 +9511,25 @@
(define_expand "vec_unpacks_lo_<mode>"
[(match_operand:<sseunpackmode> 0 "register_operand")
- (match_operand:VI124_AVX2 1 "register_operand")]
+ (match_operand:VI124_AVX512F 1 "register_operand")]
"TARGET_SSE2"
"ix86_expand_sse_unpack (operands[0], operands[1], false, false); DONE;")
(define_expand "vec_unpacks_hi_<mode>"
[(match_operand:<sseunpackmode> 0 "register_operand")
- (match_operand:VI124_AVX2 1 "register_operand")]
+ (match_operand:VI124_AVX512F 1 "register_operand")]
"TARGET_SSE2"
"ix86_expand_sse_unpack (operands[0], operands[1], false, true); DONE;")
(define_expand "vec_unpacku_lo_<mode>"
[(match_operand:<sseunpackmode> 0 "register_operand")
- (match_operand:VI124_AVX2 1 "register_operand")]
+ (match_operand:VI124_AVX512F 1 "register_operand")]
"TARGET_SSE2"
"ix86_expand_sse_unpack (operands[0], operands[1], true, false); DONE;")
(define_expand "vec_unpacku_hi_<mode>"
[(match_operand:<sseunpackmode> 0 "register_operand")
- (match_operand:VI124_AVX2 1 "register_operand")]
+ (match_operand:VI124_AVX512F 1 "register_operand")]
"TARGET_SSE2"
"ix86_expand_sse_unpack (operands[0], operands[1], true, true); DONE;")
@@ -7747,7 +9601,7 @@
(define_insn "<sse>_movmsk<ssemodesuffix><avxsizesuffix>"
[(set (match_operand:SI 0 "register_operand" "=r")
(unspec:SI
- [(match_operand:VF 1 "register_operand" "x")]
+ [(match_operand:VF_128_256 1 "register_operand" "x")]
UNSPEC_MOVMSK))]
"TARGET_SSE"
"%vmovmsk<ssemodesuffix>\t{%1, %0|%0, %1}"
@@ -8427,10 +10281,10 @@
(set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)"))
(set_attr "mode" "DI")])
-(define_insn "abs<mode>2"
- [(set (match_operand:VI124_AVX2 0 "register_operand" "=v")
- (abs:VI124_AVX2
- (match_operand:VI124_AVX2 1 "nonimmediate_operand" "vm")))]
+(define_insn "*abs<mode>2"
+ [(set (match_operand:VI124_AVX2_48_AVX512F 0 "register_operand" "=v")
+ (abs:VI124_AVX2_48_AVX512F
+ (match_operand:VI124_AVX2_48_AVX512F 1 "nonimmediate_operand" "vm")))]
"TARGET_SSSE3"
"%vpabs<ssemodesuffix>\t{%1, %0|%0, %1}"
[(set_attr "type" "sselog1")
@@ -8439,6 +10293,19 @@
(set_attr "prefix" "maybe_vex")
(set_attr "mode" "<sseinsnmode>")])
+(define_expand "abs<mode>2"
+ [(set (match_operand:VI124_AVX2_48_AVX512F 0 "register_operand")
+ (abs:VI124_AVX2_48_AVX512F
+ (match_operand:VI124_AVX2_48_AVX512F 1 "nonimmediate_operand")))]
+ "TARGET_SSE2"
+{
+ if (!TARGET_SSSE3)
+ {
+ ix86_expand_sse2_abs (operands[0], operands[1]);
+ DONE;
+ }
+})
+
(define_insn "abs<mode>2"
[(set (match_operand:MMXMODEI 0 "register_operand" "=y")
(abs:MMXMODEI
@@ -8537,10 +10404,10 @@
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(define_insn "<sse4_1>_blend<ssemodesuffix><avxsizesuffix>"
- [(set (match_operand:VF 0 "register_operand" "=x,x")
- (vec_merge:VF
- (match_operand:VF 2 "nonimmediate_operand" "xm,xm")
- (match_operand:VF 1 "register_operand" "0,x")
+ [(set (match_operand:VF_128_256 0 "register_operand" "=x,x")
+ (vec_merge:VF_128_256
+ (match_operand:VF_128_256 2 "nonimmediate_operand" "xm,xm")
+ (match_operand:VF_128_256 1 "register_operand" "0,x")
(match_operand:SI 3 "const_0_to_<blendbits>_operand")))]
"TARGET_SSE4_1"
"@
@@ -8555,11 +10422,11 @@
(set_attr "mode" "<MODE>")])
(define_insn "<sse4_1>_blendv<ssemodesuffix><avxsizesuffix>"
- [(set (match_operand:VF 0 "register_operand" "=x,x")
- (unspec:VF
- [(match_operand:VF 1 "register_operand" "0,x")
- (match_operand:VF 2 "nonimmediate_operand" "xm,xm")
- (match_operand:VF 3 "register_operand" "Yz,x")]
+ [(set (match_operand:VF_128_256 0 "register_operand" "=x,x")
+ (unspec:VF_128_256
+ [(match_operand:VF_128_256 1 "register_operand" "0,x")
+ (match_operand:VF_128_256 2 "nonimmediate_operand" "xm,xm")
+ (match_operand:VF_128_256 3 "register_operand" "Yz,x")]
UNSPEC_BLENDV))]
"TARGET_SSE4_1"
"@
@@ -8575,10 +10442,10 @@
(set_attr "mode" "<MODE>")])
(define_insn "<sse4_1>_dp<ssemodesuffix><avxsizesuffix>"
- [(set (match_operand:VF 0 "register_operand" "=x,x")
- (unspec:VF
- [(match_operand:VF 1 "nonimmediate_operand" "%0,x")
- (match_operand:VF 2 "nonimmediate_operand" "xm,xm")
+ [(set (match_operand:VF_128_256 0 "register_operand" "=x,x")
+ (unspec:VF_128_256
+ [(match_operand:VF_128_256 1 "nonimmediate_operand" "%0,x")
+ (match_operand:VF_128_256 2 "nonimmediate_operand" "xm,xm")
(match_operand:SI 3 "const_0_to_255_operand" "n,n")]
UNSPEC_DP))]
"TARGET_SSE4_1"
@@ -8773,6 +10640,16 @@
(set_attr "prefix" "maybe_vex")
(set_attr "mode" "TI")])
+(define_insn "avx512f_<code>v16qiv16si2"
+ [(set (match_operand:V16SI 0 "register_operand" "=v")
+ (any_extend:V16SI
+ (match_operand:V16QI 1 "nonimmediate_operand" "vm")))]
+ "TARGET_AVX512F"
+ "vpmov<extsuffix>bd\t{%1, %0|%0, %q1}"
+ [(set_attr "type" "ssemov")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "XI")])
+
(define_insn "avx2_<code>v8qiv8si2"
[(set (match_operand:V8SI 0 "register_operand" "=x")
(any_extend:V8SI
@@ -8803,6 +10680,16 @@
(set_attr "prefix" "maybe_vex")
(set_attr "mode" "TI")])
+(define_insn "avx512f_<code>v16hiv16si2"
+ [(set (match_operand:V16SI 0 "register_operand" "=v")
+ (any_extend:V16SI
+ (match_operand:V16HI 1 "nonimmediate_operand" "vm")))]
+ "TARGET_AVX512F"
+ "vpmov<extsuffix>wd\t{%1, %0|%0, %1}"
+ [(set_attr "type" "ssemov")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "XI")])
+
(define_insn "avx2_<code>v8hiv8si2"
[(set (match_operand:V8SI 0 "register_operand" "=x")
(any_extend:V8SI
@@ -8828,6 +10715,21 @@
(set_attr "prefix" "maybe_vex")
(set_attr "mode" "TI")])
+(define_insn "avx512f_<code>v8qiv8di2"
+ [(set (match_operand:V8DI 0 "register_operand" "=v")
+ (any_extend:V8DI
+ (vec_select:V8QI
+ (match_operand:V16QI 1 "nonimmediate_operand" "vm")
+ (parallel [(const_int 0) (const_int 1)
+ (const_int 2) (const_int 3)
+ (const_int 4) (const_int 5)
+ (const_int 6) (const_int 7)]))))]
+ "TARGET_AVX512F"
+ "vpmov<extsuffix>bq\t{%1, %0|%0, %k1}"
+ [(set_attr "type" "ssemov")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "XI")])
+
(define_insn "avx2_<code>v4qiv4di2"
[(set (match_operand:V4DI 0 "register_operand" "=x")
(any_extend:V4DI
@@ -8855,6 +10757,16 @@
(set_attr "prefix" "maybe_vex")
(set_attr "mode" "TI")])
+(define_insn "avx512f_<code>v8hiv8di2"
+ [(set (match_operand:V8DI 0 "register_operand" "=v")
+ (any_extend:V8DI
+ (match_operand:V8HI 1 "nonimmediate_operand" "vm")))]
+ "TARGET_AVX512F"
+ "vpmov<extsuffix>wq\t{%1, %0|%0, %q1}"
+ [(set_attr "type" "ssemov")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "XI")])
+
(define_insn "avx2_<code>v4hiv4di2"
[(set (match_operand:V4DI 0 "register_operand" "=x")
(any_extend:V4DI
@@ -8882,6 +10794,16 @@
(set_attr "prefix" "maybe_vex")
(set_attr "mode" "TI")])
+(define_insn "avx512f_<code>v8siv8di2"
+ [(set (match_operand:V8DI 0 "register_operand" "=v")
+ (any_extend:V8DI
+ (match_operand:V8SI 1 "nonimmediate_operand" "vm")))]
+ "TARGET_AVX512F"
+ "vpmov<extsuffix>dq\t{%1, %0|%0, %1}"
+ [(set_attr "type" "ssemov")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "XI")])
+
(define_insn "avx2_<code>v4siv4di2"
[(set (match_operand:V4DI 0 "register_operand" "=x")
(any_extend:V4DI
@@ -8909,8 +10831,8 @@
;; setting FLAGS_REG. But it is not a really compare instruction.
(define_insn "avx_vtest<ssemodesuffix><avxsizesuffix>"
[(set (reg:CC FLAGS_REG)
- (unspec:CC [(match_operand:VF 0 "register_operand" "x")
- (match_operand:VF 1 "nonimmediate_operand" "xm")]
+ (unspec:CC [(match_operand:VF_128_256 0 "register_operand" "x")
+ (match_operand:VF_128_256 1 "nonimmediate_operand" "xm")]
UNSPEC_VTESTP))]
"TARGET_AVX"
"vtest<ssemodesuffix>\t{%1, %0|%0, %1}"
@@ -8947,9 +10869,9 @@
(set_attr "mode" "TI")])
(define_insn "<sse4_1>_round<ssemodesuffix><avxsizesuffix>"
- [(set (match_operand:VF 0 "register_operand" "=x")
- (unspec:VF
- [(match_operand:VF 1 "nonimmediate_operand" "xm")
+ [(set (match_operand:VF_128_256 0 "register_operand" "=x")
+ (unspec:VF_128_256
+ [(match_operand:VF_128_256 1 "nonimmediate_operand" "xm")
(match_operand:SI 2 "const_0_to_15_operand" "n")]
UNSPEC_ROUND))]
"TARGET_ROUND"
@@ -8967,7 +10889,7 @@
(define_expand "<sse4_1>_round<ssemodesuffix>_sfix<avxsizesuffix>"
[(match_operand:<sseintvecmode> 0 "register_operand")
- (match_operand:VF1 1 "nonimmediate_operand")
+ (match_operand:VF1_128_256 1 "nonimmediate_operand")
(match_operand:SI 2 "const_0_to_15_operand")]
"TARGET_ROUND"
{
@@ -8981,6 +10903,16 @@
DONE;
})
+(define_expand "avx512f_roundpd512"
+ [(match_operand:V8DF 0 "register_operand")
+ (match_operand:V8DF 1 "nonimmediate_operand")
+ (match_operand:SI 2 "const_0_to_15_operand")]
+ "TARGET_AVX512F"
+{
+ emit_insn (gen_avx512f_rndscalev8df (operands[0], operands[1], operands[2]));
+ DONE;
+})
+
(define_expand "<sse4_1>_round<ssemodesuffix>_vec_pack_sfix<avxsizesuffix>"
[(match_operand:<ssepackfltmode> 0 "register_operand")
(match_operand:VF2 1 "nonimmediate_operand")
@@ -9076,7 +11008,7 @@
(define_expand "round<mode>2_sfix"
[(match_operand:<sseintvecmode> 0 "register_operand")
- (match_operand:VF1 1 "register_operand")]
+ (match_operand:VF1_128_256 1 "register_operand")]
"TARGET_ROUND && !flag_trapping_math"
{
rtx tmp = gen_reg_rtx (<MODE>mode);
@@ -9508,6 +11440,178 @@
(set_attr "btver2_decode" "vector,vector,vector,vector")
(set_attr "mode" "TI")])
+(define_expand "avx512pf_gatherpf<mode>"
+ [(unspec
+ [(match_operand:<avx512fmaskmode> 0 "register_or_constm1_operand")
+ (mem:<ssescalarmode>
+ (match_par_dup 5
+ [(match_operand 2 "vsib_address_operand")
+ (match_operand:VI48_512 1 "register_operand")
+ (match_operand:SI 3 "const1248_operand")]))
+ (match_operand:SI 4 "const_0_to_1_operand")]
+ UNSPEC_GATHER_PREFETCH)]
+ "TARGET_AVX512PF"
+{
+ operands[5]
+ = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[2], operands[1],
+ operands[3]), UNSPEC_VSIBADDR);
+})
+
+(define_insn "*avx512pf_gatherpf<mode>_mask"
+ [(unspec
+ [(match_operand:<avx512fmaskmode> 0 "register_operand" "k")
+ (match_operator:<ssescalarmode> 5 "vsib_mem_operator"
+ [(unspec:P
+ [(match_operand:P 2 "vsib_address_operand" "Tv")
+ (match_operand:VI48_512 1 "register_operand" "v")
+ (match_operand:SI 3 "const1248_operand" "n")]
+ UNSPEC_VSIBADDR)])
+ (match_operand:SI 4 "const_0_to_1_operand" "n")]
+ UNSPEC_GATHER_PREFETCH)]
+ "TARGET_AVX512PF"
+{
+ switch (INTVAL (operands[4]))
+ {
+ case 0:
+ return "vgatherpf0<ssemodesuffix>ps\t{%5%{%0%}|%5%{%0%}}";
+ case 1:
+ return "vgatherpf1<ssemodesuffix>ps\t{%5%{%0%}|%5%{%0%}}";
+ default:
+ gcc_unreachable ();
+ }
+}
+ [(set_attr "type" "sse")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "XI")])
+
+(define_insn "*avx512pf_gatherpf<mode>"
+ [(unspec
+ [(const_int -1)
+ (match_operator:<ssescalarmode> 4 "vsib_mem_operator"
+ [(unspec:P
+ [(match_operand:P 1 "vsib_address_operand" "Tv")
+ (match_operand:VI48_512 0 "register_operand" "v")
+ (match_operand:SI 2 "const1248_operand" "n")]
+ UNSPEC_VSIBADDR)])
+ (match_operand:SI 3 "const_0_to_1_operand" "n")]
+ UNSPEC_GATHER_PREFETCH)]
+ "TARGET_AVX512PF"
+{
+ switch (INTVAL (operands[3]))
+ {
+ case 0:
+ return "vgatherpf0<ssemodesuffix>ps\t{%4|%4}";
+ case 1:
+ return "vgatherpf1<ssemodesuffix>ps\t{%4|%4}";
+ default:
+ gcc_unreachable ();
+ }
+}
+ [(set_attr "type" "sse")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "XI")])
+
+(define_expand "avx512pf_scatterpf<mode>"
+ [(unspec
+ [(match_operand:<avx512fmaskmode> 0 "register_or_constm1_operand")
+ (mem:<ssescalarmode>
+ (match_par_dup 5
+ [(match_operand 2 "vsib_address_operand")
+ (match_operand:VI48_512 1 "register_operand")
+ (match_operand:SI 3 "const1248_operand")]))
+ (match_operand:SI 4 "const_0_to_1_operand")]
+ UNSPEC_SCATTER_PREFETCH)]
+ "TARGET_AVX512PF"
+{
+ operands[5]
+ = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[2], operands[1],
+ operands[3]), UNSPEC_VSIBADDR);
+})
+
+(define_insn "*avx512pf_scatterpf<mode>_mask"
+ [(unspec
+ [(match_operand:<avx512fmaskmode> 0 "register_operand" "k")
+ (match_operator:<ssescalarmode> 5 "vsib_mem_operator"
+ [(unspec:P
+ [(match_operand:P 2 "vsib_address_operand" "Tv")
+ (match_operand:VI48_512 1 "register_operand" "v")
+ (match_operand:SI 3 "const1248_operand" "n")]
+ UNSPEC_VSIBADDR)])
+ (match_operand:SI 4 "const_0_to_1_operand" "n")]
+ UNSPEC_SCATTER_PREFETCH)]
+ "TARGET_AVX512PF"
+{
+ switch (INTVAL (operands[4]))
+ {
+ case 0:
+ return "vscatterpf0<ssemodesuffix>ps\t{%5%{%0%}|%5%{%0%}}";
+ case 1:
+ return "vscatterpf1<ssemodesuffix>ps\t{%5%{%0%}|%5%{%0%}}";
+ default:
+ gcc_unreachable ();
+ }
+}
+ [(set_attr "type" "sse")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "XI")])
+
+(define_insn "*avx512pf_scatterpf<mode>"
+ [(unspec
+ [(const_int -1)
+ (match_operator:<ssescalarmode> 4 "vsib_mem_operator"
+ [(unspec:P
+ [(match_operand:P 1 "vsib_address_operand" "Tv")
+ (match_operand:VI48_512 0 "register_operand" "v")
+ (match_operand:SI 2 "const1248_operand" "n")]
+ UNSPEC_VSIBADDR)])
+ (match_operand:SI 3 "const_0_to_1_operand" "n")]
+ UNSPEC_SCATTER_PREFETCH)]
+ "TARGET_AVX512PF"
+{
+ switch (INTVAL (operands[3]))
+ {
+ case 0:
+ return "vscatterpf0<ssemodesuffix>ps\t{%4|%4}";
+ case 1:
+ return "vscatterpf1<ssemodesuffix>ps\t{%4|%4}";
+ default:
+ gcc_unreachable ();
+ }
+}
+ [(set_attr "type" "sse")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "XI")])
+
+(define_insn "*avx512er_exp2<mode>"
+ [(set (match_operand:VF_512 0 "register_operand" "=v")
+ (unspec:VF_512
+ [(match_operand:VF_512 1 "nonimmediate_operand" "vm")]
+ UNSPEC_EXP2))]
+ "TARGET_AVX512ER"
+ "vexp2<ssemodesuffix>\t{%1, %0|%0, %1}"
+ [(set_attr "prefix" "evex")
+ (set_attr "mode" "<MODE>")])
+
+(define_insn "*avx512er_rcp28<mode>"
+ [(set (match_operand:VF_512 0 "register_operand" "=v")
+ (unspec:VF_512
+ [(match_operand:VF_512 1 "nonimmediate_operand" "vm")]
+ UNSPEC_RCP28))]
+ "TARGET_AVX512ER"
+ "vrcp28<ssemodesuffix>\t{%1, %0|%0, %1}"
+ [(set_attr "prefix" "evex")
+ (set_attr "mode" "<MODE>")])
+
+(define_insn "avx512er_rsqrt28<mode>"
+ [(set (match_operand:VF_512 0 "register_operand" "=v")
+ (unspec:VF_512
+ [(match_operand:VF_512 1 "nonimmediate_operand" "vm")]
+ UNSPEC_RSQRT28))]
+ "TARGET_AVX512ER"
+ "vrsqrt28<ssemodesuffix>\t{%1, %0|%0, %1}"
+ [(set_attr "prefix" "evex")
+ (set_attr "mode" "<MODE>")])
+
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;
;; XOP instructions
@@ -10055,6 +12159,13 @@
})
(define_expand "vlshr<mode>3"
+ [(set (match_operand:VI48_512 0 "register_operand")
+ (lshiftrt:VI48_512
+ (match_operand:VI48_512 1 "register_operand")
+ (match_operand:VI48_512 2 "nonimmediate_operand")))]
+ "TARGET_AVX512F")
+
+(define_expand "vlshr<mode>3"
[(set (match_operand:VI48_256 0 "register_operand")
(lshiftrt:VI48_256
(match_operand:VI48_256 1 "register_operand")
@@ -10089,6 +12200,12 @@
}
})
+(define_expand "vashrv16si3"
+ [(set (match_operand:V16SI 0 "register_operand")
+ (ashiftrt:V16SI (match_operand:V16SI 1 "register_operand")
+ (match_operand:V16SI 2 "nonimmediate_operand")))]
+ "TARGET_AVX512F")
+
(define_expand "vashrv8si3"
[(set (match_operand:V8SI 0 "register_operand")
(ashiftrt:V8SI (match_operand:V8SI 1 "register_operand")
@@ -10122,6 +12239,13 @@
})
(define_expand "vashl<mode>3"
+ [(set (match_operand:VI48_512 0 "register_operand")
+ (ashift:VI48_512
+ (match_operand:VI48_512 1 "register_operand")
+ (match_operand:VI48_512 2 "nonimmediate_operand")))]
+ "TARGET_AVX512F")
+
+(define_expand "vashl<mode>3"
[(set (match_operand:VI48_256 0 "register_operand")
(ashift:VI48_256
(match_operand:VI48_256 1 "register_operand")
@@ -10341,10 +12465,10 @@
(set_attr "mode" "TI")])
(define_insn "xop_vpermil2<mode>3"
- [(set (match_operand:VF 0 "register_operand" "=x")
- (unspec:VF
- [(match_operand:VF 1 "register_operand" "x")
- (match_operand:VF 2 "nonimmediate_operand" "%x")
+ [(set (match_operand:VF_128_256 0 "register_operand" "=x")
+ (unspec:VF_128_256
+ [(match_operand:VF_128_256 1 "register_operand" "x")
+ (match_operand:VF_128_256 2 "nonimmediate_operand" "%x")
(match_operand:<sseintvecmode> 3 "nonimmediate_operand" "xm")
(match_operand:SI 4 "const_0_to_3_operand" "n")]
UNSPEC_VPERMIL2))]
@@ -10506,17 +12630,11 @@
(set_attr "btver2_decode" "vector")
(set_attr "mode" "OI")])
-(define_mode_attr AVXTOSSEMODE
- [(V4DI "V2DI") (V2DI "V2DI")
- (V8SI "V4SI") (V4SI "V4SI")
- (V16HI "V8HI") (V8HI "V8HI")
- (V32QI "V16QI") (V16QI "V16QI")])
-
(define_insn "avx2_pbroadcast<mode>"
[(set (match_operand:VI 0 "register_operand" "=x")
(vec_duplicate:VI
(vec_select:<ssescalarmode>
- (match_operand:<AVXTOSSEMODE> 1 "nonimmediate_operand" "xm")
+ (match_operand:<ssexmmmode> 1 "nonimmediate_operand" "xm")
(parallel [(const_int 0)]))))]
"TARGET_AVX2"
"vpbroadcast<ssemodesuffix>\t{%1, %0|%0, %<iptr>1}"
@@ -10540,37 +12658,37 @@
(set_attr "prefix" "vex")
(set_attr "mode" "<sseinsnmode>")])
-(define_insn "avx2_permvar<mode>"
- [(set (match_operand:VI4F_256 0 "register_operand" "=v")
- (unspec:VI4F_256
- [(match_operand:VI4F_256 1 "nonimmediate_operand" "vm")
- (match_operand:V8SI 2 "register_operand" "v")]
+(define_insn "<avx2_avx512f>_permvar<mode>"
+ [(set (match_operand:VI48F_256_512 0 "register_operand" "=v")
+ (unspec:VI48F_256_512
+ [(match_operand:VI48F_256_512 1 "nonimmediate_operand" "vm")
+ (match_operand:<sseintvecmode> 2 "register_operand" "v")]
UNSPEC_VPERMVAR))]
"TARGET_AVX2"
"vperm<ssemodesuffix>\t{%1, %2, %0|%0, %2, %1}"
[(set_attr "type" "sselog")
(set_attr "prefix" "vex")
- (set_attr "mode" "OI")])
+ (set_attr "mode" "<sseinsnmode>")])
-(define_expand "avx2_perm<mode>"
- [(match_operand:VI8F_256 0 "register_operand")
- (match_operand:VI8F_256 1 "nonimmediate_operand")
+(define_expand "<avx2_avx512f>_perm<mode>"
+ [(match_operand:VI8F_256_512 0 "register_operand")
+ (match_operand:VI8F_256_512 1 "nonimmediate_operand")
(match_operand:SI 2 "const_0_to_255_operand")]
"TARGET_AVX2"
{
int mask = INTVAL (operands[2]);
- emit_insn (gen_avx2_perm<mode>_1 (operands[0], operands[1],
- GEN_INT ((mask >> 0) & 3),
- GEN_INT ((mask >> 2) & 3),
- GEN_INT ((mask >> 4) & 3),
- GEN_INT ((mask >> 6) & 3)));
+ emit_insn (gen_<avx2_avx512f>_perm<mode>_1 (operands[0], operands[1],
+ GEN_INT ((mask >> 0) & 3),
+ GEN_INT ((mask >> 2) & 3),
+ GEN_INT ((mask >> 4) & 3),
+ GEN_INT ((mask >> 6) & 3)));
DONE;
})
-(define_insn "avx2_perm<mode>_1"
- [(set (match_operand:VI8F_256 0 "register_operand" "=v")
- (vec_select:VI8F_256
- (match_operand:VI8F_256 1 "nonimmediate_operand" "vm")
+(define_insn "<avx2_avx512f>_perm<mode>_1"
+ [(set (match_operand:VI8F_256_512 0 "register_operand" "=v")
+ (vec_select:VI8F_256_512
+ (match_operand:VI8F_256_512 1 "nonimmediate_operand" "vm")
(parallel [(match_operand 2 "const_0_to_3_operand")
(match_operand 3 "const_0_to_3_operand")
(match_operand 4 "const_0_to_3_operand")
@@ -10633,6 +12751,62 @@
(set_attr "isa" "*,avx2,noavx2")
(set_attr "mode" "V8SF")])
+(define_insn "avx512f_vec_dup<mode>"
+ [(set (match_operand:VI48F_512 0 "register_operand" "=v")
+ (vec_duplicate:VI48F_512
+ (vec_select:<ssescalarmode>
+ (match_operand:<ssexmmmode> 1 "nonimmediate_operand" "vm")
+ (parallel [(const_int 0)]))))]
+ "TARGET_AVX512F"
+ "v<sseintprefix>broadcast<bcstscalarsuff>\t{%1, %0|%0, %1}"
+ [(set_attr "type" "ssemov")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "<sseinsnmode>")])
+
+(define_insn "avx512f_broadcast<mode>"
+ [(set (match_operand:V16FI 0 "register_operand" "=v,v")
+ (vec_duplicate:V16FI
+ (match_operand:<ssexmmmode> 1 "nonimmediate_operand" "v,m")))]
+ "TARGET_AVX512F"
+ "@
+ vshuf<shuffletype>32x4\t{$0x0, %g1, %g1, %0|%0, %g1, %g1, 0x0}
+ vbroadcast<shuffletype>32x4\t{%1, %0|%0, %1}"
+ [(set_attr "type" "ssemov")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "<sseinsnmode>")])
+
+(define_insn "avx512f_broadcast<mode>"
+ [(set (match_operand:V8FI 0 "register_operand" "=v,v")
+ (vec_duplicate:V8FI
+ (match_operand:<ssehalfvecmode> 1 "nonimmediate_operand" "v,m")))]
+ "TARGET_AVX512F"
+ "@
+ vshuf<shuffletype>64x2\t{$0x44, %g1, %g1, %0|%0, %g1, %g1, 0x44}
+ vbroadcast<shuffletype>64x4\t{%1, %0|%0, %1}"
+ [(set_attr "type" "ssemov")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "<sseinsnmode>")])
+
+(define_insn "avx512f_vec_dup_gpr<mode>"
+ [(set (match_operand:VI48_512 0 "register_operand" "=v")
+ (vec_duplicate:VI48_512
+ (match_operand:<ssescalarmode> 1 "register_operand" "r")))]
+ "TARGET_AVX512F && (<MODE>mode != V8DImode || TARGET_64BIT)"
+ "vpbroadcast<bcstscalarsuff>\t{%1, %0|%0, %1}"
+ [(set_attr "type" "ssemov")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "<sseinsnmode>")])
+
+(define_insn "avx512f_vec_dup_mem<mode>"
+ [(set (match_operand:VI48F_512 0 "register_operand" "=v")
+ (vec_duplicate:VI48F_512
+ (match_operand:<ssescalarmode> 1 "nonimmediate_operand" "vm")))]
+ "TARGET_AVX512F"
+ "v<sseintprefix>broadcast<bcstscalarsuff>\t{%1, %0|%0, %1}"
+ [(set_attr "type" "ssemov")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "<sseinsnmode>")])
+
(define_insn "avx2_vbroadcasti128_<mode>"
[(set (match_operand:VI_256 0 "register_operand" "=x")
(vec_concat:VI_256
@@ -10672,6 +12846,28 @@
(set_attr "prefix" "vex")
(set_attr "mode" "<sseinsnmode>")])
+(define_insn "avx512cd_maskb_vec_dupv8di"
+ [(set (match_operand:V8DI 0 "register_operand" "=v")
+ (vec_duplicate:V8DI
+ (zero_extend:DI
+ (match_operand:QI 1 "register_operand" "k"))))]
+ "TARGET_AVX512CD"
+ "vpbroadcastmb2q\t{%1, %0|%0, %1}"
+ [(set_attr "type" "mskmov")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "XI")])
+
+(define_insn "avx512cd_maskw_vec_dupv16si"
+ [(set (match_operand:V16SI 0 "register_operand" "=v")
+ (vec_duplicate:V16SI
+ (zero_extend:SI
+ (match_operand:HI 1 "register_operand" "k"))))]
+ "TARGET_AVX512CD"
+ "vpbroadcastmw2d\t{%1, %0|%0, %1}"
+ [(set_attr "type" "mskmov")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "XI")])
+
;; Recognize broadcast as a vec_select as produced by builtin_vec_perm.
;; If it so happens that the input is in memory, use vbroadcast.
;; Otherwise use vpermilp (and in the case of 256-bit modes, vperm2f128).
@@ -10746,7 +12942,7 @@
elt * GET_MODE_SIZE (<ssescalarmode>mode));
})
-(define_expand "avx_vpermil<mode>"
+(define_expand "<sse2_avx_avx512f>_vpermil<mode>"
[(set (match_operand:VF2 0 "register_operand")
(vec_select:VF2
(match_operand:VF2 1 "nonimmediate_operand")
@@ -10756,19 +12952,18 @@
int mask = INTVAL (operands[2]);
rtx perm[<ssescalarnum>];
- perm[0] = GEN_INT (mask & 1);
- perm[1] = GEN_INT ((mask >> 1) & 1);
- if (<MODE>mode == V4DFmode)
+ int i;
+ for (i = 0; i < <ssescalarnum>; i = i + 2)
{
- perm[2] = GEN_INT (((mask >> 2) & 1) + 2);
- perm[3] = GEN_INT (((mask >> 3) & 1) + 2);
+ perm[i] = GEN_INT (((mask >> i) & 1) + i);
+ perm[i + 1] = GEN_INT (((mask >> (i + 1)) & 1) + i);
}
operands[2]
= gen_rtx_PARALLEL (VOIDmode, gen_rtvec_v (<ssescalarnum>, perm));
})
-(define_expand "avx_vpermil<mode>"
+(define_expand "<sse2_avx_avx512f>_vpermil<mode>"
[(set (match_operand:VF1 0 "register_operand")
(vec_select:VF1
(match_operand:VF1 1 "nonimmediate_operand")
@@ -10778,23 +12973,20 @@
int mask = INTVAL (operands[2]);
rtx perm[<ssescalarnum>];
- perm[0] = GEN_INT (mask & 3);
- perm[1] = GEN_INT ((mask >> 2) & 3);
- perm[2] = GEN_INT ((mask >> 4) & 3);
- perm[3] = GEN_INT ((mask >> 6) & 3);
- if (<MODE>mode == V8SFmode)
+ int i;
+ for (i = 0; i < <ssescalarnum>; i = i + 4)
{
- perm[4] = GEN_INT ((mask & 3) + 4);
- perm[5] = GEN_INT (((mask >> 2) & 3) + 4);
- perm[6] = GEN_INT (((mask >> 4) & 3) + 4);
- perm[7] = GEN_INT (((mask >> 6) & 3) + 4);
+ perm[i] = GEN_INT (((mask >> 0) & 3) + i);
+ perm[i + 1] = GEN_INT (((mask >> 2) & 3) + i);
+ perm[i + 2] = GEN_INT (((mask >> 4) & 3) + i);
+ perm[i + 3] = GEN_INT (((mask >> 6) & 3) + i);
}
operands[2]
= gen_rtx_PARALLEL (VOIDmode, gen_rtvec_v (<ssescalarnum>, perm));
})
-(define_insn "*avx_vpermilp<mode>"
+(define_insn "*<sse2_avx_avx512f>_vpermilp<mode>"
[(set (match_operand:VF 0 "register_operand" "=v")
(vec_select:VF
(match_operand:VF 1 "nonimmediate_operand" "vm")
@@ -10811,9 +13003,9 @@
(set_attr "prefix_extra" "1")
(set_attr "length_immediate" "1")
(set_attr "prefix" "vex")
- (set_attr "mode" "<MODE>")])
+ (set_attr "mode" "<sseinsnmode>")])
-(define_insn "avx_vpermilvar<mode>3"
+(define_insn "<sse2_avx_avx512f>_vpermilvar<mode>3"
[(set (match_operand:VF 0 "register_operand" "=v")
(unspec:VF
[(match_operand:VF 1 "register_operand" "v")
@@ -10823,9 +13015,35 @@
"vpermil<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
[(set_attr "type" "sselog")
(set_attr "prefix_extra" "1")
- (set_attr "prefix" "vex")
(set_attr "btver2_decode" "vector")
- (set_attr "mode" "<MODE>")])
+ (set_attr "prefix" "vex")
+ (set_attr "mode" "<sseinsnmode>")])
+
+(define_insn "avx512f_vpermi2var<mode>3"
+ [(set (match_operand:VI48F_512 0 "register_operand" "=v")
+ (unspec:VI48F_512
+ [(match_operand:VI48F_512 1 "register_operand" "v")
+ (match_operand:<sseintvecmode> 2 "register_operand" "0")
+ (match_operand:VI48F_512 3 "nonimmediate_operand" "vm")]
+ UNSPEC_VPERMI2))]
+ "TARGET_AVX512F"
+ "vpermi2<ssemodesuffix>\t{%3, %1, %0|%0, %1, %3}"
+ [(set_attr "type" "sselog")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "<sseinsnmode>")])
+
+(define_insn "avx512f_vpermt2var<mode>3"
+ [(set (match_operand:VI48F_512 0 "register_operand" "=v")
+ (unspec:VI48F_512
+ [(match_operand:<sseintvecmode> 1 "register_operand" "v")
+ (match_operand:VI48F_512 2 "register_operand" "0")
+ (match_operand:VI48F_512 3 "nonimmediate_operand" "vm")]
+ UNSPEC_VPERMT2))]
+ "TARGET_AVX512F"
+ "vpermt2<ssemodesuffix>\t{%3, %1, %0|%0, %1, %3}"
+ [(set_attr "type" "sselog")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "<sseinsnmode>")])
(define_expand "avx_vperm2f128<mode>3"
[(set (match_operand:AVX256MODE2P 0 "register_operand")
@@ -11159,6 +13377,15 @@
DONE;
})
+(define_expand "vec_init<mode>"
+ [(match_operand:VI48F_512 0 "register_operand")
+ (match_operand 1)]
+ "TARGET_AVX512F"
+{
+ ix86_expand_vector_init (false, operands[0], operands[1]);
+ DONE;
+})
+
(define_expand "avx2_extracti128"
[(match_operand:V2DI 0 "nonimmediate_operand")
(match_operand:V4DI 1 "register_operand")
@@ -11208,31 +13435,36 @@
DONE;
})
-(define_insn "avx2_ashrv<mode>"
- [(set (match_operand:VI4_AVX2 0 "register_operand" "=v")
- (ashiftrt:VI4_AVX2
- (match_operand:VI4_AVX2 1 "register_operand" "v")
- (match_operand:VI4_AVX2 2 "nonimmediate_operand" "vm")))]
+(define_insn "<avx2_avx512f>_ashrv<mode>"
+ [(set (match_operand:VI48_AVX512F 0 "register_operand" "=v")
+ (ashiftrt:VI48_AVX512F
+ (match_operand:VI48_AVX512F 1 "register_operand" "v")
+ (match_operand:VI48_AVX512F 2 "nonimmediate_operand" "vm")))]
"TARGET_AVX2"
- "vpsravd\t{%2, %1, %0|%0, %1, %2}"
+ "vpsrav<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
[(set_attr "type" "sseishft")
- (set_attr "prefix" "vex")
+ (set_attr "prefix" "maybe_evex")
(set_attr "mode" "<sseinsnmode>")])
-(define_insn "avx2_<shift_insn>v<mode>"
- [(set (match_operand:VI48_AVX2 0 "register_operand" "=v")
- (any_lshift:VI48_AVX2
- (match_operand:VI48_AVX2 1 "register_operand" "v")
- (match_operand:VI48_AVX2 2 "nonimmediate_operand" "vm")))]
+(define_insn "<avx2_avx512f>_<shift_insn>v<mode>"
+ [(set (match_operand:VI48_AVX2_48_AVX512F 0 "register_operand" "=v")
+ (any_lshift:VI48_AVX2_48_AVX512F
+ (match_operand:VI48_AVX2_48_AVX512F 1 "register_operand" "v")
+ (match_operand:VI48_AVX2_48_AVX512F 2 "nonimmediate_operand" "vm")))]
"TARGET_AVX2"
"vp<vshift>v<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
[(set_attr "type" "sseishft")
- (set_attr "prefix" "vex")
+ (set_attr "prefix" "maybe_evex")
(set_attr "mode" "<sseinsnmode>")])
+;; For avx_vec_concat<mode> insn pattern
+(define_mode_attr concat_tg_mode
+ [(V32QI "t") (V16HI "t") (V8SI "t") (V4DI "t") (V8SF "t") (V4DF "t")
+ (V64QI "g") (V32HI "g") (V16SI "g") (V8DI "g") (V16SF "g") (V8DF "g")])
+
(define_insn "avx_vec_concat<mode>"
- [(set (match_operand:V_256 0 "register_operand" "=x,x")
- (vec_concat:V_256
+ [(set (match_operand:V_256_512 0 "register_operand" "=x,x")
+ (vec_concat:V_256_512
(match_operand:<ssehalfvecmode> 1 "register_operand" "x,x")
(match_operand:<ssehalfvecmode> 2 "vector_move_operand" "xm,C")))]
"TARGET_AVX"
@@ -11240,16 +13472,24 @@
switch (which_alternative)
{
case 0:
- return "vinsert<i128>\t{$0x1, %2, %t1, %0|%0, %t1, %2, 0x1}";
+ return "vinsert<i128>\t{$0x1, %2, %<concat_tg_mode>1, %0|%0, %<concat_tg_mode>1, %2, 0x1}";
case 1:
switch (get_attr_mode (insn))
{
+ case MODE_V16SF:
+ return "vmovaps\t{%1, %t0|%t0, %1}";
+ case MODE_V8DF:
+ return "vmovapd\t{%1, %t0|%t0, %1}";
case MODE_V8SF:
return "vmovaps\t{%1, %x0|%x0, %1}";
case MODE_V4DF:
return "vmovapd\t{%1, %x0|%x0, %1}";
- default:
+ case MODE_XI:
+ return "vmovdqa\t{%1, %t0|%t0, %1}";
+ case MODE_OI:
return "vmovdqa\t{%1, %x0|%x0, %1}";
+ default:
+ gcc_unreachable ();
}
default:
gcc_unreachable ();
@@ -11258,7 +13498,7 @@
[(set_attr "type" "sselog,ssemov")
(set_attr "prefix_extra" "1,*")
(set_attr "length_immediate" "1,*")
- (set_attr "prefix" "vex")
+ (set_attr "prefix" "maybe_evex")
(set_attr "mode" "<sseinsnmode>")])
(define_insn "vcvtph2ps"
@@ -11295,6 +13535,16 @@
(set_attr "btver2_decode" "double")
(set_attr "mode" "V8SF")])
+(define_insn "avx512f_vcvtph2ps512"
+ [(set (match_operand:V16SF 0 "register_operand" "=v")
+ (unspec:V16SF [(match_operand:V16HI 1 "nonimmediate_operand" "vm")]
+ UNSPEC_VCVTPH2PS))]
+ "TARGET_AVX512F"
+ "vcvtph2ps\t{%1, %0|%0, %1}"
+ [(set_attr "type" "ssecvt")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "V16SF")])
+
(define_expand "vcvtps2ph"
[(set (match_operand:V8HI 0 "register_operand")
(vec_concat:V8HI
@@ -11341,24 +13591,37 @@
(set_attr "btver2_decode" "vector")
(set_attr "mode" "V8SF")])
+(define_insn "avx512f_vcvtps2ph512"
+ [(set (match_operand:V16HI 0 "nonimmediate_operand" "=vm")
+ (unspec:V16HI [(match_operand:V16SF 1 "register_operand" "v")
+ (match_operand:SI 2 "const_0_to_255_operand" "N")]
+ UNSPEC_VCVTPS2PH))]
+ "TARGET_AVX512F"
+ "vcvtps2ph\t{%2, %1, %0|%0, %1, %2}"
+ [(set_attr "type" "ssecvt")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "V16SF")])
+
;; For gather* insn patterns
(define_mode_iterator VEC_GATHER_MODE
[V2DI V2DF V4DI V4DF V4SI V4SF V8SI V8SF])
(define_mode_attr VEC_GATHER_IDXSI
- [(V2DI "V4SI") (V2DF "V4SI")
- (V4DI "V4SI") (V4DF "V4SI")
- (V4SI "V4SI") (V4SF "V4SI")
- (V8SI "V8SI") (V8SF "V8SI")])
+ [(V2DI "V4SI") (V4DI "V4SI") (V8DI "V8SI")
+ (V2DF "V4SI") (V4DF "V4SI") (V8DF "V8SI")
+ (V4SI "V4SI") (V8SI "V8SI") (V16SI "V16SI")
+ (V4SF "V4SI") (V8SF "V8SI") (V16SF "V16SI")])
+
(define_mode_attr VEC_GATHER_IDXDI
- [(V2DI "V2DI") (V2DF "V2DI")
- (V4DI "V4DI") (V4DF "V4DI")
- (V4SI "V2DI") (V4SF "V2DI")
- (V8SI "V4DI") (V8SF "V4DI")])
+ [(V2DI "V2DI") (V4DI "V4DI") (V8DI "V8DI")
+ (V2DF "V2DI") (V4DF "V4DI") (V8DF "V8DI")
+ (V4SI "V2DI") (V8SI "V4DI") (V16SI "V8DI")
+ (V4SF "V2DI") (V8SF "V4DI") (V16SF "V8DI")])
+
(define_mode_attr VEC_GATHER_SRCDI
- [(V2DI "V2DI") (V2DF "V2DF")
- (V4DI "V4DI") (V4DF "V4DF")
- (V4SI "V4SI") (V4SF "V4SF")
- (V8SI "V4SI") (V8SF "V4SF")])
+ [(V2DI "V2DI") (V4DI "V4DI") (V8DI "V8DI")
+ (V2DF "V2DF") (V4DF "V4DF") (V8DF "V8DF")
+ (V4SI "V4SI") (V8SI "V4SI") (V16SI "V8SI")
+ (V4SF "V4SF") (V8SF "V4SF") (V16SF "V8SF")])
(define_expand "avx2_gathersi<mode>"
[(parallel [(set (match_operand:VEC_GATHER_MODE 0 "register_operand")
@@ -11387,7 +13650,7 @@
[(match_operand:VEC_GATHER_MODE 2 "register_operand" "0")
(match_operator:<ssescalarmode> 7 "vsib_mem_operator"
[(unspec:P
- [(match_operand:P 3 "vsib_address_operand" "p")
+ [(match_operand:P 3 "vsib_address_operand" "Tv")
(match_operand:<VEC_GATHER_IDXSI> 4 "register_operand" "x")
(match_operand:SI 6 "const1248_operand" "n")]
UNSPEC_VSIBADDR)])
@@ -11407,7 +13670,7 @@
[(pc)
(match_operator:<ssescalarmode> 6 "vsib_mem_operator"
[(unspec:P
- [(match_operand:P 2 "vsib_address_operand" "p")
+ [(match_operand:P 2 "vsib_address_operand" "Tv")
(match_operand:<VEC_GATHER_IDXSI> 3 "register_operand" "x")
(match_operand:SI 5 "const1248_operand" "n")]
UNSPEC_VSIBADDR)])
@@ -11449,7 +13712,7 @@
[(match_operand:<VEC_GATHER_SRCDI> 2 "register_operand" "0")
(match_operator:<ssescalarmode> 7 "vsib_mem_operator"
[(unspec:P
- [(match_operand:P 3 "vsib_address_operand" "p")
+ [(match_operand:P 3 "vsib_address_operand" "Tv")
(match_operand:<VEC_GATHER_IDXDI> 4 "register_operand" "x")
(match_operand:SI 6 "const1248_operand" "n")]
UNSPEC_VSIBADDR)])
@@ -11469,7 +13732,7 @@
[(pc)
(match_operator:<ssescalarmode> 6 "vsib_mem_operator"
[(unspec:P
- [(match_operand:P 2 "vsib_address_operand" "p")
+ [(match_operand:P 2 "vsib_address_operand" "Tv")
(match_operand:<VEC_GATHER_IDXDI> 3 "register_operand" "x")
(match_operand:SI 5 "const1248_operand" "n")]
UNSPEC_VSIBADDR)])
@@ -11494,7 +13757,7 @@
[(match_operand:<VEC_GATHER_SRCDI> 2 "register_operand" "0")
(match_operator:<ssescalarmode> 7 "vsib_mem_operator"
[(unspec:P
- [(match_operand:P 3 "vsib_address_operand" "p")
+ [(match_operand:P 3 "vsib_address_operand" "Tv")
(match_operand:<VEC_GATHER_IDXDI> 4 "register_operand" "x")
(match_operand:SI 6 "const1248_operand" "n")]
UNSPEC_VSIBADDR)])
@@ -11517,7 +13780,7 @@
[(pc)
(match_operator:<ssescalarmode> 6 "vsib_mem_operator"
[(unspec:P
- [(match_operand:P 2 "vsib_address_operand" "p")
+ [(match_operand:P 2 "vsib_address_operand" "Tv")
(match_operand:<VEC_GATHER_IDXDI> 3 "register_operand" "x")
(match_operand:SI 5 "const1248_operand" "n")]
UNSPEC_VSIBADDR)])
@@ -11532,3 +13795,241 @@
[(set_attr "type" "ssemov")
(set_attr "prefix" "vex")
(set_attr "mode" "<sseinsnmode>")])
+
+(define_expand "avx512f_gathersi<mode>"
+ [(parallel [(set (match_operand:VI48F_512 0 "register_operand")
+ (unspec:VI48F_512
+ [(match_operand:VI48F_512 1 "register_operand")
+ (match_operand:<avx512fmaskmode> 4 "register_operand")
+ (mem:<ssescalarmode>
+ (match_par_dup 6
+ [(match_operand 2 "vsib_address_operand")
+ (match_operand:<VEC_GATHER_IDXSI> 3 "register_operand")
+ (match_operand:SI 5 "const1248_operand")]))]
+ UNSPEC_GATHER))
+ (clobber (match_scratch:<avx512fmaskmode> 7))])]
+ "TARGET_AVX512F"
+{
+ operands[6]
+ = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[2], operands[3],
+ operands[5]), UNSPEC_VSIBADDR);
+})
+
+(define_insn "*avx512f_gathersi<mode>"
+ [(set (match_operand:VI48F_512 0 "register_operand" "=&v")
+ (unspec:VI48F_512
+ [(match_operand:VI48F_512 1 "register_operand" "0")
+ (match_operand:<avx512fmaskmode> 7 "register_operand" "2")
+ (match_operator:<ssescalarmode> 6 "vsib_mem_operator"
+ [(unspec:P
+ [(match_operand:P 4 "vsib_address_operand" "Tv")
+ (match_operand:<VEC_GATHER_IDXSI> 3 "register_operand" "v")
+ (match_operand:SI 5 "const1248_operand" "n")]
+ UNSPEC_VSIBADDR)])]
+ UNSPEC_GATHER))
+ (clobber (match_scratch:<avx512fmaskmode> 2 "=&k"))]
+ "TARGET_AVX512F"
+ "v<sseintprefix>gatherd<ssemodesuffix>\t{%6, %0%{%2%}|%0%{%2%}, %g6}"
+ [(set_attr "type" "ssemov")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "<sseinsnmode>")])
+
+(define_insn "*avx512f_gathersi<mode>_2"
+ [(set (match_operand:VI48F_512 0 "register_operand" "=&v")
+ (unspec:VI48F_512
+ [(pc)
+ (match_operand:<avx512fmaskmode> 6 "register_operand" "1")
+ (match_operator:<ssescalarmode> 5 "vsib_mem_operator"
+ [(unspec:P
+ [(match_operand:P 3 "vsib_address_operand" "Tv")
+ (match_operand:<VEC_GATHER_IDXSI> 2 "register_operand" "v")
+ (match_operand:SI 4 "const1248_operand" "n")]
+ UNSPEC_VSIBADDR)])]
+ UNSPEC_GATHER))
+ (clobber (match_scratch:<avx512fmaskmode> 1 "=&k"))]
+ "TARGET_AVX512F"
+ "v<sseintprefix>gatherd<ssemodesuffix>\t{%5, %0%{%1%}|%0%{%1%}, %g5}"
+ [(set_attr "type" "ssemov")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "<sseinsnmode>")])
+
+
+(define_expand "avx512f_gatherdi<mode>"
+ [(parallel [(set (match_operand:VI48F_512 0 "register_operand")
+ (unspec:VI48F_512
+ [(match_operand:<VEC_GATHER_SRCDI> 1 "register_operand")
+ (match_operand:QI 4 "register_operand")
+ (mem:<ssescalarmode>
+ (match_par_dup 6
+ [(match_operand 2 "vsib_address_operand")
+ (match_operand:<VEC_GATHER_IDXDI> 3 "register_operand")
+ (match_operand:SI 5 "const1248_operand")]))]
+ UNSPEC_GATHER))
+ (clobber (match_scratch:QI 7))])]
+ "TARGET_AVX512F"
+{
+ operands[6]
+ = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[2], operands[3],
+ operands[5]), UNSPEC_VSIBADDR);
+})
+
+(define_insn "*avx512f_gatherdi<mode>"
+ [(set (match_operand:VI48F_512 0 "register_operand" "=&v")
+ (unspec:VI48F_512
+ [(match_operand:<VEC_GATHER_SRCDI> 1 "register_operand" "0")
+ (match_operand:QI 7 "register_operand" "2")
+ (match_operator:<ssescalarmode> 6 "vsib_mem_operator"
+ [(unspec:P
+ [(match_operand:P 4 "vsib_address_operand" "Tv")
+ (match_operand:<VEC_GATHER_IDXDI> 3 "register_operand" "v")
+ (match_operand:SI 5 "const1248_operand" "n")]
+ UNSPEC_VSIBADDR)])]
+ UNSPEC_GATHER))
+ (clobber (match_scratch:QI 2 "=&k"))]
+ "TARGET_AVX512F"
+ "v<sseintprefix>gatherq<ssemodesuffix>\t{%6, %1%{%2%}|%1%{%2%}, %g6}"
+ [(set_attr "type" "ssemov")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "<sseinsnmode>")])
+
+(define_insn "*avx512f_gatherdi<mode>_2"
+ [(set (match_operand:VI48F_512 0 "register_operand" "=&v")
+ (unspec:VI48F_512
+ [(pc)
+ (match_operand:QI 6 "register_operand" "1")
+ (match_operator:<ssescalarmode> 5 "vsib_mem_operator"
+ [(unspec:P
+ [(match_operand:P 3 "vsib_address_operand" "Tv")
+ (match_operand:<VEC_GATHER_IDXDI> 2 "register_operand" "v")
+ (match_operand:SI 4 "const1248_operand" "n")]
+ UNSPEC_VSIBADDR)])]
+ UNSPEC_GATHER))
+ (clobber (match_scratch:QI 1 "=&k"))]
+ "TARGET_AVX512F"
+{
+ if (<MODE>mode != <VEC_GATHER_SRCDI>mode)
+ return "v<sseintprefix>gatherq<ssemodesuffix>\t{%5, %t0%{%1%}|%t0%{%1%}, %g5}";
+ return "v<sseintprefix>gatherq<ssemodesuffix>\t{%5, %0%{%1%}|%0%{%1%}, %g5}";
+}
+ [(set_attr "type" "ssemov")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "<sseinsnmode>")])
+
+(define_expand "avx512f_scattersi<mode>"
+ [(parallel [(set (mem:VI48F_512
+ (match_par_dup 5
+ [(match_operand 0 "vsib_address_operand")
+ (match_operand:<VEC_GATHER_IDXSI> 2 "register_operand")
+ (match_operand:SI 4 "const1248_operand")]))
+ (unspec:VI48F_512
+ [(match_operand:<avx512fmaskmode> 1 "register_operand")
+ (match_operand:VI48F_512 3 "register_operand")]
+ UNSPEC_SCATTER))
+ (clobber (match_scratch:<avx512fmaskmode> 6))])]
+ "TARGET_AVX512F"
+{
+ operands[5]
+ = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[0], operands[2],
+ operands[4]), UNSPEC_VSIBADDR);
+})
+
+(define_insn "*avx512f_scattersi<mode>"
+ [(set (match_operator:VI48F_512 5 "vsib_mem_operator"
+ [(unspec:P
+ [(match_operand:P 0 "vsib_address_operand" "Tv")
+ (match_operand:<VEC_GATHER_IDXSI> 2 "register_operand" "v")
+ (match_operand:SI 4 "const1248_operand" "n")]
+ UNSPEC_VSIBADDR)])
+ (unspec:VI48F_512
+ [(match_operand:<avx512fmaskmode> 6 "register_operand" "1")
+ (match_operand:VI48F_512 3 "register_operand" "v")]
+ UNSPEC_SCATTER))
+ (clobber (match_scratch:<avx512fmaskmode> 1 "=&k"))]
+ "TARGET_AVX512F"
+ "v<sseintprefix>scatterd<ssemodesuffix>\t{%3, %5%{%1%}|%5%{%1%}, %3}"
+ [(set_attr "type" "ssemov")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "<sseinsnmode>")])
+
+(define_expand "avx512f_scatterdi<mode>"
+ [(parallel [(set (mem:VI48F_512
+ (match_par_dup 5
+ [(match_operand 0 "vsib_address_operand")
+ (match_operand:V8DI 2 "register_operand")
+ (match_operand:SI 4 "const1248_operand")]))
+ (unspec:VI48F_512
+ [(match_operand:QI 1 "register_operand")
+ (match_operand:<VEC_GATHER_SRCDI> 3 "register_operand")]
+ UNSPEC_SCATTER))
+ (clobber (match_scratch:QI 6))])]
+ "TARGET_AVX512F"
+{
+ operands[5]
+ = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[0], operands[2],
+ operands[4]), UNSPEC_VSIBADDR);
+})
+
+(define_insn "*avx512f_scatterdi<mode>"
+ [(set (match_operator:VI48F_512 5 "vsib_mem_operator"
+ [(unspec:P
+ [(match_operand:P 0 "vsib_address_operand" "Tv")
+ (match_operand:V8DI 2 "register_operand" "v")
+ (match_operand:SI 4 "const1248_operand" "n")]
+ UNSPEC_VSIBADDR)])
+ (unspec:VI48F_512
+ [(match_operand:QI 6 "register_operand" "1")
+ (match_operand:<VEC_GATHER_SRCDI> 3 "register_operand" "v")]
+ UNSPEC_SCATTER))
+ (clobber (match_scratch:QI 1 "=&k"))]
+ "TARGET_AVX512F"
+ "v<sseintprefix>scatterq<ssemodesuffix>\t{%3, %5%{%1%}|%5%{%1%}, %3}"
+ [(set_attr "type" "ssemov")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "<sseinsnmode>")])
+
+(define_insn "avx512f_getmant<mode>"
+ [(set (match_operand:VF_512 0 "register_operand" "=v")
+ (unspec:VF_512
+ [(match_operand:VF_512 1 "nonimmediate_operand" "vm")
+ (match_operand:SI 2 "const_0_to_15_operand")]
+ UNSPEC_GETMANT))]
+ "TARGET_AVX512F"
+ "vgetmant<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}";
+ [(set_attr "prefix" "evex")
+ (set_attr "mode" "<MODE>")])
+
+(define_insn "avx512f_getmant<mode>"
+ [(set (match_operand:VF_128 0 "register_operand" "=v")
+ (vec_merge:VF_128
+ (unspec:VF_128
+ [(match_operand:VF_128 1 "register_operand" "v")
+ (match_operand:VF_128 2 "nonimmediate_operand" "vm")
+ (match_operand:SI 3 "const_0_to_15_operand")]
+ UNSPEC_GETMANT)
+ (match_dup 1)
+ (const_int 1)))]
+ "TARGET_AVX512F"
+ "vgetmant<ssescalarmodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}";
+ [(set_attr "prefix" "evex")
+ (set_attr "mode" "<ssescalarmode>")])
+
+(define_insn "clz<mode>2"
+ [(set (match_operand:VI48_512 0 "register_operand" "=v")
+ (clz:VI48_512
+ (match_operand:VI48_512 1 "nonimmediate_operand" "vm")))]
+ "TARGET_AVX512CD"
+ "vplzcnt<ssemodesuffix>\t{%1, %0|%0, %1}"
+ [(set_attr "type" "sse")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "<sseinsnmode>")])
+
+(define_insn "conflict<mode>"
+ [(set (match_operand:VI48_512 0 "register_operand" "=v")
+ (unspec:VI48_512
+ [(match_operand:VI48_512 1 "nonimmediate_operand" "vm")]
+ UNSPEC_CONFLICT))]
+ "TARGET_AVX512CD"
+ "vpconflict<ssemodesuffix>\t{%1, %0|%0, %1}"
+ [(set_attr "type" "sse")
+ (set_attr "prefix" "evex")
+ (set_attr "mode" "<sseinsnmode>")])
diff --git a/gcc/config/i386/sync.md b/gcc/config/i386/sync.md
index 9e5835662e1..8408a2bfe43 100644
--- a/gcc/config/i386/sync.md
+++ b/gcc/config/i386/sync.md
@@ -430,10 +430,21 @@
const char *xchg = "xchg{<imodesuffix>}\t%%<regprefix>bx, %5";
if (swap)
- output_asm_insn (xchg, operands);
+ {
+ output_asm_insn (xchg, operands);
+ if (ix86_emit_cfi ())
+ {
+ output_asm_insn (".cfi_remember_state", operands);
+ output_asm_insn (".cfi_register\t%%<regprefix>bx, %5", operands);
+ }
+ }
output_asm_insn ("lock{%;} %K7cmpxchg<doublemodesuffix>b\t%2", operands);
if (swap)
- output_asm_insn (xchg, operands);
+ {
+ output_asm_insn (xchg, operands);
+ if (ix86_emit_cfi ())
+ output_asm_insn (".cfi_restore_state", operands);
+ }
return "";
})
diff --git a/gcc/config/i386/t-i386 b/gcc/config/i386/t-i386
index 07624cc575e..1a76c4152f6 100644
--- a/gcc/config/i386/t-i386
+++ b/gcc/config/i386/t-i386
@@ -16,22 +16,9 @@
# along with GCC; see the file COPYING3. If not see
# <http://www.gnu.org/licenses/>.
-i386.o: $(CONFIG_H) $(SYSTEM_H) coretypes.h dumpfile.h $(TM_H) \
- $(RTL_H) $(TREE_H) $(TM_P_H) $(REGS_H) hard-reg-set.h \
- $(REAL_H) insn-config.h conditions.h output.h insn-codes.h \
- $(INSN_ATTR_H) $(FLAGS_H) $(C_COMMON_H) except.h $(FUNCTION_H) \
- $(RECOG_H) $(EXPR_H) $(OPTABS_H) toplev.h $(BASIC_BLOCK_H) \
- $(GGC_H) $(TARGET_H) $(TARGET_DEF_H) langhooks.h $(CGRAPH_H) \
- $(TREE_GIMPLE_H) $(DWARF2_H) $(DF_H) tm-constrs.h $(PARAMS_H) \
- i386-builtin-types.inc debug.h dwarf2out.h sbitmap.h $(FIBHEAP_H) \
- $(OPTS_H) $(DIAGNOSTIC_H) $(COMMON_TARGET_H) $(CONTEXT_H) $(PASS_MANAGER_H)
-
-i386-c.o: $(srcdir)/config/i386/i386-c.c \
- $(srcdir)/config/i386/i386-protos.h $(CONFIG_H) $(SYSTEM_H) coretypes.h \
- $(TM_H) $(TREE_H) $(TM_P_H) $(FLAGS_H) $(C_COMMON_H) $(GGC_H) \
- $(TARGET_H) $(TARGET_DEF_H) $(CPPLIB_H) $(C_PRAGMA_H)
- $(COMPILER) -c $(ALL_COMPILERFLAGS) $(ALL_CPPFLAGS) $(INCLUDES) \
- $(srcdir)/config/i386/i386-c.c
+i386-c.o: $(srcdir)/config/i386/i386-c.c i386-builtin-types.inc
+ $(COMPILE) $<
+ $(POSTCOMPILE)
i386-builtin-types.inc: s-i386-bt ; @true
diff --git a/gcc/config/i386/t-rtems b/gcc/config/i386/t-rtems
index 6161ec10090..fef4c22e9c1 100644
--- a/gcc/config/i386/t-rtems
+++ b/gcc/config/i386/t-rtems
@@ -17,11 +17,10 @@
# <http://www.gnu.org/licenses/>.
#
-MULTILIB_OPTIONS = mtune=i486/mtune=pentium/mtune=pentiumpro \
-msoft-float
+MULTILIB_OPTIONS = mtune=i486/mtune=pentium/mtune=pentiumpro msoft-float
MULTILIB_DIRNAMES= m486 mpentium mpentiumpro soft-float
-MULTILIB_MATCHES = msoft-float=mno-m80387
-MULTILIB_MATCHES += mtune?pentium=mtune?k6 mtune?pentiumpro=mtune?mathlon
+MULTILIB_MATCHES = msoft-float=mno-80387
+MULTILIB_MATCHES += mtune?pentium=mtune?k6 mtune?pentiumpro=mtune?athlon
MULTILIB_EXCEPTIONS = \
mtune=pentium/*msoft-float* \
mtune=pentiumpro/*msoft-float*
diff --git a/gcc/config/i386/winnt.c b/gcc/config/i386/winnt.c
index 8cf6b3c527a..94155d89a7f 100644
--- a/gcc/config/i386/winnt.c
+++ b/gcc/config/i386/winnt.c
@@ -35,6 +35,7 @@ along with GCC; see the file COPYING3. If not see
#include "ggc.h"
#include "target.h"
#include "except.h"
+#include "gimple.h"
#include "lto-streamer.h"
/* i386/PE specific attribute support.
@@ -1178,10 +1179,10 @@ i386_pe_seh_unwind_emit (FILE *asm_out_file, rtx insn)
for (note = REG_NOTES (insn); note ; note = XEXP (note, 1))
{
- pat = XEXP (note, 0);
switch (REG_NOTE_KIND (note))
{
case REG_FRAME_RELATED_EXPR:
+ pat = XEXP (note, 0);
goto found;
case REG_CFA_DEF_CFA:
@@ -1195,6 +1196,7 @@ i386_pe_seh_unwind_emit (FILE *asm_out_file, rtx insn)
gcc_unreachable ();
case REG_CFA_ADJUST_CFA:
+ pat = XEXP (note, 0);
if (pat == NULL)
{
pat = PATTERN (insn);
@@ -1206,6 +1208,7 @@ i386_pe_seh_unwind_emit (FILE *asm_out_file, rtx insn)
break;
case REG_CFA_OFFSET:
+ pat = XEXP (note, 0);
if (pat == NULL)
pat = single_set (insn);
seh_cfa_offset (asm_out_file, seh, pat);
diff --git a/gcc/config/i386/x-darwin b/gcc/config/i386/x-darwin
index f0196bac41d..4967d695ce9 100644
--- a/gcc/config/i386/x-darwin
+++ b/gcc/config/i386/x-darwin
@@ -1,4 +1,3 @@
-host-i386-darwin.o : $(srcdir)/config/i386/host-i386-darwin.c \
- $(CONFIG_H) $(SYSTEM_H) coretypes.h hosthooks.h $(HOSTHOOKS_DEF_H) \
- config/host-darwin.h
- $(COMPILER) -c $(ALL_COMPILERFLAGS) $(ALL_CPPFLAGS) $(INCLUDES) $<
+host-i386-darwin.o : $(srcdir)/config/i386/host-i386-darwin.c
+ $(COMPILE) $<
+ $(POSTCOMPILE)
diff --git a/gcc/config/i386/x-i386 b/gcc/config/i386/x-i386
index 2bf8fed5db5..1f3db1d19cf 100644
--- a/gcc/config/i386/x-i386
+++ b/gcc/config/i386/x-i386
@@ -1,4 +1,3 @@
-driver-i386.o : $(srcdir)/config/i386/driver-i386.c \
- $(srcdir)/config/i386/cpuid.h \
- $(CONFIG_H) $(SYSTEM_H) $(TM_H) coretypes.h
- $(COMPILER) -c $(ALL_COMPILERFLAGS) $(ALL_CPPFLAGS) $(INCLUDES) $<
+driver-i386.o : $(srcdir)/config/i386/driver-i386.c
+ $(COMPILE) $<
+ $(POSTCOMPILE)
diff --git a/gcc/config/i386/x86-tune.def b/gcc/config/i386/x86-tune.def
index fc19df19d79..23879f9ec18 100644
--- a/gcc/config/i386/x86-tune.def
+++ b/gcc/config/i386/x86-tune.def
@@ -18,213 +18,488 @@ a copy of the GCC Runtime Library Exception along with this program;
see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
<http://www.gnu.org/licenses/>. */
-/* X86_TUNE_USE_LEAVE: Leave does not affect Nocona SPEC2000 results
- negatively, so enabling for Generic64 seems like good code size
- tradeoff. We can't enable it for 32bit generic because it does not
- work well with PPro base chips. */
-DEF_TUNE (X86_TUNE_USE_LEAVE, "use_leave",
- m_386 | m_CORE_ALL | m_K6_GEODE | m_AMD_MULTIPLE | m_GENERIC)
-DEF_TUNE (X86_TUNE_PUSH_MEMORY, "push_memory",
- m_386 | m_P4_NOCONA | m_CORE_ALL | m_K6_GEODE | m_AMD_MULTIPLE
+/* Tuning for a given CPU XXXX consists of:
+ - adding new CPU into:
+ - adding PROCESSOR_XXX to processor_type (in i386.h)
+ - possibly adding XXX into CPU attribute in i386.md
+ - adding XXX to processor_alias_table (in i386.c)
+ - introducing ix86_XXX_cost in i386.c
+ - Stringop generation table can be build based on test_stringop
+ - script (once rest of tuning is complete)
+ - designing a scheduler model in
+ - XXXX.md file
+ - Updating ix86_issue_rate and ix86_adjust_cost in i386.md
+ - possibly updating ia32_multipass_dfa_lookahead, ix86_sched_reorder
+ and ix86_sched_init_global if those tricks are needed.
+ - Tunning the flags bellow. Those are split into sections and each
+ section is very roughly ordered by importance. */
+
+/*****************************************************************************/
+/* Scheduling flags. */
+/*****************************************************************************/
+
+/* X86_TUNE_SCHEDULE: Enable scheduling. */
+DEF_TUNE (X86_TUNE_SCHEDULE, "schedule",
+ m_PENT | m_PPRO | m_CORE_ALL | m_ATOM | m_SLM | m_K6_GEODE
+ | m_AMD_MULTIPLE | m_GENERIC)
+
+/* X86_TUNE_PARTIAL_REG_DEPENDENCY: Enable more register renaming
+ on modern chips. Preffer stores affecting whole integer register
+ over partial stores. For example preffer MOVZBL or MOVQ to load 8bit
+ value over movb. */
+DEF_TUNE (X86_TUNE_PARTIAL_REG_DEPENDENCY, "partial_reg_dependency",
+ m_P4_NOCONA | m_CORE_ALL | m_ATOM | m_SLM | m_AMD_MULTIPLE
| m_GENERIC)
-DEF_TUNE (X86_TUNE_ZERO_EXTEND_WITH_AND, "zero_extend_with_and", m_486 | m_PENT)
-DEF_TUNE (X86_TUNE_UNROLL_STRLEN, "unroll_strlen",
- m_486 | m_PENT | m_PPRO | m_ATOM | m_SLM | m_CORE_ALL | m_K6
- | m_AMD_MULTIPLE | m_GENERIC)
-/* X86_TUNE_BRANCH_PREDICTION_HINTS: Branch hints were put in P4 based
- on simulation result. But after P4 was made, no performance benefit
- was observed with branch hints. It also increases the code size.
- As a result, icc never generates branch hints. */
-DEF_TUNE (X86_TUNE_BRANCH_PREDICTION_HINTS, "branch_prediction_hints", 0)
-DEF_TUNE (X86_TUNE_DOUBLE_WITH_ADD, "double_with_add", ~m_386)
-DEF_TUNE (X86_TUNE_USE_SAHF, "use_sahf",
- m_PPRO | m_P4_NOCONA | m_CORE_ALL | m_ATOM | m_SLM | m_K6_GEODE
- | m_K8 | m_AMDFAM10 | m_BDVER | m_BTVER | m_GENERIC)
+
+/* X86_TUNE_SSE_PARTIAL_REG_DEPENDENCY: This knob promotes all store
+ destinations to be 128bit to allow register renaming on 128bit SSE units,
+ but usually results in one extra microop on 64bit SSE units.
+ Experimental results shows that disabling this option on P4 brings over 20%
+ SPECfp regression, while enabling it on K8 brings roughly 2.4% regression
+ that can be partly masked by careful scheduling of moves. */
+DEF_TUNE (X86_TUNE_SSE_PARTIAL_REG_DEPENDENCY, "sse_partial_reg_dependency",
+ m_PPRO | m_P4_NOCONA | m_CORE_ALL | m_ATOM | m_SLM | m_AMDFAM10
+ | m_BDVER | m_GENERIC)
+
+/* X86_TUNE_SSE_SPLIT_REGS: Set for machines where the type and dependencies
+ are resolved on SSE register parts instead of whole registers, so we may
+ maintain just lower part of scalar values in proper format leaving the
+ upper part undefined. */
+DEF_TUNE (X86_TUNE_SSE_SPLIT_REGS, "sse_split_regs", m_ATHLON_K8)
+
+/* X86_TUNE_PARTIAL_FLAG_REG_STALL: this flag disables use of of flags
+ set by instructions affecting just some flags (in particular shifts).
+ This is because Core2 resolves dependencies on whole flags register
+ and such sequences introduce false dependency on previous instruction
+ setting full flags.
+
+ The flags does not affect generation of INC and DEC that is controlled
+ by X86_TUNE_USE_INCDEC.
+
+ This flag may be dropped from generic once core2-corei5 machines are
+ rare enough. */
+DEF_TUNE (X86_TUNE_PARTIAL_FLAG_REG_STALL, "partial_flag_reg_stall",
+ m_CORE2 | m_GENERIC)
+
/* X86_TUNE_MOVX: Enable to zero extend integer registers to avoid
partial dependencies. */
DEF_TUNE (X86_TUNE_MOVX, "movx",
m_PPRO | m_P4_NOCONA | m_CORE_ALL | m_ATOM | m_SLM | m_GEODE
| m_AMD_MULTIPLE | m_GENERIC)
-/* X86_TUNE_PARTIAL_REG_STALL: We probably ought to watch for partial
- register stalls on Generic32 compilation setting as well. However
- in current implementation the partial register stalls are not eliminated
- very well - they can be introduced via subregs synthesized by combine
- and can happen in caller/callee saving sequences. */
-DEF_TUNE (X86_TUNE_PARTIAL_REG_STALL, "partial_reg_stall", m_PPRO)
-DEF_TUNE (X86_TUNE_PARTIAL_FLAG_REG_STALL, "partial_flag_reg_stall",
- m_CORE_ALL | m_GENERIC)
-/* X86_TUNE_LCP_STALL: Avoid an expensive length-changing prefix stall
- * on 16-bit immediate moves into memory on Core2 and Corei7. */
-DEF_TUNE (X86_TUNE_LCP_STALL, "lcp_stall", m_CORE_ALL | m_GENERIC)
-DEF_TUNE (X86_TUNE_USE_HIMODE_FIOP, "use_himode_fiop",
- m_386 | m_486 | m_K6_GEODE)
-DEF_TUNE (X86_TUNE_USE_SIMODE_FIOP, "use_simode_fiop",
- ~(m_PENT | m_PPRO | m_CORE_ALL | m_ATOM
- | m_SLM | m_AMD_MULTIPLE | m_GENERIC))
-DEF_TUNE (X86_TUNE_USE_MOV0, "use_mov0", m_K6)
-DEF_TUNE (X86_TUNE_USE_CLTD, "use_cltd", ~(m_PENT | m_ATOM | m_SLM | m_K6))
-/* X86_TUNE_USE_XCHGB: Use xchgb %rh,%rl instead of rolw/rorw $8,rx. */
-DEF_TUNE (X86_TUNE_USE_XCHGB, "use_xchgb", m_PENT4)
-DEF_TUNE (X86_TUNE_SPLIT_LONG_MOVES, "split_long_moves", m_PPRO)
-DEF_TUNE (X86_TUNE_READ_MODIFY_WRITE, "read_modify_write", ~m_PENT)
-DEF_TUNE (X86_TUNE_READ_MODIFY, "read_modify", ~(m_PENT | m_PPRO))
-DEF_TUNE (X86_TUNE_PROMOTE_QIMODE, "promote_qimode",
- m_386 | m_486 | m_PENT | m_CORE_ALL | m_ATOM | m_SLM
- | m_K6_GEODE | m_AMD_MULTIPLE | m_GENERIC)
-DEF_TUNE (X86_TUNE_FAST_PREFIX, "fast_prefix", ~(m_386 | m_486 | m_PENT))
-DEF_TUNE (X86_TUNE_SINGLE_STRINGOP, "single_stringop", m_386 | m_P4_NOCONA)
-DEF_TUNE (X86_TUNE_QIMODE_MATH, "qimode_math", ~0)
-/* X86_TUNE_HIMODE_MATH: On PPro this flag is meant to avoid partial
- register stalls. Just like X86_TUNE_PARTIAL_REG_STALL this option
- might be considered for Generic32 if our scheme for avoiding partial
- stalls was more effective. */
-DEF_TUNE (X86_TUNE_HIMODE_MATH, "himode_math", ~m_PPRO)
-DEF_TUNE (X86_TUNE_PROMOTE_QI_REGS, "promote_qi_regs", 0)
-DEF_TUNE (X86_TUNE_PROMOTE_HI_REGS, "promote_hi_regs", m_PPRO)
-/* X86_TUNE_SINGLE_POP: Enable if single pop insn is preferred
- over esp addition. */
-DEF_TUNE (X86_TUNE_SINGLE_POP, "single_pop", m_386 | m_486 | m_PENT | m_PPRO)
-/* X86_TUNE_DOUBLE_POP: Enable if double pop insn is preferred
- over esp addition. */
-DEF_TUNE (X86_TUNE_DOUBLE_POP, "double_pop", m_PENT)
+
+/* X86_TUNE_MEMORY_MISMATCH_STALL: Avoid partial stores that are followed by
+ full sized loads. */
+DEF_TUNE (X86_TUNE_MEMORY_MISMATCH_STALL, "memory_mismatch_stall",
+ m_P4_NOCONA | m_CORE_ALL | m_ATOM | m_SLM | m_AMD_MULTIPLE | m_GENERIC)
+
+/* X86_TUNE_FUSE_CMP_AND_BRANCH_32: Fuse compare with a subsequent
+ conditional jump instruction for 32 bit TARGET.
+ FIXME: revisit for generic. */
+DEF_TUNE (X86_TUNE_FUSE_CMP_AND_BRANCH_32, "fuse_cmp_and_branch_32",
+ m_CORE_ALL | m_BDVER)
+
+/* X86_TUNE_FUSE_CMP_AND_BRANCH_64: Fuse compare with a subsequent
+ conditional jump instruction for TARGET_64BIT.
+ FIXME: revisit for generic. */
+DEF_TUNE (X86_TUNE_FUSE_CMP_AND_BRANCH_64, "fuse_cmp_and_branch_64",
+ m_COREI7 | m_COREI7_AVX | m_HASWELL | m_BDVER)
+
+/* X86_TUNE_FUSE_CMP_AND_BRANCH_SOFLAGS: Fuse compare with a
+ subsequent conditional jump instruction when the condition jump
+ check sign flag (SF) or overflow flag (OF). */
+DEF_TUNE (X86_TUNE_FUSE_CMP_AND_BRANCH_SOFLAGS, "fuse_cmp_and_branch_soflags",
+ m_COREI7 | m_COREI7_AVX | m_HASWELL | m_BDVER)
+
+/* X86_TUNE_FUSE_ALU_AND_BRANCH: Fuse alu with a subsequent conditional
+ jump instruction when the alu instruction produces the CCFLAG consumed by
+ the conditional jump instruction. */
+DEF_TUNE (X86_TUNE_FUSE_ALU_AND_BRANCH, "fuse_alu_and_branch",
+ m_COREI7_AVX | m_HASWELL)
+
+/* X86_TUNE_REASSOC_INT_TO_PARALLEL: Try to produce parallel computations
+ during reassociation of integer computation. */
+DEF_TUNE (X86_TUNE_REASSOC_INT_TO_PARALLEL, "reassoc_int_to_parallel",
+ m_ATOM)
+
+/* X86_TUNE_REASSOC_FP_TO_PARALLEL: Try to produce parallel computations
+ during reassociation of fp computation. */
+DEF_TUNE (X86_TUNE_REASSOC_FP_TO_PARALLEL, "reassoc_fp_to_parallel",
+ m_ATOM | m_SLM | m_HASWELL | m_BDVER1 | m_BDVER2 | m_GENERIC)
+
+/*****************************************************************************/
+/* Function prologue, epilogue and function calling sequences. */
+/*****************************************************************************/
+
+/* X86_TUNE_ACCUMULATE_OUTGOING_ARGS: Allocate stack space for outgoing
+ arguments in prologue/epilogue instead of separately for each call
+ by push/pop instructions.
+ This increase code size by about 5% in 32bit mode, less so in 64bit mode
+ because parameters are passed in registers. It is considerable
+ win for targets without stack engine that prevents multple push operations
+ to happen in parallel.
+
+ FIXME: the flags is incorrectly enabled for amdfam10, Bulldozer,
+ Bobcat and Generic. This is because disabling it causes large
+ regression on mgrid due to IRA limitation leading to unecessary
+ use of the frame pointer in 32bit mode. */
+DEF_TUNE (X86_TUNE_ACCUMULATE_OUTGOING_ARGS, "accumulate_outgoing_args",
+ m_PPRO | m_P4_NOCONA | m_ATOM | m_SLM | m_AMD_MULTIPLE | m_GENERIC)
+
+/* X86_TUNE_PROLOGUE_USING_MOVE: Do not use push/pop in prologues that are
+ considered on critical path. */
+DEF_TUNE (X86_TUNE_PROLOGUE_USING_MOVE, "prologue_using_move",
+ m_PPRO | m_ATHLON_K8)
+
+/* X86_TUNE_PROLOGUE_USING_MOVE: Do not use push/pop in epilogues that are
+ considered on critical path. */
+DEF_TUNE (X86_TUNE_EPILOGUE_USING_MOVE, "epilogue_using_move",
+ m_PPRO | m_ATHLON_K8)
+
+/* X86_TUNE_USE_LEAVE: Use "leave" instruction in epilogues where it fits. */
+DEF_TUNE (X86_TUNE_USE_LEAVE, "use_leave",
+ m_386 | m_CORE_ALL | m_K6_GEODE | m_AMD_MULTIPLE | m_GENERIC)
+
+/* X86_TUNE_PUSH_MEMORY: Enable generation of "push mem" instructions.
+ Some chips, like 486 and Pentium works faster with separate load
+ and push instructions. */
+DEF_TUNE (X86_TUNE_PUSH_MEMORY, "push_memory",
+ m_386 | m_P4_NOCONA | m_CORE_ALL | m_K6_GEODE | m_AMD_MULTIPLE
+ | m_GENERIC)
+
/* X86_TUNE_SINGLE_PUSH: Enable if single push insn is preferred
over esp subtraction. */
DEF_TUNE (X86_TUNE_SINGLE_PUSH, "single_push", m_386 | m_486 | m_PENT
| m_K6_GEODE)
+
/* X86_TUNE_DOUBLE_PUSH. Enable if double push insn is preferred
over esp subtraction. */
DEF_TUNE (X86_TUNE_DOUBLE_PUSH, "double_push", m_PENT | m_K6_GEODE)
+
+/* X86_TUNE_SINGLE_POP: Enable if single pop insn is preferred
+ over esp addition. */
+DEF_TUNE (X86_TUNE_SINGLE_POP, "single_pop", m_386 | m_486 | m_PENT | m_PPRO)
+
+/* X86_TUNE_DOUBLE_POP: Enable if double pop insn is preferred
+ over esp addition. */
+DEF_TUNE (X86_TUNE_DOUBLE_POP, "double_pop", m_PENT)
+
+/*****************************************************************************/
+/* Branch predictor tuning */
+/*****************************************************************************/
+
+/* X86_TUNE_PAD_SHORT_FUNCTION: Make every function to be at least 4
+ instructions long. */
+DEF_TUNE (X86_TUNE_PAD_SHORT_FUNCTION, "pad_short_function", m_ATOM)
+
+/* X86_TUNE_PAD_RETURNS: Place NOP before every RET that is a destination
+ of conditional jump or directly preceded by other jump instruction.
+ This is important for AND K8-AMDFAM10 because the branch prediction
+ architecture expect at most one jump per 2 byte window. Failing to
+ pad returns leads to misaligned return stack. */
+DEF_TUNE (X86_TUNE_PAD_RETURNS, "pad_returns",
+ m_ATHLON_K8 | m_AMDFAM10 | m_GENERIC)
+
+/* X86_TUNE_FOUR_JUMP_LIMIT: Some CPU cores are not able to predict more
+ than 4 branch instructions in the 16 byte window. */
+DEF_TUNE (X86_TUNE_FOUR_JUMP_LIMIT, "four_jump_limit",
+ m_PPRO | m_P4_NOCONA | m_ATOM | m_SLM | m_ATHLON_K8 | m_AMDFAM10)
+
+/*****************************************************************************/
+/* Integer instruction selection tuning */
+/*****************************************************************************/
+
+/* X86_TUNE_SOFTWARE_PREFETCHING_BENEFICIAL: Enable software prefetching
+ at -O3. For the moment, the prefetching seems badly tuned for Intel
+ chips. */
+DEF_TUNE (X86_TUNE_SOFTWARE_PREFETCHING_BENEFICIAL, "software_prefetching_beneficial",
+ m_K6_GEODE | m_AMD_MULTIPLE)
+
+/* X86_TUNE_LCP_STALL: Avoid an expensive length-changing prefix stall
+ on 16-bit immediate moves into memory on Core2 and Corei7. */
+DEF_TUNE (X86_TUNE_LCP_STALL, "lcp_stall", m_CORE_ALL | m_GENERIC)
+
+/* X86_TUNE_READ_MODIFY: Enable use of read-modify instructions such
+ as "add mem, reg". */
+DEF_TUNE (X86_TUNE_READ_MODIFY, "read_modify", ~(m_PENT | m_PPRO))
+
+/* X86_TUNE_USE_INCDEC: Enable use of inc/dec instructions. */
+DEF_TUNE (X86_TUNE_USE_INCDEC, "use_incdec",
+ ~(m_P4_NOCONA | m_CORE_ALL | m_ATOM | m_SLM | m_GENERIC))
+
/* X86_TUNE_INTEGER_DFMODE_MOVES: Enable if integer moves are preferred
for DFmode copies */
DEF_TUNE (X86_TUNE_INTEGER_DFMODE_MOVES, "integer_dfmode_moves",
~(m_PPRO | m_P4_NOCONA | m_CORE_ALL | m_ATOM | m_SLM
| m_GEODE | m_AMD_MULTIPLE | m_GENERIC))
-DEF_TUNE (X86_TUNE_PARTIAL_REG_DEPENDENCY, "partial_reg_dependency",
- m_P4_NOCONA | m_CORE_ALL | m_ATOM | m_SLM | m_AMD_MULTIPLE
- | m_GENERIC)
-/* X86_TUNE_SSE_PARTIAL_REG_DEPENDENCY: In the Generic model we have a
- conflict here in between PPro/Pentium4 based chips that thread 128bit
- SSE registers as single units versus K8 based chips that divide SSE
- registers to two 64bit halves. This knob promotes all store destinations
- to be 128bit to allow register renaming on 128bit SSE units, but usually
- results in one extra microop on 64bit SSE units. Experimental results
- shows that disabling this option on P4 brings over 20% SPECfp regression,
- while enabling it on K8 brings roughly 2.4% regression that can be partly
- masked by careful scheduling of moves. */
-DEF_TUNE (X86_TUNE_SSE_PARTIAL_REG_DEPENDENCY, "sse_partial_reg_dependency",
- m_PPRO | m_P4_NOCONA | m_CORE_ALL | m_ATOM | m_SLM | m_AMDFAM10
- | m_BDVER | m_GENERIC)
+
+/* X86_TUNE_OPT_AGU: Optimize for Address Generation Unit. This flag
+ will impact LEA instruction selection. */
+DEF_TUNE (X86_TUNE_OPT_AGU, "opt_agu", m_ATOM | m_SLM)
+
+/* X86_TUNE_SLOW_IMUL_IMM32_MEM: Imul of 32-bit constant and memory is
+ vector path on AMD machines.
+ FIXME: Do we need to enable this for core? */
+DEF_TUNE (X86_TUNE_SLOW_IMUL_IMM32_MEM, "slow_imul_imm32_mem",
+ m_K8 | m_AMDFAM10)
+
+/* X86_TUNE_SLOW_IMUL_IMM8: Imul of 8-bit constant is vector path on AMD
+ machines.
+ FIXME: Do we need to enable this for core? */
+DEF_TUNE (X86_TUNE_SLOW_IMUL_IMM8, "slow_imul_imm8",
+ m_K8 | m_AMDFAM10)
+
+/* X86_TUNE_AVOID_MEM_OPND_FOR_CMOVE: Try to avoid memory operands for
+ a conditional move. */
+DEF_TUNE (X86_TUNE_AVOID_MEM_OPND_FOR_CMOVE, "avoid_mem_opnd_for_cmove",
+ m_ATOM | m_SLM)
+
+/* X86_TUNE_SINGLE_STRINGOP: Enable use of single string operations, such
+ as MOVS and STOS (without a REP prefix) to move/set sequences of bytes. */
+DEF_TUNE (X86_TUNE_SINGLE_STRINGOP, "single_stringop", m_386 | m_P4_NOCONA)
+
+/* X86_TUNE_MISALIGNED_MOVE_STRING_PRO_EPILOGUES: Enable generation of
+ compact prologues and epilogues by issuing a misaligned moves. This
+ requires target to handle misaligned moves and partial memory stalls
+ reasonably well.
+ FIXME: This may actualy be a win on more targets than listed here. */
+DEF_TUNE (X86_TUNE_MISALIGNED_MOVE_STRING_PRO_EPILOGUES,
+ "misaligned_move_string_pro_epilogues",
+ m_386 | m_486 | m_CORE_ALL | m_AMD_MULTIPLE | m_GENERIC)
+
+/* X86_TUNE_USE_SAHF: Controls use of SAHF. */
+DEF_TUNE (X86_TUNE_USE_SAHF, "use_sahf",
+ m_PPRO | m_P4_NOCONA | m_CORE_ALL | m_ATOM | m_SLM | m_K6_GEODE
+ | m_K8 | m_AMDFAM10 | m_BDVER | m_BTVER | m_GENERIC)
+
+/* X86_TUNE_USE_CLTD: Controls use of CLTD and CTQO instructions. */
+DEF_TUNE (X86_TUNE_USE_CLTD, "use_cltd", ~(m_PENT | m_ATOM | m_SLM | m_K6))
+
+/* X86_TUNE_USE_BT: Enable use of BT (bit test) instructions. */
+DEF_TUNE (X86_TUNE_USE_BT, "use_bt",
+ m_CORE_ALL | m_ATOM | m_SLM | m_AMD_MULTIPLE | m_GENERIC)
+
+/*****************************************************************************/
+/* 387 instruction selection tuning */
+/*****************************************************************************/
+
+/* X86_TUNE_USE_HIMODE_FIOP: Enables use of x87 instructions with 16bit
+ integer operand.
+ FIXME: Why this is disabled for modern chips? */
+DEF_TUNE (X86_TUNE_USE_HIMODE_FIOP, "use_himode_fiop",
+ m_386 | m_486 | m_K6_GEODE)
+
+/* X86_TUNE_USE_SIMODE_FIOP: Enables use of x87 instructions with 32bit
+ integer operand. */
+DEF_TUNE (X86_TUNE_USE_SIMODE_FIOP, "use_simode_fiop",
+ ~(m_PENT | m_PPRO | m_CORE_ALL | m_ATOM
+ | m_SLM | m_AMD_MULTIPLE | m_GENERIC))
+
+/* X86_TUNE_USE_FFREEP: Use freep instruction instead of fstp. */
+DEF_TUNE (X86_TUNE_USE_FFREEP, "use_ffreep", m_AMD_MULTIPLE)
+
+/* X86_TUNE_EXT_80387_CONSTANTS: Use fancy 80387 constants, such as PI. */
+DEF_TUNE (X86_TUNE_EXT_80387_CONSTANTS, "ext_80387_constants",
+ m_PPRO | m_P4_NOCONA | m_CORE_ALL | m_ATOM | m_SLM | m_K6_GEODE
+ | m_ATHLON_K8 | m_GENERIC)
+
+/*****************************************************************************/
+/* SSE instruction selection tuning */
+/*****************************************************************************/
+
+/* X86_TUNE_VECTORIZE_DOUBLE: Enable double precision vector
+ instructions. */
+DEF_TUNE (X86_TUNE_VECTORIZE_DOUBLE, "vectorize_double", ~m_ATOM)
+
+/* X86_TUNE_GENERAL_REGS_SSE_SPILL: Try to spill general regs to SSE
+ regs instead of memory. */
+DEF_TUNE (X86_TUNE_GENERAL_REGS_SSE_SPILL, "general_regs_sse_spill",
+ m_CORE_ALL)
+
+/* X86_TUNE_SSE_UNALIGNED_LOAD_OPTIMAL: Use movups for misaligned loads instead
+ of a sequence loading registers by parts. */
DEF_TUNE (X86_TUNE_SSE_UNALIGNED_LOAD_OPTIMAL, "sse_unaligned_load_optimal",
- m_COREI7 | m_AMDFAM10 | m_BDVER | m_BTVER | m_SLM)
+ m_COREI7 | m_COREI7_AVX | m_AMDFAM10 | m_BDVER | m_BTVER | m_SLM | m_GENERIC)
+
+/* X86_TUNE_SSE_UNALIGNED_STORE_OPTIMAL: Use movups for misaligned stores instead
+ of a sequence loading registers by parts. */
DEF_TUNE (X86_TUNE_SSE_UNALIGNED_STORE_OPTIMAL, "sse_unaligned_store_optimal",
- m_COREI7 | m_BDVER | m_SLM)
+ m_COREI7 | m_COREI7_AVX | m_BDVER | m_SLM | m_GENERIC)
+
+/* Use packed single precision instructions where posisble. I.e. movups instead
+ of movupd. */
DEF_TUNE (X86_TUNE_SSE_PACKED_SINGLE_INSN_OPTIMAL, "sse_packed_single_insn_optimal",
m_BDVER)
-/* X86_TUNE_SSE_SPLIT_REGS: Set for machines where the type and dependencies
- are resolved on SSE register parts instead of whole registers, so we may
- maintain just lower part of scalar values in proper format leaving the
- upper part undefined. */
-DEF_TUNE (X86_TUNE_SSE_SPLIT_REGS, "sse_split_regs", m_ATHLON_K8)
-DEF_TUNE (X86_TUNE_SSE_TYPELESS_STORES, "sse_typeless_stores", m_AMD_MULTIPLE)
-DEF_TUNE (X86_TUNE_SSE_LOAD0_BY_PXOR, "sse_load0_by_pxor", m_PPRO | m_P4_NOCONA)
-DEF_TUNE (X86_TUNE_MEMORY_MISMATCH_STALL, "memory_mismatch_stall",
- m_P4_NOCONA | m_CORE_ALL | m_ATOM | m_SLM | m_AMD_MULTIPLE | m_GENERIC)
-DEF_TUNE (X86_TUNE_PROLOGUE_USING_MOVE, "prologue_using_move",
- m_PPRO | m_ATHLON_K8)
-DEF_TUNE (X86_TUNE_EPILOGUE_USING_MOVE, "epilogue_using_move",
- m_PPRO | m_ATHLON_K8)
-DEF_TUNE (X86_TUNE_SHIFT1, "shift1", ~m_486)
-DEF_TUNE (X86_TUNE_USE_FFREEP, "use_ffreep", m_AMD_MULTIPLE)
+
+/* X86_TUNE_SSE_TYPELESS_STORES: Always movaps/movups for 128bit stores. */
+DEF_TUNE (X86_TUNE_SSE_TYPELESS_STORES, "sse_typeless_stores",
+ m_AMD_MULTIPLE | m_CORE_ALL | m_GENERIC)
+
+/* X86_TUNE_SSE_LOAD0_BY_PXOR: Always use pxor to load0 as opposed to
+ xorps/xorpd and other variants. */
+DEF_TUNE (X86_TUNE_SSE_LOAD0_BY_PXOR, "sse_load0_by_pxor",
+ m_PPRO | m_P4_NOCONA | m_CORE_ALL | m_BDVER | m_BTVER | m_GENERIC)
+
+/* X86_TUNE_INTER_UNIT_MOVES_TO_VEC: Enable moves in from integer
+ to SSE registers. If disabled, the moves will be done by storing
+ the value to memory and reloading. */
DEF_TUNE (X86_TUNE_INTER_UNIT_MOVES_TO_VEC, "inter_unit_moves_to_vec",
~(m_AMD_MULTIPLE | m_GENERIC))
+
+/* X86_TUNE_INTER_UNIT_MOVES_TO_VEC: Enable moves in from SSE
+ to integer registers. If disabled, the moves will be done by storing
+ the value to memory and reloading. */
DEF_TUNE (X86_TUNE_INTER_UNIT_MOVES_FROM_VEC, "inter_unit_moves_from_vec",
~m_ATHLON_K8)
+
+/* X86_TUNE_INTER_UNIT_CONVERSIONS: Enable float<->integer conversions
+ to use both SSE and integer registers at a same time.
+ FIXME: revisit importance of this for generic. */
DEF_TUNE (X86_TUNE_INTER_UNIT_CONVERSIONS, "inter_unit_conversions",
- ~(m_AMDFAM10 | m_BDVER ))
-/* X86_TUNE_FOUR_JUMP_LIMIT: Some CPU cores are not able to predict more
- than 4 branch instructions in the 16 byte window. */
-DEF_TUNE (X86_TUNE_FOUR_JUMP_LIMIT, "four_jump_limit",
- m_PPRO | m_P4_NOCONA | m_CORE_ALL | m_ATOM | m_SLM| m_AMD_MULTIPLE
- | m_GENERIC)
-DEF_TUNE (X86_TUNE_SCHEDULE, "schedule",
- m_PENT | m_PPRO | m_CORE_ALL | m_ATOM | m_SLM | m_K6_GEODE
- | m_AMD_MULTIPLE | m_GENERIC)
-DEF_TUNE (X86_TUNE_USE_BT, "use_bt",
- m_CORE_ALL | m_ATOM | m_SLM | m_AMD_MULTIPLE | m_GENERIC)
-DEF_TUNE (X86_TUNE_USE_INCDEC, "use_incdec",
- ~(m_P4_NOCONA | m_CORE_ALL | m_ATOM | m_SLM | m_GENERIC))
-DEF_TUNE (X86_TUNE_PAD_RETURNS, "pad_returns",
- m_CORE_ALL | m_AMD_MULTIPLE | m_GENERIC)
-DEF_TUNE (X86_TUNE_PAD_SHORT_FUNCTION, "pad_short_function", m_ATOM)
-DEF_TUNE (X86_TUNE_EXT_80387_CONSTANTS, "ext_80387_constants",
- m_PPRO | m_P4_NOCONA | m_CORE_ALL | m_ATOM | m_SLM | m_K6_GEODE
- | m_ATHLON_K8 | m_GENERIC)
-DEF_TUNE (X86_TUNE_AVOID_VECTOR_DECODE, "avoid_vector_decode",
- m_CORE_ALL | m_K8 | m_GENERIC)
+ ~(m_AMDFAM10 | m_BDVER))
+
+/* X86_TUNE_SPLIT_MEM_OPND_FOR_FP_CONVERTS: Try to split memory operand for
+ fp converts to destination register. */
+DEF_TUNE (X86_TUNE_SPLIT_MEM_OPND_FOR_FP_CONVERTS, "split_mem_opnd_for_fp_converts",
+ m_SLM)
+
+/* X86_TUNE_USE_VECTOR_FP_CONVERTS: Prefer vector packed SSE conversion
+ from FP to FP. This form of instructions avoids partial write to the
+ destination. */
+DEF_TUNE (X86_TUNE_USE_VECTOR_FP_CONVERTS, "use_vector_fp_converts",
+ m_AMDFAM10)
+
+/* X86_TUNE_USE_VECTOR_CONVERTS: Prefer vector packed SSE conversion
+ from integer to FP. */
+DEF_TUNE (X86_TUNE_USE_VECTOR_CONVERTS, "use_vector_converts", m_AMDFAM10)
+
+/*****************************************************************************/
+/* AVX instruction selection tuning (some of SSE flags affects AVX, too) */
+/*****************************************************************************/
+
+/* X86_TUNE_AVX256_UNALIGNED_LOAD_OPTIMAL: if true, unaligned loads are
+ split. */
+DEF_TUNE (X86_TUNE_AVX256_UNALIGNED_LOAD_OPTIMAL, "256_unaligned_load_optimal",
+ ~(m_COREI7 | m_GENERIC))
+
+/* X86_TUNE_AVX256_UNALIGNED_STORE_OPTIMAL: if true, unaligned loads are
+ split. */
+DEF_TUNE (X86_TUNE_AVX256_UNALIGNED_STORE_OPTIMAL, "256_unaligned_load_optimal",
+ ~(m_COREI7 | m_BDVER | m_GENERIC))
+
+/* X86_TUNE_AVX128_OPTIMAL: Enable 128-bit AVX instruction generation for
+ the auto-vectorizer. */
+DEF_TUNE (X86_TUNE_AVX128_OPTIMAL, "avx128_optimal", m_BDVER | m_BTVER2)
+
+/*****************************************************************************/
+/* Historical relics: tuning flags that helps a specific old CPU designs */
+/*****************************************************************************/
+
+/* X86_TUNE_DOUBLE_WITH_ADD: Use add instead of sal to double value in
+ an integer register. */
+DEF_TUNE (X86_TUNE_DOUBLE_WITH_ADD, "double_with_add", ~m_386)
+
+/* X86_TUNE_ALWAYS_FANCY_MATH_387: controls use of fancy 387 operations,
+ such as fsqrt, fprem, fsin, fcos, fsincos etc.
+ Should be enabled for all targets that always has coprocesor. */
+DEF_TUNE (X86_TUNE_ALWAYS_FANCY_MATH_387, "always_fancy_math_387",
+ ~(m_386 | m_486))
+
+/* X86_TUNE_UNROLL_STRLEN: Produce (quite lame) unrolled sequence for
+ inline strlen. This affects only -minline-all-stringops mode. By
+ default we always dispatch to a library since our internal strlen
+ is bad. */
+DEF_TUNE (X86_TUNE_UNROLL_STRLEN, "unroll_strlen", ~m_386)
+
+/* X86_TUNE_SHIFT1: Enables use of short encoding of "sal reg" instead of
+ longer "sal $1, reg". */
+DEF_TUNE (X86_TUNE_SHIFT1, "shift1", ~m_486)
+
+/* X86_TUNE_ZERO_EXTEND_WITH_AND: Use AND instruction instead
+ of mozbl/movwl. */
+DEF_TUNE (X86_TUNE_ZERO_EXTEND_WITH_AND, "zero_extend_with_and", m_486 | m_PENT)
+
/* X86_TUNE_PROMOTE_HIMODE_IMUL: Modern CPUs have same latency for HImode
and SImode multiply, but 386 and 486 do HImode multiply faster. */
DEF_TUNE (X86_TUNE_PROMOTE_HIMODE_IMUL, "promote_himode_imul",
~(m_386 | m_486))
-/* X86_TUNE_SLOW_IMUL_IMM32_MEM: Imul of 32-bit constant and memory is
- vector path on AMD machines. */
-DEF_TUNE (X86_TUNE_SLOW_IMUL_IMM32_MEM, "slow_imul_imm32_mem",
- m_CORE_ALL | m_K8 | m_AMDFAM10 | m_BDVER | m_BTVER | m_GENERIC)
-/* X86_TUNE_SLOW_IMUL_IMM8: Imul of 8-bit constant is vector path on AMD
- machines. */
-DEF_TUNE (X86_TUNE_SLOW_IMUL_IMM8, "slow_imul_imm8",
- m_CORE_ALL | m_K8 | m_AMDFAM10 | m_BDVER | m_BTVER | m_GENERIC)
+
+/* X86_TUNE_FAST_PREFIX: Enable demoting some 32bit or 64bit arithmetic
+ into 16bit/8bit when resulting sequence is shorter. For example
+ for "and $-65536, reg" to 16bit store of 0. */
+DEF_TUNE (X86_TUNE_FAST_PREFIX, "fast_prefix", ~(m_386 | m_486 | m_PENT))
+
+/* X86_TUNE_READ_MODIFY_WRITE: Enable use of read modify write instructions
+ such as "add $1, mem". */
+DEF_TUNE (X86_TUNE_READ_MODIFY_WRITE, "read_modify_write", ~m_PENT)
+
/* X86_TUNE_MOVE_M1_VIA_OR: On pentiums, it is faster to load -1 via OR
than a MOV. */
DEF_TUNE (X86_TUNE_MOVE_M1_VIA_OR, "move_m1_via_or", m_PENT)
+
/* X86_TUNE_NOT_UNPAIRABLE: NOT is not pairable on Pentium, while XOR is,
but one byte longer. */
DEF_TUNE (X86_TUNE_NOT_UNPAIRABLE, "not_unpairable", m_PENT)
+
+/* X86_TUNE_PARTIAL_REG_STALL: Pentium pro, unlike later chips, handled
+ use of partial registers by renaming. This improved performance of 16bit
+ code where upper halves of registers are not used. It also leads to
+ an penalty whenever a 16bit store is followed by 32bit use. This flag
+ disables production of such sequences in common cases.
+ See also X86_TUNE_HIMODE_MATH.
+
+ In current implementation the partial register stalls are not eliminated
+ very well - they can be introduced via subregs synthesized by combine
+ and can happen in caller/callee saving sequences. */
+DEF_TUNE (X86_TUNE_PARTIAL_REG_STALL, "partial_reg_stall", m_PPRO)
+
+/* X86_TUNE_PROMOTE_QIMODE: When it is cheap, turn 8bit arithmetic to
+ corresponding 32bit arithmetic. */
+DEF_TUNE (X86_TUNE_PROMOTE_QIMODE, "promote_qimode",
+ ~m_PPRO)
+
+/* X86_TUNE_PROMOTE_HI_REGS: Same, but for 16bit artihmetic. Again we avoid
+ partial register stalls on PentiumPro targets. */
+DEF_TUNE (X86_TUNE_PROMOTE_HI_REGS, "promote_hi_regs", m_PPRO)
+
+/* X86_TUNE_HIMODE_MATH: Enable use of 16bit arithmetic.
+ On PPro this flag is meant to avoid partial register stalls. */
+DEF_TUNE (X86_TUNE_HIMODE_MATH, "himode_math", ~m_PPRO)
+
+/* X86_TUNE_SPLIT_LONG_MOVES: Avoid instructions moving immediates
+ directly to memory. */
+DEF_TUNE (X86_TUNE_SPLIT_LONG_MOVES, "split_long_moves", m_PPRO)
+
+/* X86_TUNE_USE_XCHGB: Use xchgb %rh,%rl instead of rolw/rorw $8,rx. */
+DEF_TUNE (X86_TUNE_USE_XCHGB, "use_xchgb", m_PENT4)
+
+/* X86_TUNE_USE_MOV0: Use "mov $0, reg" instead of "xor reg, reg" to clear
+ integer register. */
+DEF_TUNE (X86_TUNE_USE_MOV0, "use_mov0", m_K6)
+
/* X86_TUNE_NOT_VECTORMODE: On AMD K6, NOT is vector decoded with memory
operand that cannot be represented using a modRM byte. The XOR
replacement is long decoded, so this split helps here as well. */
DEF_TUNE (X86_TUNE_NOT_VECTORMODE, "not_vectormode", m_K6)
-/* X86_TUNE_USE_VECTOR_FP_CONVERTS: Prefer vector packed SSE conversion
- from FP to FP. */
-DEF_TUNE (X86_TUNE_USE_VECTOR_FP_CONVERTS, "use_vector_fp_converts",
- m_CORE_ALL | m_AMDFAM10 | m_GENERIC)
-/* X86_TUNE_USE_VECTOR_CONVERTS: Prefer vector packed SSE conversion
- from integer to FP. */
-DEF_TUNE (X86_TUNE_USE_VECTOR_CONVERTS, "use_vector_converts", m_AMDFAM10)
-/* X86_TUNE_FUSE_CMP_AND_BRANCH: Fuse a compare or test instruction
- with a subsequent conditional jump instruction into a single
- compare-and-branch uop. */
-DEF_TUNE (X86_TUNE_FUSE_CMP_AND_BRANCH, "fuse_cmp_and_branch", m_BDVER)
-/* X86_TUNE_OPT_AGU: Optimize for Address Generation Unit. This flag
- will impact LEA instruction selection. */
-DEF_TUNE (X86_TUNE_OPT_AGU, "opt_agu", m_ATOM | m_SLM)
-/* X86_TUNE_VECTORIZE_DOUBLE: Enable double precision vector
- instructions. */
-DEF_TUNE (X86_TUNE_VECTORIZE_DOUBLE, "vectorize_double", ~m_ATOM)
-/* X86_TUNE_SOFTWARE_PREFETCHING_BENEFICIAL: Enable software prefetching
- at -O3. For the moment, the prefetching seems badly tuned for Intel
- chips. */
-DEF_TUNE (X86_TUNE_SOFTWARE_PREFETCHING_BENEFICIAL, "software_prefetching_beneficial",
- m_K6_GEODE | m_AMD_MULTIPLE)
-/* X86_TUNE_AVX128_OPTIMAL: Enable 128-bit AVX instruction generation for
- the auto-vectorizer. */
-DEF_TUNE (X86_TUNE_AVX128_OPTIMAL, "avx128_optimal", m_BDVER | m_BTVER2)
-/* X86_TUNE_REASSOC_INT_TO_PARALLEL: Try to produce parallel computations
- during reassociation of integer computation. */
-DEF_TUNE (X86_TUNE_REASSOC_INT_TO_PARALLEL, "reassoc_int_to_parallel",
- m_ATOM)
-/* X86_TUNE_REASSOC_FP_TO_PARALLEL: Try to produce parallel computations
- during reassociation of fp computation. */
-DEF_TUNE (X86_TUNE_REASSOC_FP_TO_PARALLEL, "reassoc_fp_to_parallel",
- m_ATOM | m_SLM | m_HASWELL | m_BDVER1 | m_BDVER2)
-/* X86_TUNE_GENERAL_REGS_SSE_SPILL: Try to spill general regs to SSE
- regs instead of memory. */
-DEF_TUNE (X86_TUNE_GENERAL_REGS_SSE_SPILL, "general_regs_sse_spill",
- m_CORE_ALL)
-/* X86_TUNE_AVOID_MEM_OPND_FOR_CMOVE: Try to avoid memory operands for
- a conditional move. */
-DEF_TUNE (X86_TUNE_AVOID_MEM_OPND_FOR_CMOVE, "avoid_mem_opnd_for_cmove",
- m_ATOM | m_SLM)
-/* X86_TUNE_SPLIT_MEM_OPND_FOR_FP_CONVERTS: Try to split memory operand for
- fp converts to destination register. */
-DEF_TUNE (X86_TUNE_SPLIT_MEM_OPND_FOR_FP_CONVERTS, "split_mem_opnd_for_fp_converts",
- m_SLM)
+
+/* X86_TUNE_AVOID_VECTOR_DECODE: Enable splitters that avoid vector decoded
+ forms of instructions on K8 targets. */
+DEF_TUNE (X86_TUNE_AVOID_VECTOR_DECODE, "avoid_vector_decode",
+ m_K8)
+
+/*****************************************************************************/
+/* This never worked well before. */
+/*****************************************************************************/
+
+/* X86_TUNE_BRANCH_PREDICTION_HINTS: Branch hints were put in P4 based
+ on simulation result. But after P4 was made, no performance benefit
+ was observed with branch hints. It also increases the code size.
+ As a result, icc never generates branch hints. */
+DEF_TUNE (X86_TUNE_BRANCH_PREDICTION_HINTS, "branch_prediction_hints", 0)
+
+/* X86_TUNE_QIMODE_MATH: Enable use of 8bit arithmetic. */
+DEF_TUNE (X86_TUNE_QIMODE_MATH, "qimode_math", ~0)
+
+/* X86_TUNE_PROMOTE_QI_REGS: This enables generic code that promotes all 8bit
+ arithmetic to 32bit via PROMOTE_MODE macro. This code generation scheme
+ is usually used for RISC targets. */
+DEF_TUNE (X86_TUNE_PROMOTE_QI_REGS, "promote_qi_regs", 0)