diff options
Diffstat (limited to 'gcc/tree-vect-stmts.c')
| -rw-r--r-- | gcc/tree-vect-stmts.c | 66 |
1 files changed, 51 insertions, 15 deletions
diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c index 4961ccbccc7..2dbc0353421 100644 --- a/gcc/tree-vect-stmts.c +++ b/gcc/tree-vect-stmts.c @@ -2343,7 +2343,8 @@ vectorizable_shift (gimple stmt, gimple_stmt_iterator *gsi, /* Function vectorizable_operation. - Check if STMT performs a binary or unary operation that can be vectorized. + Check if STMT performs a binary, unary or ternary operation that can + be vectorized. If VEC_STMT is also passed, vectorize the STMT: create a vectorized stmt to replace it, put it in VEC_STMT, and insert it at BSI. Return FALSE if not a vectorizable STMT, TRUE otherwise. */ @@ -2354,7 +2355,7 @@ vectorizable_operation (gimple stmt, gimple_stmt_iterator *gsi, { tree vec_dest; tree scalar_dest; - tree op0, op1 = NULL; + tree op0, op1 = NULL_TREE, op2 = NULL_TREE; stmt_vec_info stmt_info = vinfo_for_stmt (stmt); tree vectype; loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); @@ -2366,7 +2367,8 @@ vectorizable_operation (gimple stmt, gimple_stmt_iterator *gsi, int icode; tree def; gimple def_stmt; - enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type}; + enum vect_def_type dt[3] + = {vect_unknown_def_type, vect_unknown_def_type, vect_unknown_def_type}; gimple new_stmt = NULL; stmt_vec_info prev_stmt_info; int nunits_in; @@ -2374,8 +2376,8 @@ vectorizable_operation (gimple stmt, gimple_stmt_iterator *gsi, tree vectype_out; int ncopies; int j, i; - VEC(tree,heap) *vec_oprnds0 = NULL, *vec_oprnds1 = NULL; - tree vop0, vop1; + VEC(tree,heap) *vec_oprnds0 = NULL, *vec_oprnds1 = NULL, *vec_oprnds2 = NULL; + tree vop0, vop1, vop2; bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info); int vf; @@ -2401,10 +2403,11 @@ vectorizable_operation (gimple stmt, gimple_stmt_iterator *gsi, /* Support only unary or binary operations. */ op_type = TREE_CODE_LENGTH (code); - if (op_type != unary_op && op_type != binary_op) + if (op_type != unary_op && op_type != binary_op && op_type != ternary_op) { if (vect_print_dump_info (REPORT_DETAILS)) - fprintf (vect_dump, "num. args = %d (not unary/binary op).", op_type); + fprintf (vect_dump, "num. args = %d (not unary/binary/ternary op).", + op_type); return false; } @@ -2441,7 +2444,7 @@ vectorizable_operation (gimple stmt, gimple_stmt_iterator *gsi, if (nunits_out != nunits_in) return false; - if (op_type == binary_op) + if (op_type == binary_op || op_type == ternary_op) { op1 = gimple_assign_rhs2 (stmt); if (!vect_is_simple_use (op1, loop_vinfo, bb_vinfo, &def_stmt, &def, @@ -2452,6 +2455,17 @@ vectorizable_operation (gimple stmt, gimple_stmt_iterator *gsi, return false; } } + if (op_type == ternary_op) + { + op2 = gimple_assign_rhs3 (stmt); + if (!vect_is_simple_use (op2, loop_vinfo, bb_vinfo, &def_stmt, &def, + &dt[2])) + { + if (vect_print_dump_info (REPORT_DETAILS)) + fprintf (vect_dump, "use not simple."); + return false; + } + } if (loop_vinfo) vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo); @@ -2473,7 +2487,7 @@ vectorizable_operation (gimple stmt, gimple_stmt_iterator *gsi, || code == RROTATE_EXPR) return false; - optab = optab_for_tree_code (code, vectype, optab_default); + optab = optab_for_tree_code (code, vectype, optab_default); /* Supportable by target? */ if (!optab) @@ -2534,8 +2548,10 @@ vectorizable_operation (gimple stmt, gimple_stmt_iterator *gsi, if (!slp_node) { vec_oprnds0 = VEC_alloc (tree, heap, 1); - if (op_type == binary_op) + if (op_type == binary_op || op_type == ternary_op) vec_oprnds1 = VEC_alloc (tree, heap, 1); + if (op_type == ternary_op) + vec_oprnds2 = VEC_alloc (tree, heap, 1); } /* In case the vectorization factor (VF) is bigger than the number @@ -2597,22 +2613,40 @@ vectorizable_operation (gimple stmt, gimple_stmt_iterator *gsi, /* Handle uses. */ if (j == 0) { - if (op_type == binary_op) + if (op_type == binary_op || op_type == ternary_op) vect_get_vec_defs (op0, op1, stmt, &vec_oprnds0, &vec_oprnds1, slp_node); else vect_get_vec_defs (op0, NULL_TREE, stmt, &vec_oprnds0, NULL, slp_node); + if (op_type == ternary_op) + { + vec_oprnds2 = VEC_alloc (tree, heap, 1); + VEC_quick_push (tree, vec_oprnds2, + vect_get_vec_def_for_operand (op2, stmt, NULL)); + } } else - vect_get_vec_defs_for_stmt_copy (dt, &vec_oprnds0, &vec_oprnds1); + { + vect_get_vec_defs_for_stmt_copy (dt, &vec_oprnds0, &vec_oprnds1); + if (op_type == ternary_op) + { + tree vec_oprnd = VEC_pop (tree, vec_oprnds2); + VEC_quick_push (tree, vec_oprnds2, + vect_get_vec_def_for_stmt_copy (dt[2], + vec_oprnd)); + } + } /* Arguments are ready. Create the new vector stmt. */ FOR_EACH_VEC_ELT (tree, vec_oprnds0, i, vop0) { - vop1 = ((op_type == binary_op) - ? VEC_index (tree, vec_oprnds1, i) : NULL); - new_stmt = gimple_build_assign_with_ops (code, vec_dest, vop0, vop1); + vop1 = ((op_type == binary_op || op_type == ternary_op) + ? VEC_index (tree, vec_oprnds1, i) : NULL_TREE); + vop2 = ((op_type == ternary_op) + ? VEC_index (tree, vec_oprnds2, i) : NULL_TREE); + new_stmt = gimple_build_assign_with_ops3 (code, vec_dest, + vop0, vop1, vop2); new_temp = make_ssa_name (vec_dest, new_stmt); gimple_assign_set_lhs (new_stmt, new_temp); vect_finish_stmt_generation (stmt, new_stmt, gsi); @@ -2633,6 +2667,8 @@ vectorizable_operation (gimple stmt, gimple_stmt_iterator *gsi, VEC_free (tree, heap, vec_oprnds0); if (vec_oprnds1) VEC_free (tree, heap, vec_oprnds1); + if (vec_oprnds2) + VEC_free (tree, heap, vec_oprnds2); return true; } |