diff options
Diffstat (limited to 'gcc')
| -rw-r--r-- | gcc/ChangeLog | 21 | ||||
| -rw-r--r-- | gcc/testsuite/ChangeLog | 4 | ||||
| -rw-r--r-- | gcc/testsuite/gcc.dg/vect/no-vfa-vect-depend-1.c | 2 | ||||
| -rw-r--r-- | gcc/tree-vect-data-refs.c | 148 | ||||
| -rw-r--r-- | gcc/tree-vect-loop.c | 35 | ||||
| -rw-r--r-- | gcc/tree-vect-slp.c | 25 | ||||
| -rw-r--r-- | gcc/tree-vectorizer.h | 8 |
7 files changed, 190 insertions, 53 deletions
diff --git a/gcc/ChangeLog b/gcc/ChangeLog index 09b704aaf48..32b9987565d 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,24 @@ +2010-04-13 Richard Guenther <rguenther@suse.de> + + * tree-vect-data-refs.c (vect_analyze_data_ref_dependence): + Only add RW dependence for dependence distance zero. + Adjust maximal vectorization factor according to dependences. + Move alignment handling ... + (vect_find_same_alignment_drs): ... here. New function. + (vect_analyze_data_ref_dependences): Adjust. + (vect_analyze_data_refs_alignment): Call vect_find_same_alignment_drs. + (vect_analyze_data_refs): Adjust minimal vectorization factor + according to data references. + * tree-vect-loop.c (vect_analyze_loop): Analyze data-ref + dependences before determining the vectorization factor. + Analyze alignment after determining the vectorization factor. + * tree-vect-slp.c ((vect_slp_analyze_bb): Analyze data-ref + dependences before alignment. + * tree-vectorizer.h (vect_analyze_data_ref_dependences): + Adjust prototype. + (vect_analyze_data_refs): Likewise. + (MAX_VECTORIZATION_FACTOR): New define. + 2010-04-13 Duncan Sands <baldrick@free.fr> * except.h (lang_eh_type_covers): Remove. diff --git a/gcc/testsuite/ChangeLog b/gcc/testsuite/ChangeLog index dcce334ca4d..55f5e635821 100644 --- a/gcc/testsuite/ChangeLog +++ b/gcc/testsuite/ChangeLog @@ -1,3 +1,7 @@ +2010-04-13 Richard Guenther <rguenther@suse.de> + + * gcc.dg/vect/no-vfa-vect-depend-1.c: Adjust. + 2010-04-13 Eric Botcazou <ebotcazou@adacore.com> * gnat.dg/derived_type1.adb: New test. diff --git a/gcc/testsuite/gcc.dg/vect/no-vfa-vect-depend-1.c b/gcc/testsuite/gcc.dg/vect/no-vfa-vect-depend-1.c index 329478d39fe..5679ff765c1 100644 --- a/gcc/testsuite/gcc.dg/vect/no-vfa-vect-depend-1.c +++ b/gcc/testsuite/gcc.dg/vect/no-vfa-vect-depend-1.c @@ -51,6 +51,6 @@ int main (void) } /* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" {xfail vect_no_align } } } */ -/* { dg-final { scan-tree-dump-times "dependence distance >= VF or negative" 1 "vect" } } */ +/* { dg-final { scan-tree-dump-times "dependence distance negative" 1 "vect" } } */ /* { dg-final { cleanup-tree-dump "vect" } } */ diff --git a/gcc/tree-vect-data-refs.c b/gcc/tree-vect-data-refs.c index a4d6a5f9bc4..ba537a062ef 100644 --- a/gcc/tree-vect-data-refs.c +++ b/gcc/tree-vect-data-refs.c @@ -487,21 +487,19 @@ vect_mark_for_runtime_alias_test (ddr_p ddr, loop_vec_info loop_vinfo) Return TRUE if there (might) exist a dependence between a memory-reference DRA and a memory-reference DRB. When versioning for alias may check a - dependence at run-time, return FALSE. */ + dependence at run-time, return FALSE. Adjust *MAX_VF according to + the data dependence. */ static bool vect_analyze_data_ref_dependence (struct data_dependence_relation *ddr, - loop_vec_info loop_vinfo) + loop_vec_info loop_vinfo, int *max_vf) { unsigned int i; struct loop *loop = NULL; - int vectorization_factor = 0; struct data_reference *dra = DDR_A (ddr); struct data_reference *drb = DDR_B (ddr); stmt_vec_info stmtinfo_a = vinfo_for_stmt (DR_STMT (dra)); stmt_vec_info stmtinfo_b = vinfo_for_stmt (DR_STMT (drb)); - int dra_size = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (DR_REF (dra)))); - int drb_size = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (DR_REF (drb)))); lambda_vector dist_v; unsigned int loop_depth; @@ -513,10 +511,7 @@ vect_analyze_data_ref_dependence (struct data_dependence_relation *ddr, } if (loop_vinfo) - { - loop = LOOP_VINFO_LOOP (loop_vinfo); - vectorization_factor = LOOP_VINFO_VECT_FACTOR (loop_vinfo); - } + loop = LOOP_VINFO_LOOP (loop_vinfo); if ((DR_IS_READ (dra) && DR_IS_READ (drb) && loop_vinfo) || dra == drb) return false; @@ -595,17 +590,11 @@ vect_analyze_data_ref_dependence (struct data_dependence_relation *ddr, if (vect_print_dump_info (REPORT_DR_DETAILS)) fprintf (vect_dump, "dependence distance = %d.", dist); - /* Same loop iteration. */ - if (dist % vectorization_factor == 0 && dra_size == drb_size) + if (dist == 0) { - /* Two references with distance zero have the same alignment. */ - VEC_safe_push (dr_p, heap, STMT_VINFO_SAME_ALIGN_REFS (stmtinfo_a), drb); - VEC_safe_push (dr_p, heap, STMT_VINFO_SAME_ALIGN_REFS (stmtinfo_b), dra); - if (vect_print_dump_info (REPORT_ALIGNMENT)) - fprintf (vect_dump, "accesses have the same alignment."); if (vect_print_dump_info (REPORT_DR_DETAILS)) { - fprintf (vect_dump, "dependence distance modulo vf == 0 between "); + fprintf (vect_dump, "dependence distance == 0 between "); print_generic_expr (vect_dump, DR_REF (dra), TDF_SLIM); fprintf (vect_dump, " and "); print_generic_expr (vect_dump, DR_REF (drb), TDF_SLIM); @@ -621,18 +610,36 @@ vect_analyze_data_ref_dependence (struct data_dependence_relation *ddr, DR_GROUP_READ_WRITE_DEPENDENCE (stmtinfo_b) = true; } - continue; + continue; + } + + if (dist > 0 && DDR_REVERSED_P (ddr)) + { + /* If DDR_REVERSED_P the order of the data-refs in DDR was + reversed (to make distance vector positive), and the actual + distance is negative. */ + if (vect_print_dump_info (REPORT_DR_DETAILS)) + fprintf (vect_dump, "dependence distance negative."); + continue; + } + + if (abs (dist) >= 2 + && abs (dist) < *max_vf) + { + /* The dependence distance requires reduction of the maximal + vectorization factor. */ + *max_vf = abs (dist); + if (vect_print_dump_info (REPORT_DR_DETAILS)) + fprintf (vect_dump, "adjusting maximal vectorization factor to %i", + *max_vf); } - if (abs (dist) >= vectorization_factor - || (dist > 0 && DDR_REVERSED_P (ddr))) + if (abs (dist) >= *max_vf) { /* Dependence distance does not create dependence, as far as - vectorization is concerned, in this case. If DDR_REVERSED_P the - order of the data-refs in DDR was reversed (to make distance - vector positive), and the actual distance is negative. */ + vectorization is concerned, in this case. */ if (vect_print_dump_info (REPORT_DR_DETAILS)) - fprintf (vect_dump, "dependence distance >= VF or negative."); + fprintf (vect_dump, "dependence distance >= VF."); continue; } @@ -654,11 +661,12 @@ vect_analyze_data_ref_dependence (struct data_dependence_relation *ddr, /* Function vect_analyze_data_ref_dependences. Examine all the data references in the loop, and make sure there do not - exist any data dependences between them. */ + exist any data dependences between them. Set *MAX_VF according to + the maximum vectorization factor the data dependences allow. */ bool vect_analyze_data_ref_dependences (loop_vec_info loop_vinfo, - bb_vec_info bb_vinfo) + bb_vec_info bb_vinfo, int *max_vf) { unsigned int i; VEC (ddr_p, heap) *ddrs = NULL; @@ -673,7 +681,7 @@ vect_analyze_data_ref_dependences (loop_vec_info loop_vinfo, ddrs = BB_VINFO_DDRS (bb_vinfo); for (i = 0; VEC_iterate (ddr_p, ddrs, i, ddr); i++) - if (vect_analyze_data_ref_dependence (ddr, loop_vinfo)) + if (vect_analyze_data_ref_dependence (ddr, loop_vinfo, max_vf)) return false; return true; @@ -1410,6 +1418,69 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo) } +/* Function vect_find_same_alignment_drs. + + Update group and alignment relations according to the chosen + vectorization factor. */ + +static void +vect_find_same_alignment_drs (struct data_dependence_relation *ddr, + loop_vec_info loop_vinfo) +{ + unsigned int i; + struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo); + int vectorization_factor = LOOP_VINFO_VECT_FACTOR (loop_vinfo); + struct data_reference *dra = DDR_A (ddr); + struct data_reference *drb = DDR_B (ddr); + stmt_vec_info stmtinfo_a = vinfo_for_stmt (DR_STMT (dra)); + stmt_vec_info stmtinfo_b = vinfo_for_stmt (DR_STMT (drb)); + int dra_size = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (DR_REF (dra)))); + int drb_size = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (DR_REF (drb)))); + lambda_vector dist_v; + unsigned int loop_depth; + + if (DDR_ARE_DEPENDENT (ddr) == chrec_known) + return; + + if ((DR_IS_READ (dra) && DR_IS_READ (drb)) || dra == drb) + return; + + if (DDR_ARE_DEPENDENT (ddr) == chrec_dont_know) + return; + + /* Loop-based vectorization and known data dependence. */ + if (DDR_NUM_DIST_VECTS (ddr) == 0) + return; + + loop_depth = index_in_loop_nest (loop->num, DDR_LOOP_NEST (ddr)); + for (i = 0; VEC_iterate (lambda_vector, DDR_DIST_VECTS (ddr), i, dist_v); i++) + { + int dist = dist_v[loop_depth]; + + if (vect_print_dump_info (REPORT_DR_DETAILS)) + fprintf (vect_dump, "dependence distance = %d.", dist); + + /* Same loop iteration. */ + if (dist == 0 + || (dist % vectorization_factor == 0 && dra_size == drb_size)) + { + /* Two references with distance zero have the same alignment. */ + VEC_safe_push (dr_p, heap, STMT_VINFO_SAME_ALIGN_REFS (stmtinfo_a), drb); + VEC_safe_push (dr_p, heap, STMT_VINFO_SAME_ALIGN_REFS (stmtinfo_b), dra); + if (vect_print_dump_info (REPORT_ALIGNMENT)) + fprintf (vect_dump, "accesses have the same alignment."); + if (vect_print_dump_info (REPORT_DR_DETAILS)) + { + fprintf (vect_dump, "dependence distance modulo vf == 0 between "); + print_generic_expr (vect_dump, DR_REF (dra), TDF_SLIM); + fprintf (vect_dump, " and "); + print_generic_expr (vect_dump, DR_REF (drb), TDF_SLIM); + } + } + } +} + + /* Function vect_analyze_data_refs_alignment Analyze the alignment of the data-references in the loop. @@ -1422,6 +1493,18 @@ vect_analyze_data_refs_alignment (loop_vec_info loop_vinfo, if (vect_print_dump_info (REPORT_DETAILS)) fprintf (vect_dump, "=== vect_analyze_data_refs_alignment ==="); + /* Mark groups of data references with same alignment using + data dependence information. */ + if (loop_vinfo) + { + VEC (ddr_p, heap) *ddrs = LOOP_VINFO_DDRS (loop_vinfo); + struct data_dependence_relation *ddr; + unsigned int i; + + for (i = 0; VEC_iterate (ddr_p, ddrs, i, ddr); i++) + vect_find_same_alignment_drs (ddr, loop_vinfo); + } + if (!vect_compute_data_refs_alignment (loop_vinfo, bb_vinfo)) { if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS)) @@ -1852,7 +1935,9 @@ vect_prune_runtime_alias_test_list (loop_vec_info loop_vinfo) */ bool -vect_analyze_data_refs (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo) +vect_analyze_data_refs (loop_vec_info loop_vinfo, + bb_vec_info bb_vinfo, + int *min_vf) { struct loop *loop = NULL; basic_block bb = NULL; @@ -1907,6 +1992,7 @@ vect_analyze_data_refs (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo) gimple stmt; stmt_vec_info stmt_info; tree base, offset, init; + int vf; if (!dr || !DR_REF (dr)) { @@ -2079,6 +2165,12 @@ vect_analyze_data_refs (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo) } return false; } + + /* Adjust the minimal vectorization factor according to the + vector type. */ + vf = TYPE_VECTOR_SUBPARTS (STMT_VINFO_VECTYPE (stmt_info)); + if (vf > *min_vf) + *min_vf = vf; } return true; diff --git a/gcc/tree-vect-loop.c b/gcc/tree-vect-loop.c index c6547957341..809f3e15a02 100644 --- a/gcc/tree-vect-loop.c +++ b/gcc/tree-vect-loop.c @@ -1354,6 +1354,8 @@ vect_analyze_loop (struct loop *loop) { bool ok; loop_vec_info loop_vinfo; + int max_vf = MAX_VECTORIZATION_FACTOR; + int min_vf = 2; if (vect_print_dump_info (REPORT_DETAILS)) fprintf (vect_dump, "===== analyze_loop_nest ====="); @@ -1378,12 +1380,13 @@ vect_analyze_loop (struct loop *loop) } /* Find all data references in the loop (which correspond to vdefs/vuses) - and analyze their evolution in the loop. + and analyze their evolution in the loop. Also adjust the minimal + vectorization factor according to the loads and stores. FORNOW: Handle only simple, array references, which alignment can be forced, and aligned pointer-references. */ - ok = vect_analyze_data_refs (loop_vinfo, NULL); + ok = vect_analyze_data_refs (loop_vinfo, NULL, &min_vf); if (!ok) { if (vect_print_dump_info (REPORT_DETAILS)) @@ -1410,14 +1413,17 @@ vect_analyze_loop (struct loop *loop) return NULL; } - /* Analyze the alignment of the data-refs in the loop. - Fail if a data reference is found that cannot be vectorized. */ + /* Analyze data dependences between the data-refs in the loop + and adjust the maximum vectorization factor according to + the dependences. + FORNOW: fail at the first data dependence that we encounter. */ - ok = vect_analyze_data_refs_alignment (loop_vinfo, NULL); - if (!ok) + ok = vect_analyze_data_ref_dependences (loop_vinfo, NULL, &max_vf); + if (!ok + || max_vf < min_vf) { if (vect_print_dump_info (REPORT_DETAILS)) - fprintf (vect_dump, "bad data alignment."); + fprintf (vect_dump, "bad data dependence."); destroy_loop_vec_info (loop_vinfo, true); return NULL; } @@ -1430,15 +1436,22 @@ vect_analyze_loop (struct loop *loop) destroy_loop_vec_info (loop_vinfo, true); return NULL; } + if (max_vf < LOOP_VINFO_VECT_FACTOR (loop_vinfo)) + { + if (vect_print_dump_info (REPORT_DETAILS)) + fprintf (vect_dump, "bad data dependence."); + destroy_loop_vec_info (loop_vinfo, true); + return NULL; + } - /* Analyze data dependences between the data-refs in the loop. - FORNOW: fail at the first data dependence that we encounter. */ + /* Analyze the alignment of the data-refs in the loop. + Fail if a data reference is found that cannot be vectorized. */ - ok = vect_analyze_data_ref_dependences (loop_vinfo, NULL); + ok = vect_analyze_data_refs_alignment (loop_vinfo, NULL); if (!ok) { if (vect_print_dump_info (REPORT_DETAILS)) - fprintf (vect_dump, "bad data dependence."); + fprintf (vect_dump, "bad data alignment."); destroy_loop_vec_info (loop_vinfo, true); return NULL; } diff --git a/gcc/tree-vect-slp.c b/gcc/tree-vect-slp.c index 0e4f6ce6151..d25d34787e0 100644 --- a/gcc/tree-vect-slp.c +++ b/gcc/tree-vect-slp.c @@ -1270,6 +1270,8 @@ vect_slp_analyze_bb (basic_block bb) slp_instance instance; int i, insns = 0; gimple_stmt_iterator gsi; + int min_vf = 2; + int max_vf = MAX_VECTORIZATION_FACTOR; if (vect_print_dump_info (REPORT_DETAILS)) fprintf (vect_dump, "===vect_slp_analyze_bb===\n"); @@ -1296,7 +1298,7 @@ vect_slp_analyze_bb (basic_block bb) if (!bb_vinfo) return NULL; - if (!vect_analyze_data_refs (NULL, bb_vinfo)) + if (!vect_analyze_data_refs (NULL, bb_vinfo, &min_vf)) { if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS)) fprintf (vect_dump, "not vectorized: unhandled data-ref in basic " @@ -1317,6 +1319,17 @@ vect_slp_analyze_bb (basic_block bb) return NULL; } + if (!vect_analyze_data_ref_dependences (NULL, bb_vinfo, &max_vf) + || min_vf > max_vf) + { + if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS)) + fprintf (vect_dump, "not vectorized: unhandled data dependence " + "in basic block.\n"); + + destroy_bb_vec_info (bb_vinfo); + return NULL; + } + if (!vect_analyze_data_refs_alignment (NULL, bb_vinfo)) { if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS)) @@ -1327,16 +1340,6 @@ vect_slp_analyze_bb (basic_block bb) return NULL; } - if (!vect_analyze_data_ref_dependences (NULL, bb_vinfo)) - { - if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS)) - fprintf (vect_dump, "not vectorized: unhandled data dependence in basic" - " block.\n"); - - destroy_bb_vec_info (bb_vinfo); - return NULL; - } - if (!vect_analyze_data_ref_accesses (NULL, bb_vinfo)) { if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS)) diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h index b46bc52c25b..52b2a7ec59f 100644 --- a/gcc/tree-vectorizer.h +++ b/gcc/tree-vectorizer.h @@ -601,6 +601,9 @@ typedef struct _stmt_vec_info { conversion. */ #define MAX_INTERM_CVT_STEPS 3 +/* The maximum vectorization factor supported by any target (V32QI). */ +#define MAX_VECTORIZATION_FACTOR 32 + /* Avoid GTY(()) on stmt_vec_info. */ typedef void *vec_void_p; DEF_VEC_P (vec_void_p); @@ -802,13 +805,14 @@ extern enum dr_alignment_support vect_supportable_dr_alignment (struct data_reference *); extern tree vect_get_smallest_scalar_type (gimple, HOST_WIDE_INT *, HOST_WIDE_INT *); -extern bool vect_analyze_data_ref_dependences (loop_vec_info, bb_vec_info); +extern bool vect_analyze_data_ref_dependences (loop_vec_info, bb_vec_info, + int *); extern bool vect_enhance_data_refs_alignment (loop_vec_info); extern bool vect_analyze_data_refs_alignment (loop_vec_info, bb_vec_info); extern bool vect_verify_datarefs_alignment (loop_vec_info, bb_vec_info); extern bool vect_analyze_data_ref_accesses (loop_vec_info, bb_vec_info); extern bool vect_prune_runtime_alias_test_list (loop_vec_info); -extern bool vect_analyze_data_refs (loop_vec_info, bb_vec_info); +extern bool vect_analyze_data_refs (loop_vec_info, bb_vec_info, int *); extern tree vect_create_data_ref_ptr (gimple, struct loop *, tree, tree *, gimple *, bool, bool *); extern tree bump_vector_ptr (tree, gimple, gimple_stmt_iterator *, gimple, tree); |