diff options
| -rw-r--r-- | gcc/ChangeLog | 13 | ||||
| -rw-r--r-- | gcc/cfgloop.h | 13 | ||||
| -rw-r--r-- | gcc/cfgloopmanip.c | 3 | ||||
| -rw-r--r-- | gcc/internal-fn.c | 8 | ||||
| -rw-r--r-- | gcc/internal-fn.def | 1 | ||||
| -rw-r--r-- | gcc/tree-vectorizer.c | 89 |
6 files changed, 116 insertions, 11 deletions
diff --git a/gcc/ChangeLog b/gcc/ChangeLog index 1f0b0b43999..fa8c40cacdd 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,16 @@ +2017-07-05 Bin Cheng <bin.cheng@arm.com> + + * cfgloop.h (struct loop): Add comment. New field orig_loop_num. + * cfgloopmanip.c (lv_adjust_loop_entry_edge): Comment change. + * internal-fn.c (expand_LOOP_DIST_ALIAS): New function. + * internal-fn.def (LOOP_DIST_ALIAS): New. + * tree-vectorizer.c (fold_loop_vectorized_call): Rename to ... + (fold_loop_internal_call): ... this. + (vect_loop_dist_alias_call): New function. + (set_uid_loop_bbs): Call fold_loop_internal_call. + (vectorize_loops): Fold IFN_LOOP_VECTORIZED and IFN_LOOP_DIST_ALIAS + internal calls. + 2017-07-04 Uros Bizjak <ubizjak@gmail.com> PR target/81300 diff --git a/gcc/cfgloop.h b/gcc/cfgloop.h index a8bec1d48af..e7ffa236437 100644 --- a/gcc/cfgloop.h +++ b/gcc/cfgloop.h @@ -114,7 +114,8 @@ struct GTY ((chain_next ("%h.next"))) control_iv { /* Structure to hold information for each natural loop. */ struct GTY ((chain_next ("%h.next"))) loop { - /* Index into loops array. */ + /* Index into loops array. Note indices will never be reused after loop + is destroyed. */ int num; /* Number of loop insns. */ @@ -225,6 +226,16 @@ struct GTY ((chain_next ("%h.next"))) loop { builtins. */ tree simduid; + /* In loop optimization, it's common to generate loops from the original + loop. This field records the index of the original loop which can be + used to track the original loop from newly generated loops. This can + be done by calling function get_loop (cfun, orig_loop_num). Note the + original loop could be destroyed for various reasons thus no longer + exists, as a result, function call to get_loop returns NULL pointer. + In this case, this field should not be used and needs to be cleared + whenever possible. */ + int orig_loop_num; + /* Upper bound on number of iterations of a loop. */ struct nb_iter_bound *bounds; diff --git a/gcc/cfgloopmanip.c b/gcc/cfgloopmanip.c index a7d0e612eb2..3f4ff988c74 100644 --- a/gcc/cfgloopmanip.c +++ b/gcc/cfgloopmanip.c @@ -1623,7 +1623,8 @@ force_single_succ_latches (void) THEN_PROB is the probability of then branch of the condition. ELSE_PROB is the probability of else branch. Note that they may be both - REG_BR_PROB_BASE when condition is IFN_LOOP_VECTORIZED. */ + REG_BR_PROB_BASE when condition is IFN_LOOP_VECTORIZED or + IFN_LOOP_DIST_ALIAS. */ static basic_block lv_adjust_loop_entry_edge (basic_block first_head, basic_block second_head, diff --git a/gcc/internal-fn.c b/gcc/internal-fn.c index d276dfecf9a..18466cd53b3 100644 --- a/gcc/internal-fn.c +++ b/gcc/internal-fn.c @@ -2250,6 +2250,14 @@ expand_LOOP_VECTORIZED (internal_fn, gcall *) gcc_unreachable (); } +/* This should get folded in tree-vectorizer.c. */ + +static void +expand_LOOP_DIST_ALIAS (internal_fn, gcall *) +{ + gcc_unreachable (); +} + /* Expand MASK_LOAD call STMT using optab OPTAB. */ static void diff --git a/gcc/internal-fn.def b/gcc/internal-fn.def index e162d81121c..79c19fb8e7a 100644 --- a/gcc/internal-fn.def +++ b/gcc/internal-fn.def @@ -158,6 +158,7 @@ DEF_INTERNAL_FN (GOMP_SIMD_LAST_LANE, ECF_CONST | ECF_LEAF | ECF_NOTHROW, NULL) DEF_INTERNAL_FN (GOMP_SIMD_ORDERED_START, ECF_LEAF | ECF_NOTHROW, NULL) DEF_INTERNAL_FN (GOMP_SIMD_ORDERED_END, ECF_LEAF | ECF_NOTHROW, NULL) DEF_INTERNAL_FN (LOOP_VECTORIZED, ECF_NOVOPS | ECF_LEAF | ECF_NOTHROW, NULL) +DEF_INTERNAL_FN (LOOP_DIST_ALIAS, ECF_NOVOPS | ECF_LEAF | ECF_NOTHROW, NULL) DEF_INTERNAL_FN (ANNOTATE, ECF_CONST | ECF_LEAF | ECF_NOTHROW, NULL) DEF_INTERNAL_FN (UBSAN_NULL, ECF_LEAF | ECF_NOTHROW, ".R.") DEF_INTERNAL_FN (UBSAN_BOUNDS, ECF_LEAF | ECF_NOTHROW, NULL) diff --git a/gcc/tree-vectorizer.c b/gcc/tree-vectorizer.c index 0d62c829ef5..33a1f580474 100644 --- a/gcc/tree-vectorizer.c +++ b/gcc/tree-vectorizer.c @@ -448,11 +448,11 @@ vect_loop_vectorized_call (struct loop *loop) return NULL; } -/* Fold LOOP_VECTORIZED internal call G to VALUE and - update any immediate uses of it's LHS. */ +/* Fold loop internal call G like IFN_LOOP_VECTORIZED/IFN_LOOP_DIST_ALIAS + to VALUE and update any immediate uses of it's LHS. */ static void -fold_loop_vectorized_call (gimple *g, tree value) +fold_loop_internal_call (gimple *g, tree value) { tree lhs = gimple_call_lhs (g); use_operand_p use_p; @@ -469,6 +469,60 @@ fold_loop_vectorized_call (gimple *g, tree value) } } +/* If LOOP has been versioned during loop distribution, return the gurading + internal call. */ + +static gimple * +vect_loop_dist_alias_call (struct loop *loop) +{ + basic_block bb; + basic_block entry; + struct loop *outer, *orig; + gimple_stmt_iterator gsi; + gimple *g; + + if (loop->orig_loop_num == 0) + return NULL; + + orig = get_loop (cfun, loop->orig_loop_num); + if (orig == NULL) + { + /* The original loop is somehow destroyed. Clear the information. */ + loop->orig_loop_num = 0; + return NULL; + } + + if (loop != orig) + bb = nearest_common_dominator (CDI_DOMINATORS, loop->header, orig->header); + else + bb = loop_preheader_edge (loop)->src; + + outer = bb->loop_father; + entry = ENTRY_BLOCK_PTR_FOR_FN (cfun); + + /* Look upward in dominance tree. */ + for (; bb != entry && flow_bb_inside_loop_p (outer, bb); + bb = get_immediate_dominator (CDI_DOMINATORS, bb)) + { + g = last_stmt (bb); + if (g == NULL || gimple_code (g) != GIMPLE_COND) + continue; + + gsi = gsi_for_stmt (g); + gsi_prev (&gsi); + if (gsi_end_p (gsi)) + continue; + + g = gsi_stmt (gsi); + /* The guarding internal function call must have the same distribution + alias id. */ + if (gimple_call_internal_p (g, IFN_LOOP_DIST_ALIAS) + && (tree_to_shwi (gimple_call_arg (g, 0)) == loop->orig_loop_num)) + return g; + } + return NULL; +} + /* Set the uids of all the statements in basic blocks inside loop represented by LOOP_VINFO. LOOP_VECTORIZED_CALL is the internal call guarding the loop which has been if converted. */ @@ -494,7 +548,7 @@ set_uid_loop_bbs (loop_vec_info loop_vinfo, gimple *loop_vectorized_call) { arg = gimple_call_arg (g, 0); get_loop (cfun, tree_to_shwi (arg))->dont_vectorize = true; - fold_loop_vectorized_call (g, boolean_false_node); + fold_loop_internal_call (g, boolean_false_node); } } bbs = get_loop_body (scalar_loop); @@ -595,7 +649,7 @@ vectorize_loops (void) else { loop_vec_info loop_vinfo, orig_loop_vinfo; - gimple *loop_vectorized_call; + gimple *loop_vectorized_call, *loop_dist_alias_call; try_vectorize: if (!((flag_tree_loop_vectorize && optimize_loop_nest_for_speed_p (loop)) @@ -603,6 +657,7 @@ vectorize_loops (void) continue; orig_loop_vinfo = NULL; loop_vectorized_call = vect_loop_vectorized_call (loop); + loop_dist_alias_call = vect_loop_dist_alias_call (loop); vectorize_epilogue: vect_location = find_loop_location (loop); if (LOCATION_LOCUS (vect_location) != UNKNOWN_LOCATION @@ -652,8 +707,8 @@ vectorize_loops (void) { dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, vect_location, "basic block vectorized\n"); - fold_loop_vectorized_call (loop_vectorized_call, - boolean_true_node); + fold_loop_internal_call (loop_vectorized_call, + boolean_true_node); loop_vectorized_call = NULL; ret |= TODO_cleanup_cfg; } @@ -706,10 +761,17 @@ vectorize_loops (void) if (loop_vectorized_call) { - fold_loop_vectorized_call (loop_vectorized_call, boolean_true_node); + fold_loop_internal_call (loop_vectorized_call, boolean_true_node); loop_vectorized_call = NULL; ret |= TODO_cleanup_cfg; } + if (loop_dist_alias_call) + { + tree value = gimple_call_arg (loop_dist_alias_call, 1); + fold_loop_internal_call (loop_dist_alias_call, value); + loop_dist_alias_call = NULL; + ret |= TODO_cleanup_cfg; + } if (new_loop) { @@ -741,7 +803,16 @@ vectorize_loops (void) gimple *g = vect_loop_vectorized_call (loop); if (g) { - fold_loop_vectorized_call (g, boolean_false_node); + fold_loop_internal_call (g, boolean_false_node); + ret |= TODO_cleanup_cfg; + g = NULL; + } + else + g = vect_loop_dist_alias_call (loop); + + if (g) + { + fold_loop_internal_call (g, boolean_false_node); ret |= TODO_cleanup_cfg; } } |