2012-07-24 Bill Schmidt <wschmidt@linux.ibm.com>

	* doc/tm.texi: Regenerate.
	* targhooks.c (default_init_cost): Add prologue and epilogue costs.
	(default_add_stmt_cost): Likewise; also handle NULL stmt_info.
	(default_finish_cost): Add prologue and epilogue costs.
	* targhooks.h (default_add_stmt_cost): Change parameter list.
	(default_finish_cost): Likewise.
	* target.def (init_cost): Change documentation string.
	(add_stmt_cost): Change documentation string and parameter list.
	(finish_cost): Likewise.
	* target.h (vect_cost_model_location): New enum.
	* tree-vectorizer.h (struct _slp_tree): Remove cost substruct.
	(struct _slp_instance): Remove cost substruct; rename stmt_cost_vec
	to body_cost_vec.
	(SLP_INSTANCE_OUTSIDE_OF_LOOP_COST): Remove.
	(SLP_INSTANCE_STMT_COST_VEC): Rename to SLP_INSTANCE_BODY_COST_VEC.
	(SLP_TREE_OUTSIDE_OF_LOOP_COST): Remove.
	(struct _vect_peel_extended_info): Rename stmt_cost_vec to
	body_cost_vec.
	(struct _stmt_vec_info): Remove cost substruct.
	(STMT_VINFO_OUTSIDE_OF_LOOP_COST): Remove.
	(stmt_vinfo_set_outside_of_loop_cost): Remove.
	(builtin_vectorization_cost): New function.
	(vect_get_stmt_cost): Change to use builtin_vectorization_cost.
	(add_stmt_cost): Change parameter list.
	(finish_cost): Likewise.
	(vect_model_simple_cost): Likewise.
	(vect_model_store_cost): Likewise.
	(vect_model_load_cost): Likewise.
	(record_stmt_cost): Likewise.
	(vect_get_load_cost): Likewise.
	(vect_get_known_peeling_cost): Likewise.
	* tree-vect-loop.c (vect_get_known_peeling_cost): Change parameter
	list; call record_stmt_cost for prologue and epilogue costs.
	(vect_estimate_min_profitable_iters): Call add_stmt_cost for
	prologue and epilogue costs; remove computation of vec_outside_cost;
	return vec_prologue_cost and vec_epilogue_cost from finish_cost.
	(vect_model_reduction_cost): Revise call to add_stmt_cost for body
	costs; call add_stmt_cost for prologue and epilogue costs.
	(vect_model_induction_cost): Revise call to add_stmt_cost for body
	costs; call add_stmt_cost for prologue costs.
	* tree-vect-data-refs.c (vect_get_data_access_cost): Change parameter
	list for function and arguments for calls to vect_get_load_cost and
	vect_get_store_cost.
	(vect_peeling_hash_get_lowest_cost): Change argument list for calls to
	vect_get_data_access_cost and vect_get_known_peeling_cost; use
	temporary vectors prologue_cost_vec and epilogue_cost_vec for the
	latter call and discard their results; rename stmt_cost_vec to
	body_cost_vec; correct possible storage leak for body_cost_vec.
	(vect_peeling_hash_choose_best_peeling): Rename stmt_cost_vec to
	body_cost_vec.
	(vect_enhance_data_refs_alignment): Rename stmt_cost_vec to
	body_cost_vec; add extra dummy parameter on calls to
	vect_get_data_access_cost; tolerate null si->stmt; add vect_body to
	argument list on call to add_stmt_cost.
	* tree-vect-stmts.c (record_stmt_cost): Change parameter list;
	rename stmt_cost_vec to body_cost_vec; tolerate null stmt_info; call
	builtin_vectorization_cost; add "where" parameter on call to
	add_stmt_cost.
	(vect_model_simple_cost): Change parameter list; call record_stmt_cost
	for prologue costs; remove call to stmt_vinfo_set_outside_of_loop_cost;
	rename stmt_cost_vec to body_cost_vec.
	(vect_model_promotion_demotion_cost): Add vect_body argument to call
	to add_stmt_cost; call add_stmt_cost for prologue costs; remove call
	to stmt_vinfo_set_outside_of_loop_cost.
	(vect_model_store_cost): Change parameter list; call record_stmt_cost
	for prologue costs; add vect_body argument to call to record_stmt_cost;
	rename stmt_cost_vec to body_cost_vec; remove call to
	stmt_vinfo_set_outside_of_loop_cost.
	(vect_get_store_cost): Rename stmt_cost_vec to body_cost_vec; add
	vect_body argument to calls to record_stmt_cost.
	(vect_model_load_cost): Change parameter list; rename stmt_cost_vec to
	body_cost_vec; add vect_body argument to calls to record_stmt_cost;
	remove call to stmt_vinfo_set_outside_of_loop_cost.
	(vect_get_load_cost): Change parameter list; rename stmt_cost_vec to
	body_cost_vec; add vect_body argument to calls to record_stmt_cost;
	call record_stmt_cost for prologue costs.
	(vectorizable_store): Change argument list for call to
	vect_model_store_cost.
	(vectorizable_load): Change argument list for call to
	vect_model_load_cost.
	(new_stmt_vec_info): Remove assignment to
	STMT_VINFO_OUTSIDE_OF_LOOP_COST.
	* config/spu/spu.c (spu_init_cost): Add prologue and epilogue costs.
	(spu_add_stmt_cost): Likewise; also handle NULL stmt_info.
	(spu_finish_cost): Add prologue and epilogue costs.
	* config/i386/i386.c (i386_init_cost): Add prologue and epilogue costs.
	(i386_add_stmt_cost): Likewise; also handle NULL stmt_info.
	(i386_finish_cost): Add prologue and epilogue costs.
	* config/rs6000/rs6000.c (rs6000_init_cost): Add prologue and epilogue
	costs.
	(rs6000_add_stmt_cost): Likewise; also handle NULL stmt_info.
	(rs6000_finish_cost): Add prologue and epilogue costs.
	* tree-vect-slp.c (vect_free_slp_instance): Rename
	SLP_INSTANCE_STMT_COST_VEC to SLP_INSTANCE_BODY_COST_VEC.
	(vect_create_new_slp_node): Remove assignment to
	SLP_TREE_OUTSIDE_OF_LOOP_COST.
	(vect_get_and_check_slp_defs): Change parameter list; change argument
	lists to calls to vect_model_store_cost and vect_model_simple_cost.
	(vect_build_slp_tree): Change parameter list; change argument lists
	to calls to vect_model_load_cost, vect_get_and_check_slp_defs, and
	recursive self-calls; remove setting of outside_cost from
	SLP_TREE_OUTSIDE_OF_LOOP_COST; add vect_body argument to call to
	record_stmt_cost.
	(vect_analyze_slp_instance): Rename stmt_cost_vec to body_cost_vec;
	rename SLP_INSTANCE_STMT_COST_VEC to SLP_INSTANCE_BODY_COST_VEC;
	remove assignment to SLP_INSTANCE_OUTSIDE_OF_LOOP_COST; record SLP
	prologue costs.
	(vect_bb_vectorization_profitable_p): Rename stmt_cost_vec to
	body_cost_vec; handle null ci->stmt; add vect_body argument to call
	to add_stmt_cost; simplify calls to targetm.vectorize.
	builtin_vectorization_cost; return vec_prologue_cost and
	vec_epilogue_cost from finish_cost.
	(vect_update_slp_costs_according_to_vf): Rename stmt_cost_vec to
	body_cost_vec; add vect_body argument to call to add_stmt_cost.



git-svn-id: svn+ssh://gcc.gnu.org/svn/gcc/trunk@189836 138bc75d-0d04-0410-961f-82ee72b054a4

1 files changed, 51 insertions, 29 deletions

diff --git a/gcc/tree-vect-data-refs.c b/gcc/tree-vect-data-refs.c
index eefd9fa143f..81a95210084 100644
--- a/gcc/tree-vect-data-refs.c
+++ b/gcc/tree-vect-data-refs.c
@@ -1212,10 +1212,11 @@ vector_alignment_reachable_p (struct data_reference *dr)
 
 /* Calculate the cost of the memory access represented by DR.  */
 
-static stmt_vector_for_cost
+static void
 vect_get_data_access_cost (struct data_reference *dr,
                            unsigned int *inside_cost,
-                           unsigned int *outside_cost)
+                           unsigned int *outside_cost,
+			   stmt_vector_for_cost *body_cost_vec)
 {
   gimple stmt = DR_STMT (dr);
   stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
@@ -1223,19 +1224,16 @@ vect_get_data_access_cost (struct data_reference *dr,
   loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
   int vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
   int ncopies = vf / nunits;
-  stmt_vector_for_cost stmt_cost_vec = VEC_alloc (stmt_info_for_cost, heap, 2);
 
   if (DR_IS_READ (dr))
-    vect_get_load_cost (dr, ncopies, true, inside_cost,
-			outside_cost, &stmt_cost_vec);
+    vect_get_load_cost (dr, ncopies, true, inside_cost, outside_cost,
+			NULL, body_cost_vec, false);
   else
-    vect_get_store_cost (dr, ncopies, inside_cost, &stmt_cost_vec);
+    vect_get_store_cost (dr, ncopies, inside_cost, body_cost_vec);
 
   if (vect_print_dump_info (REPORT_COST))
     fprintf (vect_dump, "vect_get_data_access_cost: inside_cost = %d, "
              "outside_cost = %d.", *inside_cost, *outside_cost);
-
-  return stmt_cost_vec;
 }
 
 
@@ -1328,7 +1326,12 @@ vect_peeling_hash_get_lowest_cost (void **slot, void *data)
   loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
   VEC (data_reference_p, heap) *datarefs = LOOP_VINFO_DATAREFS (loop_vinfo);
   struct data_reference *dr;
-  stmt_vector_for_cost stmt_cost_vec = NULL;
+  stmt_vector_for_cost prologue_cost_vec, body_cost_vec, epilogue_cost_vec;
+  int single_iter_cost;
+
+  prologue_cost_vec = VEC_alloc (stmt_info_for_cost, heap, 2);
+  body_cost_vec     = VEC_alloc (stmt_info_for_cost, heap, 2);
+  epilogue_cost_vec = VEC_alloc (stmt_info_for_cost, heap, 2);
 
   FOR_EACH_VEC_ELT (data_reference_p, datarefs, i, dr)
     {
@@ -1342,23 +1345,35 @@ vect_peeling_hash_get_lowest_cost (void **slot, void *data)
 
       save_misalignment = DR_MISALIGNMENT (dr);
       vect_update_misalignment_for_peel (dr, elem->dr, elem->npeel);
-      stmt_cost_vec = vect_get_data_access_cost (dr, &inside_cost,
-						 &outside_cost);
+      vect_get_data_access_cost (dr, &inside_cost, &outside_cost,
+				 &body_cost_vec);
       SET_DR_MISALIGNMENT (dr, save_misalignment);
     }
 
-  outside_cost += vect_get_known_peeling_cost (loop_vinfo, elem->npeel, &dummy,
-                         vect_get_single_scalar_iteration_cost (loop_vinfo));
+  single_iter_cost = vect_get_single_scalar_iteration_cost (loop_vinfo);
+  outside_cost += vect_get_known_peeling_cost (loop_vinfo, elem->npeel,
+					       &dummy, single_iter_cost,
+					       &prologue_cost_vec,
+					       &epilogue_cost_vec);
+
+  /* Prologue and epilogue costs are added to the target model later.
+     These costs depend only on the scalar iteration cost, the
+     number of peeling iterations finally chosen, and the number of
+     misaligned statements.  So discard the information found here.  */
+  VEC_free (stmt_info_for_cost, heap, prologue_cost_vec);
+  VEC_free (stmt_info_for_cost, heap, epilogue_cost_vec);
 
   if (inside_cost < min->inside_cost
       || (inside_cost == min->inside_cost && outside_cost < min->outside_cost))
     {
       min->inside_cost = inside_cost;
       min->outside_cost = outside_cost;
-      min->stmt_cost_vec = stmt_cost_vec;
+      min->body_cost_vec = body_cost_vec;
       min->peel_info.dr = elem->dr;
       min->peel_info.npeel = elem->npeel;
     }
+  else
+    VEC_free (stmt_info_for_cost, heap, body_cost_vec);
 
   return 1;
 }
@@ -1371,12 +1386,12 @@ vect_peeling_hash_get_lowest_cost (void **slot, void *data)
 static struct data_reference *
 vect_peeling_hash_choose_best_peeling (loop_vec_info loop_vinfo,
                                        unsigned int *npeel,
-				       stmt_vector_for_cost *stmt_cost_vec)
+				       stmt_vector_for_cost *body_cost_vec)
 {
    struct _vect_peel_extended_info res;
 
    res.peel_info.dr = NULL;
-   res.stmt_cost_vec = NULL;
+   res.body_cost_vec = NULL;
 
    if (flag_vect_cost_model)
      {
@@ -1393,7 +1408,7 @@ vect_peeling_hash_choose_best_peeling (loop_vec_info loop_vinfo,
      }
 
    *npeel = res.peel_info.npeel;
-   *stmt_cost_vec = res.stmt_cost_vec;
+   *body_cost_vec = res.body_cost_vec;
    return res.peel_info.dr;
 }
 
@@ -1510,7 +1525,7 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo)
   unsigned possible_npeel_number = 1;
   tree vectype;
   unsigned int nelements, mis, same_align_drs_max = 0;
-  stmt_vector_for_cost stmt_cost_vec = NULL;
+  stmt_vector_for_cost body_cost_vec = NULL;
 
   if (vect_print_dump_info (REPORT_DETAILS))
     fprintf (vect_dump, "=== vect_enhance_data_refs_alignment ===");
@@ -1714,11 +1729,14 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo)
           unsigned int store_inside_cost = 0, store_outside_cost = 0;
           unsigned int load_inside_penalty = 0, load_outside_penalty = 0;
           unsigned int store_inside_penalty = 0, store_outside_penalty = 0;
+	  stmt_vector_for_cost dummy = VEC_alloc (stmt_info_for_cost, heap, 2);
+
+          vect_get_data_access_cost (dr0, &load_inside_cost, &load_outside_cost,
+				     &dummy);
+          vect_get_data_access_cost (first_store, &store_inside_cost,
+				     &store_outside_cost, &dummy);
 
-          (void) vect_get_data_access_cost (dr0, &load_inside_cost,
-					    &load_outside_cost);
-          (void) vect_get_data_access_cost (first_store, &store_inside_cost,
-					    &store_outside_cost);
+	  VEC_free (stmt_info_for_cost, heap, dummy);
 
           /* Calculate the penalty for leaving FIRST_STORE unaligned (by
              aligning the load DR0).  */
@@ -1783,7 +1801,7 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo)
 
       /* Choose the best peeling from the hash table.  */
       dr0 = vect_peeling_hash_choose_best_peeling (loop_vinfo, &npeel,
-						   &stmt_cost_vec);
+						   &body_cost_vec);
       if (!dr0 || !npeel)
         do_peeling = false;
     }
@@ -1868,6 +1886,7 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo)
       if (do_peeling)
         {
 	  stmt_info_for_cost *si;
+	  void *data = LOOP_VINFO_TARGET_COST_DATA (loop_vinfo);
 
           /* (1.2) Update the DR_MISALIGNMENT of each data reference DR_i.
              If the misalignment of DR_i is identical to that of dr0 then set
@@ -1895,13 +1914,16 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo)
 	  /* We've delayed passing the inside-loop peeling costs to the
 	     target cost model until we were sure peeling would happen.
 	     Do so now.  */
-	  if (stmt_cost_vec)
+	  if (body_cost_vec)
 	    {
-	      FOR_EACH_VEC_ELT (stmt_info_for_cost, stmt_cost_vec, i, si)
-		(void) add_stmt_cost (LOOP_VINFO_TARGET_COST_DATA (loop_vinfo),
-				      si->count, si->kind,
-				      vinfo_for_stmt (si->stmt), si->misalign);
-	      VEC_free (stmt_info_for_cost, heap, stmt_cost_vec);
+	      FOR_EACH_VEC_ELT (stmt_info_for_cost, body_cost_vec, i, si)
+		{
+		  struct _stmt_vec_info *stmt_info
+		    = si->stmt ? vinfo_for_stmt (si->stmt) : NULL;
+		  (void) add_stmt_cost (data, si->count, si->kind, stmt_info,
+					si->misalign, vect_body);
+		}
+	      VEC_free (stmt_info_for_cost, heap, body_cost_vec);
 	    }
 
 	  stat = vect_verify_datarefs_alignment (loop_vinfo, NULL);