diff options
| -rw-r--r-- | gcc/ChangeLog | 57 | ||||
| -rw-r--r-- | gcc/Makefile.in | 2 | ||||
| -rw-r--r-- | gcc/cgraph.c | 337 | ||||
| -rw-r--r-- | gcc/cgraph.h | 46 | ||||
| -rw-r--r-- | gcc/cgraphbuild.c | 11 | ||||
| -rw-r--r-- | gcc/cgraphunit.c | 236 | ||||
| -rw-r--r-- | gcc/ipa-cp.c | 214 | ||||
| -rw-r--r-- | gcc/ipa-inline.c | 2 | ||||
| -rw-r--r-- | gcc/ipa-prop.c | 2 | ||||
| -rw-r--r-- | gcc/ipa-prop.h | 14 | ||||
| -rw-r--r-- | gcc/ipa-struct-reorg.c | 5 | ||||
| -rw-r--r-- | gcc/ipa.c | 34 | ||||
| -rw-r--r-- | gcc/passes.c | 2 | ||||
| -rw-r--r-- | gcc/tree-cfg.c | 2 | ||||
| -rw-r--r-- | gcc/tree-inline.c | 218 | ||||
| -rw-r--r-- | gcc/tree-inline.h | 2 |
16 files changed, 839 insertions, 345 deletions
diff --git a/gcc/ChangeLog b/gcc/ChangeLog index ae61006756c..508436b9677 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,60 @@ +2009-05-08 Jan Hubicka <jh@suse.cz> + + * cgraphbuild.c (compute_call_stmt_bb_frequency): Accept function argument; + handle correctly when profile is absent. + (build_cgraph_edges): Update. + (rebuild_cgraph_edges): Update. + * cgraph.c: Do not include varrau.h + (cgraph_set_call_stmt_including_clones, cgraph_create_edge_including_clones): + New function + (cgraph_update_edges_for_call_stmt_node): New stati cfunction. + (cgraph_update_edges_for_call_stmt): Handle clones. + (cgraph_remove_node): Handle clone tree. + (cgraph_remove_node_and_inline_clones): New function. + (dump_cgraph_node): Dump clone tree. + (cgraph_clone_node): Handle clone tree. + (clone_function_name): Bring here from tree-inline.c + (cgraph_create_virtual_clone): New function. + * cgraph.h (ipa_replace_map): Move ehre from ipa.h + (cgraph_clone_info): New function + (strut cgraph_node): Add clone_info and new clone tree pointers. + (cgraph_remove_node_and_inline_clones, cgraph_set_call_stmt_including_clones, + cgraph_create_edge_including_clones, cgraph_create_virtual_clone): Declare. + (cgraph_function_versioning): Use VEC argument. + (compute_call_stmt_bb_frequency): Update prototype. + (cgraph_materialize_all_clones): New function. + * ipa-cp.c (ipcp_update_cloned_node): Remove. + (ipcp_create_replace_map): Update to VECtors. + (ipcp_update_callgraph): Use virtual clones. + (ipcp_update_bb_counts, ipcp_update_edges_counts): Remove. + (ipcp_update_profiling): Do not update local profiling. + (ipcp_insert_stage): Use VECtors and virtual clones. + * cgraphunit.c (verify_cgraph_node): Verify clone tree. + (clone_of_p): New function. + (cgraph_preserve_function_body_p): Use clone tree. + (cgraph_optimize): Materialize clones. + (cgraph_function_versioning): Update for VECtors. + (save_inline_function_body): Use clone tree. + (cgraph_materialize_clone, cgraph_materialize_all_clones): New functions. + * ipa-inline.c (cgraph_default_inline_p): Use analyzed flags. + * ipa.c: Include gimple.h. + (cgraph_remove_unreachable_nodes): Use clone tree. + * ipa-prop.c (ipa_note_param_call): Update call of compute_call_stmt_bb_frequency. + * ipa-prop.h (ipa_replace_map): Move to cgraph.h. + * tree-inline.c: Do not include varray.h; do not include gt-tree-inline.h + (copy_bb): Handle updating of clone tree; add new edge when new call + appears. + (expand_call_inline): Be strict about every call having edge. + (clone_fn_id_num, clone_function_name): Move to cgraph.c. + (delete_unreachable_blocks_update_callgraph): New function. + (tree_function_versioning): Use VECtors; always remove unreachable blocks + and fold conditionals. + * tree-inline.h: Do not include varray.h + (tree_function_versioning): Remove. + * Makefile.in (GTFILES): Remove tree-inline.c + * passes.c (do_per_function): Do only functions having body. + * ipa-struct-reorg.c (do_reorg_1, collect_data_accesses): Handle cone tree. + 2009-05-08 H.J. Lu <hongjiu.lu@intel.com> Andrew Morrow <acm@google.com> diff --git a/gcc/Makefile.in b/gcc/Makefile.in index 4cd07f747d1..9a4d62b007a 100644 --- a/gcc/Makefile.in +++ b/gcc/Makefile.in @@ -3342,7 +3342,7 @@ GTFILES = $(CPP_ID_DATA_H) $(srcdir)/input.h $(srcdir)/coretypes.h \ $(srcdir)/tree-ssa-propagate.c \ $(srcdir)/tree-phinodes.c \ $(srcdir)/ipa-reference.c \ - $(srcdir)/tree-ssa-structalias.c $(srcdir)/tree-inline.c \ + $(srcdir)/tree-ssa-structalias.c \ $(srcdir)/tree-ssa-alias.h \ @all_gtfiles@ diff --git a/gcc/cgraph.c b/gcc/cgraph.c index ce696e211b6..2f68d94e3dd 100644 --- a/gcc/cgraph.c +++ b/gcc/cgraph.c @@ -78,7 +78,6 @@ The callgraph: #include "target.h" #include "basic-block.h" #include "cgraph.h" -#include "varray.h" #include "output.h" #include "intl.h" #include "gimple.h" @@ -628,7 +627,7 @@ cgraph_edge (struct cgraph_node *node, gimple call_stmt) } -/* Change field call_smt of edge E to NEW_STMT. */ +/* Change field call_stmt of edge E to NEW_STMT. */ void cgraph_set_call_stmt (struct cgraph_edge *e, gimple new_stmt) @@ -655,6 +654,79 @@ cgraph_set_call_stmt (struct cgraph_edge *e, gimple new_stmt) } } +/* Like cgraph_set_call_stmt but walk the clone tree and update all clones sharing + same function body. */ + +void +cgraph_set_call_stmt_including_clones (struct cgraph_node *orig, + gimple old_stmt, gimple new_stmt) +{ + struct cgraph_node *node; + struct cgraph_edge *edge = cgraph_edge (orig, old_stmt); + + if (edge) + cgraph_set_call_stmt (edge, new_stmt); + if (orig->clones) + for (node = orig->clones; node != orig;) + { + struct cgraph_edge *edge = cgraph_edge (node, old_stmt); + if (edge) + cgraph_set_call_stmt (edge, new_stmt); + if (node->clones) + node = node->clones; + else if (node->next_sibling_clone) + node = node->next_sibling_clone; + else + { + while (node != orig && !node->next_sibling_clone) + node = node->clone_of; + if (node != orig) + node = node->next_sibling_clone; + } + } +} + +/* Like cgraph_create_edge walk the clone tree and update all clones sharing + same function body. + + TODO: COUNT and LOOP_DEPTH should be properly distributed based on relative + frequencies of the clones. + */ + +void +cgraph_create_edge_including_clones (struct cgraph_node *orig, struct cgraph_node *callee, + gimple stmt, gcov_type count, int freq, + int loop_depth, + cgraph_inline_failed_t reason) +{ + struct cgraph_node *node; + + cgraph_create_edge (orig, callee, stmt, count, freq, loop_depth)->inline_failed = + reason; + + if (orig->clones) + for (node = orig->clones; node != orig;) + { + /* It is possible that we already constant propagated into the clone + and turned indirect call into dirrect call. */ + if (!cgraph_edge (node, stmt)) + cgraph_create_edge (node, callee, stmt, count, freq, + loop_depth)->inline_failed = reason; + + if (node->clones) + node = node->clones; + else if (node->next_sibling_clone) + node = node->next_sibling_clone; + else + { + while (node != orig && !node->next_sibling_clone) + node = node->clone_of; + if (node != orig) + node = node->next_sibling_clone; + } + } +} + /* Give initial reasons why inlining would fail on EDGE. This gets either nullified or usually overwritten by more precise reasons later. */ @@ -828,12 +900,12 @@ cgraph_redirect_edge_callee (struct cgraph_edge *e, struct cgraph_node *n) /* Update or remove the corresponding cgraph edge if a GIMPLE_CALL OLD_STMT changed into NEW_STMT. */ -void -cgraph_update_edges_for_call_stmt (gimple old_stmt, gimple new_stmt) +static void +cgraph_update_edges_for_call_stmt_node (struct cgraph_node *node, + gimple old_stmt, gimple new_stmt) { tree new_call = (is_gimple_call (new_stmt)) ? gimple_call_fn (new_stmt) : 0; tree old_call = (is_gimple_call (old_stmt)) ? gimple_call_fn (old_stmt) : 0; - struct cgraph_node *node = cgraph_node (cfun->decl); if (old_call != new_call) { @@ -870,6 +942,34 @@ cgraph_update_edges_for_call_stmt (gimple old_stmt, gimple new_stmt) } } +/* Update or remove the corresponding cgraph edge if a GIMPLE_CALL + OLD_STMT changed into NEW_STMT. */ + +void +cgraph_update_edges_for_call_stmt (gimple old_stmt, gimple new_stmt) +{ + struct cgraph_node *orig = cgraph_node (cfun->decl); + struct cgraph_node *node; + + cgraph_update_edges_for_call_stmt_node (orig, old_stmt, new_stmt); + if (orig->clones) + for (node = orig->clones; node != orig;) + { + cgraph_update_edges_for_call_stmt_node (node, old_stmt, new_stmt); + if (node->clones) + node = node->clones; + else if (node->next_sibling_clone) + node = node->next_sibling_clone; + else + { + while (node != orig && !node->next_sibling_clone) + node = node->clone_of; + if (node != orig) + node = node->next_sibling_clone; + } + } +} + /* Remove all callees from the node. */ @@ -998,24 +1098,100 @@ cgraph_remove_node (struct cgraph_node *node) slot = htab_find_slot (cgraph_hash, node, NO_INSERT); if (*slot == node) { - if (node->next_clone) - { - struct cgraph_node *new_node = node->next_clone; + struct cgraph_node *next_inline_clone; - *slot = new_node; - node->next_clone->prev_clone = NULL; - } + for (next_inline_clone = node->clones; + next_inline_clone && next_inline_clone->decl != node->decl; + next_inline_clone = next_inline_clone->next_sibling_clone) + ; + + /* If there is inline clone of the node being removed, we need + to put it into the position of removed node and reorganize all + other clones to be based on it. */ + if (next_inline_clone) + { + struct cgraph_node *n; + struct cgraph_node *new_clones; + + *slot = next_inline_clone; + + /* Unlink inline clone from the list of clones of removed node. */ + if (next_inline_clone->next_sibling_clone) + next_inline_clone->next_sibling_clone->prev_sibling_clone + = next_inline_clone->prev_sibling_clone; + if (next_inline_clone->prev_sibling_clone) + { + next_inline_clone->prev_sibling_clone->next_sibling_clone + = next_inline_clone->next_sibling_clone; + } + else + node->clones = next_inline_clone->next_sibling_clone; + + new_clones = node->clones; + node->clones = NULL; + + /* Copy clone info. */ + next_inline_clone->clone = node->clone; + + /* Now place it into clone tree at same level at NODE. */ + next_inline_clone->clone_of = node->clone_of; + next_inline_clone->prev_sibling_clone = NULL; + next_inline_clone->next_sibling_clone = NULL; + if (node->clone_of) + { + next_inline_clone->next_sibling_clone = node->clone_of->clones; + node->clone_of->clones = next_inline_clone; + } + + /* Merge the clone list. */ + if (new_clones) + { + if (!next_inline_clone->clones) + next_inline_clone->clones = new_clones; + else + { + n = next_inline_clone->clones; + while (n->next_sibling_clone) + n = n->next_sibling_clone; + n->next_sibling_clone = new_clones; + new_clones->prev_sibling_clone = n; + } + } + + /* Update clone_of pointers. */ + n = new_clones; + while (n) + { + n->clone_of = next_inline_clone; + n = n->next_sibling_clone; + } + } else { htab_clear_slot (cgraph_hash, slot); kill_body = true; } + } else + gcc_assert (node->clone_of); + if (node->prev_sibling_clone) + node->prev_sibling_clone->next_sibling_clone = node->next_sibling_clone; + else if (node->clone_of) + node->clone_of->clones = node->next_sibling_clone; + if (node->next_sibling_clone) + node->next_sibling_clone->prev_sibling_clone = node->prev_sibling_clone; + if (node->clones) { - node->prev_clone->next_clone = node->next_clone; - if (node->next_clone) - node->next_clone->prev_clone = node->prev_clone; + struct cgraph_node *n; + + for (n = node->clones; n->next_sibling_clone; n = n->next_sibling_clone) + n->clone_of = node->clone_of; + n->clone_of = node->clone_of; + n->next_sibling_clone = node->clone_of->clones; + if (node->clone_of->clones) + node->clone_of->clones->prev_sibling_clone = n; + node->clone_of->clones = node->clones; } /* While all the clones are removed after being proceeded, the function @@ -1025,7 +1201,7 @@ cgraph_remove_node (struct cgraph_node *node) if (!kill_body && *slot) { struct cgraph_node *n = (struct cgraph_node *) *slot; - if (!n->next_clone && !n->global.inlined_to + if (!n->clones && !n->clone_of && !n->global.inlined_to && (cgraph_global_info_ready && (TREE_ASM_WRITTEN (n->decl) || DECL_EXTERNAL (n->decl)))) kill_body = true; @@ -1059,6 +1235,21 @@ cgraph_remove_node (struct cgraph_node *node) free_nodes = node; } +/* Remove the node from cgraph. */ + +void +cgraph_remove_node_and_inline_clones (struct cgraph_node *node) +{ + struct cgraph_edge *e, *next; + for (e = node->callees; e; e = next) + { + next = e->next_callee; + if (!e->inline_failed) + cgraph_remove_node_and_inline_clones (e->callee); + } + cgraph_remove_node (node); +} + /* Notify finalize_compilation_unit that given node is reachable. */ void @@ -1166,6 +1357,10 @@ dump_cgraph_node (FILE *f, struct cgraph_node *node) fprintf (f, " (inline copy in %s/%i)", cgraph_node_name (node->global.inlined_to), node->global.inlined_to->uid); + if (node->clone_of) + fprintf (f, " (clone of %s/%i)", + cgraph_node_name (node->clone_of), + node->clone_of->uid); if (cgraph_function_flags_ready) fprintf (f, " availability:%s", cgraph_availability_names [cgraph_function_body_availability (node)]); @@ -1382,6 +1577,7 @@ cgraph_clone_node (struct cgraph_node *n, gcov_type count, int freq, new_node->global = n->global; new_node->rtl = n->rtl; new_node->count = count; + new_node->clone = n->clone; if (n->count) { if (new_node->count > n->count) @@ -1402,16 +1598,117 @@ cgraph_clone_node (struct cgraph_node *n, gcov_type count, int freq, cgraph_clone_edge (e, new_node, e->call_stmt, count_scale, freq, loop_nest, update_original); - new_node->next_clone = n->next_clone; - new_node->prev_clone = n; - n->next_clone = new_node; - if (new_node->next_clone) - new_node->next_clone->prev_clone = new_node; + new_node->next_sibling_clone = n->clones; + if (n->clones) + n->clones->prev_sibling_clone = new_node; + n->clones = new_node; + new_node->clone_of = n; cgraph_call_node_duplication_hooks (n, new_node); return new_node; } +/* Create a new name for omp child function. Returns an identifier. */ + +static GTY(()) unsigned int clone_fn_id_num; + +static tree +clone_function_name (tree decl) +{ + tree name = DECL_ASSEMBLER_NAME (decl); + size_t len = IDENTIFIER_LENGTH (name); + char *tmp_name, *prefix; + + prefix = XALLOCAVEC (char, len + strlen ("_clone") + 1); + memcpy (prefix, IDENTIFIER_POINTER (name), len); + strcpy (prefix + len, "_clone"); +#ifndef NO_DOT_IN_LABEL + prefix[len] = '.'; +#elif !defined NO_DOLLAR_IN_LABEL + prefix[len] = '$'; +#endif + ASM_FORMAT_PRIVATE_NAME (tmp_name, prefix, clone_fn_id_num++); + return get_identifier (tmp_name); +} + +/* Create callgraph node clone with new declaration. The actual body will + be copied later at compilation stage. + + TODO: after merging in ipa-sra use function call notes instead of args_to_skip + bitmap interface. + */ +struct cgraph_node * +cgraph_create_virtual_clone (struct cgraph_node *old_node, + VEC(cgraph_edge_p,heap) *redirect_callers, + VEC(ipa_replace_map_p,gc) *tree_map, + bitmap args_to_skip) +{ + tree old_decl = old_node->decl; + struct cgraph_node *new_node = NULL; + tree new_decl; + struct cgraph_node key, **slot; + unsigned i; + struct cgraph_edge *e; + + gcc_assert (tree_versionable_function_p (old_decl)); + + /* Make a new FUNCTION_DECL tree node */ + if (!args_to_skip) + new_decl = copy_node (old_decl); + else + new_decl = build_function_decl_skip_args (old_decl, args_to_skip); + DECL_STRUCT_FUNCTION (new_decl) = NULL; + + /* Generate a new name for the new version. */ + DECL_NAME (new_decl) = clone_function_name (old_decl); + SET_DECL_ASSEMBLER_NAME (new_decl, DECL_NAME (new_decl)); + SET_DECL_RTL (new_decl, NULL); + + new_node = cgraph_clone_node (old_node, old_node->count, + CGRAPH_FREQ_BASE, 0, false); + new_node->decl = new_decl; + /* Update the properties. + Make clone visible only within this translation unit. Make sure + that is not weak also. + ??? We cannot use COMDAT linkage because there is no + ABI support for this. */ + DECL_EXTERNAL (new_node->decl) = 0; + DECL_ONE_ONLY (new_node->decl) = 0; + TREE_PUBLIC (new_node->decl) = 0; + DECL_COMDAT (new_node->decl) = 0; + DECL_WEAK (new_node->decl) = 0; + new_node->clone.tree_map = tree_map; + new_node->clone.args_to_skip = args_to_skip; + new_node->local.externally_visible = 0; + new_node->local.local = 1; + new_node->lowered = true; + new_node->reachable = true; + + key.decl = new_decl; + slot = (struct cgraph_node **) htab_find_slot (cgraph_hash, &key, INSERT); + gcc_assert (!*slot); + *slot = new_node; + if (assembler_name_hash) + { + void **aslot; + tree name = DECL_ASSEMBLER_NAME (new_decl); + + aslot = htab_find_slot_with_hash (assembler_name_hash, name, + decl_assembler_name_hash (name), + INSERT); + gcc_assert (!*aslot); + *aslot = new_node; + } + for (i = 0; VEC_iterate (cgraph_edge_p, redirect_callers, i, e); i++) + { + /* Redirect calls to the old version node to point to its new + version. */ + cgraph_redirect_edge_callee (e, new_node); + } + + return new_node; +} + /* NODE is no longer nested function; update cgraph accordingly. */ void cgraph_unnest_node (struct cgraph_node *node) diff --git a/gcc/cgraph.h b/gcc/cgraph.h index b100fa6ddc4..b94fcffbc77 100644 --- a/gcc/cgraph.h +++ b/gcc/cgraph.h @@ -119,6 +119,29 @@ struct GTY(()) cgraph_rtl_info { unsigned int preferred_incoming_stack_boundary; }; +/* Represent which DECL tree (or reference to such tree) + will be replaced by another tree while versioning. */ +struct GTY(()) ipa_replace_map +{ + /* The tree that will be replaced. */ + tree old_tree; + /* The new (replacing) tree. */ + tree new_tree; + /* True when a substitution should be done, false otherwise. */ + bool replace_p; + /* True when we replace a reference to old_tree. */ + bool ref_p; +}; +typedef struct ipa_replace_map *ipa_replace_map_p; +DEF_VEC_P(ipa_replace_map_p); +DEF_VEC_ALLOC_P(ipa_replace_map_p,gc); + +struct GTY(()) cgraph_clone_info +{ + VEC(ipa_replace_map_p,gc)* tree_map; + bitmap args_to_skip; +}; + /* The cgraph data structure. Each function decl has assigned cgraph_node listing callees and callers. */ @@ -137,8 +160,10 @@ struct GTY((chain_next ("%h.next"), chain_prev ("%h.previous"))) cgraph_node { /* Pointer to the next function in cgraph_nodes_queue. */ struct cgraph_node *next_needed; /* Pointer to the next clone. */ - struct cgraph_node *next_clone; - struct cgraph_node *prev_clone; + struct cgraph_node *next_sibling_clone; + struct cgraph_node *prev_sibling_clone; + struct cgraph_node *clones; + struct cgraph_node *clone_of; /* For functions with many calls sites it holds map from call expression to the edge to speed up cgraph_edge function. */ htab_t GTY((param_is (struct cgraph_edge))) call_site_hash; @@ -148,6 +173,7 @@ struct GTY((chain_next ("%h.next"), chain_prev ("%h.previous"))) cgraph_node { struct cgraph_local_info local; struct cgraph_global_info global; struct cgraph_rtl_info rtl; + struct cgraph_clone_info clone; /* Expected number of executions: calculated in profile.c. */ gcov_type count; @@ -344,6 +370,7 @@ void debug_cgraph_node (struct cgraph_node *); void cgraph_insert_node_to_hashtable (struct cgraph_node *node); void cgraph_remove_edge (struct cgraph_edge *); void cgraph_remove_node (struct cgraph_node *); +void cgraph_remove_node_and_inline_clones (struct cgraph_node *); void cgraph_release_function_body (struct cgraph_node *); void cgraph_node_remove_callees (struct cgraph_node *node); struct cgraph_edge *cgraph_create_edge (struct cgraph_node *, @@ -353,6 +380,11 @@ struct cgraph_node *cgraph_node (tree); struct cgraph_node *cgraph_node_for_asm (tree asmname); struct cgraph_edge *cgraph_edge (struct cgraph_node *, gimple); void cgraph_set_call_stmt (struct cgraph_edge *, gimple); +void cgraph_set_call_stmt_including_clones (struct cgraph_node *, gimple, gimple); +void cgraph_create_edge_including_clones (struct cgraph_node *, + struct cgraph_node *, + gimple, gcov_type, int, int, + cgraph_inline_failed_t); void cgraph_update_edges_for_call_stmt (gimple, gimple); struct cgraph_local_info *cgraph_local_info (tree); struct cgraph_global_info *cgraph_global_info (tree); @@ -374,6 +406,10 @@ void cgraph_unnest_node (struct cgraph_node *); enum availability cgraph_function_body_availability (struct cgraph_node *); void cgraph_add_new_function (tree, bool); const char* cgraph_inline_failed_string (cgraph_inline_failed_t); +struct cgraph_node * cgraph_create_virtual_clone (struct cgraph_node *old_node, + VEC(cgraph_edge_p,heap)*, + VEC(ipa_replace_map_p,gc)* tree_map, + bitmap args_to_skip); /* In cgraphunit.c */ void cgraph_finalize_function (tree, bool); @@ -391,8 +427,9 @@ void cgraph_reset_static_var_maps (void); void init_cgraph (void); struct cgraph_node *cgraph_function_versioning (struct cgraph_node *, VEC(cgraph_edge_p,heap)*, - varray_type, + VEC(ipa_replace_map_p,gc)*, bitmap); +void tree_function_versioning (tree, tree, VEC (ipa_replace_map_p,gc)*, bool, bitmap); void cgraph_analyze_function (struct cgraph_node *); struct cgraph_node *save_inline_function_body (struct cgraph_node *); void record_references_in_initializer (tree); @@ -421,10 +458,11 @@ struct cgraph_2edge_hook_list *cgraph_add_edge_duplication_hook (cgraph_2edge_ho void cgraph_remove_edge_duplication_hook (struct cgraph_2edge_hook_list *); struct cgraph_2node_hook_list *cgraph_add_node_duplication_hook (cgraph_2node_hook, void *); void cgraph_remove_node_duplication_hook (struct cgraph_2node_hook_list *); +void cgraph_materialize_all_clones (void); /* In cgraphbuild.c */ unsigned int rebuild_cgraph_edges (void); -int compute_call_stmt_bb_frequency (basic_block bb); +int compute_call_stmt_bb_frequency (tree, basic_block bb); /* In ipa.c */ bool cgraph_remove_unreachable_nodes (bool, FILE *); diff --git a/gcc/cgraphbuild.c b/gcc/cgraphbuild.c index f244a1e315d..fb56ce5354a 100644 --- a/gcc/cgraphbuild.c +++ b/gcc/cgraphbuild.c @@ -81,11 +81,14 @@ record_reference (tree *tp, int *walk_subtrees, void *data ATTRIBUTE_UNUSED) /* Computes the frequency of the call statement so that it can be stored in cgraph_edge. BB is the basic block of the call statement. */ int -compute_call_stmt_bb_frequency (basic_block bb) +compute_call_stmt_bb_frequency (tree decl, basic_block bb) { int entry_freq = ENTRY_BLOCK_PTR->frequency; int freq = bb->frequency; + if (profile_status_for_function (DECL_STRUCT_FUNCTION (decl)) == PROFILE_ABSENT) + return CGRAPH_FREQ_BASE; + if (!entry_freq) entry_freq = 1, freq++; @@ -121,7 +124,7 @@ build_cgraph_edges (void) size_t i; size_t n = gimple_call_num_args (stmt); cgraph_create_edge (node, cgraph_node (decl), stmt, - bb->count, compute_call_stmt_bb_frequency (bb), + bb->count, compute_call_stmt_bb_frequency (current_function_decl, bb), bb->loop_depth); for (i = 0; i < n; i++) walk_tree (gimple_call_arg_ptr (stmt, i), record_reference, @@ -224,7 +227,9 @@ rebuild_cgraph_edges (void) if (is_gimple_call (stmt) && (decl = gimple_call_fndecl (stmt))) cgraph_create_edge (node, cgraph_node (decl), stmt, - bb->count, compute_call_stmt_bb_frequency (bb), + bb->count, + compute_call_stmt_bb_frequency + (current_function_decl, bb), bb->loop_depth); } diff --git a/gcc/cgraphunit.c b/gcc/cgraphunit.c index 9b7ca8c9b84..9366ebe2869 100644 --- a/gcc/cgraphunit.c +++ b/gcc/cgraphunit.c @@ -548,12 +548,20 @@ cgraph_mark_if_needed (tree decl) cgraph_mark_needed_node (node); } +/* Return TRUE if NODE2 is equivalent to NODE or its clone. */ +static bool +clone_of_p (struct cgraph_node *node, struct cgraph_node *node2) +{ + while (node != node2 && node2) + node2 = node2->clone_of; + return node2 != NULL; +} + /* Verify cgraph nodes of given cgraph node. */ void verify_cgraph_node (struct cgraph_node *node) { struct cgraph_edge *e; - struct cgraph_node *main_clone; struct function *this_cfun = DECL_STRUCT_FUNCTION (node->decl); struct function *saved_cfun = cfun; basic_block this_block; @@ -629,17 +637,53 @@ verify_cgraph_node (struct cgraph_node *node) error_found = true; } - for (main_clone = cgraph_node (node->decl); main_clone; - main_clone = main_clone->next_clone) - if (main_clone == node) - break; if (!cgraph_node (node->decl)) { error ("node not found in cgraph_hash"); error_found = true; } - if (node->analyzed + if (node->clone_of) + { + struct cgraph_node *n; + for (n = node->clone_of->clones; n; n = n->next_sibling_clone) + if (n == node) + break; + if (!n) + { + error ("node has wrong clone_of"); + error_found = true; + } + } + if (node->clones) + { + struct cgraph_node *n; + for (n = node->clones; n; n = n->next_sibling_clone) + if (n->clone_of != node) + break; + if (n) + { + error ("node has wrong clone list"); + error_found = true; + } + } + if ((node->prev_sibling_clone || node->next_sibling_clone) && !node->clone_of) + { + error ("node is in clone list but it is not clone"); + error_found = true; + } + if (!node->prev_sibling_clone && node->clone_of && node->clone_of->clones != node) + { + error ("node has wrong prev_clone pointer"); + error_found = true; + } + if (node->prev_sibling_clone && node->prev_sibling_clone->next_sibling_clone != node) + { + error ("double linked list of clones corrupted"); + error_found = true; + } + + if (node->analyzed && gimple_has_body_p (node->decl) && !TREE_ASM_WRITTEN (node->decl) && (!DECL_EXTERNAL (node->decl) || node->global.inlined_to)) { @@ -668,8 +712,8 @@ verify_cgraph_node (struct cgraph_node *node) debug_gimple_stmt (stmt); error_found = true; } - if (e->callee->decl != cgraph_node (decl)->decl - && e->inline_failed) + if (!clone_of_p (cgraph_node (decl), e->callee) + && !e->callee->global.inlined_to) { error ("edge points to wrong declaration:"); debug_tree (e->callee->decl); @@ -1227,9 +1271,9 @@ cgraph_preserve_function_body_p (tree decl) gcc_assert (cgraph_global_info_ready); /* Look if there is any clone around. */ - for (node = cgraph_node (decl); node; node = node->next_clone) - if (node->global.inlined_to) - return true; + node = cgraph_node (decl); + if (node->clones) + return true; return false; } @@ -1312,6 +1356,7 @@ cgraph_optimize (void) verify_cgraph (); #endif + cgraph_materialize_all_clones (); cgraph_mark_functions_to_output (); cgraph_state = CGRAPH_STATE_EXPANSION; @@ -1528,7 +1573,7 @@ cgraph_copy_node_for_versioning (struct cgraph_node *old_version, struct cgraph_node * cgraph_function_versioning (struct cgraph_node *old_version_node, VEC(cgraph_edge_p,heap) *redirect_callers, - varray_type tree_map, + VEC (ipa_replace_map_p,gc)* tree_map, bitmap args_to_skip) { tree old_decl = old_version_node->decl; @@ -1581,19 +1626,50 @@ cgraph_function_versioning (struct cgraph_node *old_version_node, struct cgraph_node * save_inline_function_body (struct cgraph_node *node) { - struct cgraph_node *first_clone; + struct cgraph_node *first_clone, *n; gcc_assert (node == cgraph_node (node->decl)); cgraph_lower_function (node); - first_clone = node->next_clone; + first_clone = node->clones; first_clone->decl = copy_node (node->decl); - node->next_clone = NULL; - first_clone->prev_clone = NULL; cgraph_insert_node_to_hashtable (first_clone); gcc_assert (first_clone == cgraph_node (first_clone->decl)); + if (first_clone->next_sibling_clone) + { + for (n = first_clone->next_sibling_clone; n->next_sibling_clone; n = n->next_sibling_clone) + n->clone_of = first_clone; + n->clone_of = first_clone; + n->next_sibling_clone = first_clone->clones; + if (first_clone->clones) + first_clone->clones->prev_sibling_clone = n; + first_clone->clones = first_clone->next_sibling_clone; + first_clone->next_sibling_clone->prev_sibling_clone = NULL; + first_clone->next_sibling_clone = NULL; + gcc_assert (!first_clone->prev_sibling_clone); + } + first_clone->clone_of = NULL; + node->clones = NULL; + + if (first_clone->clones) + for (n = first_clone->clones; n != first_clone;) + { + gcc_assert (n->decl == node->decl); + n->decl = first_clone->decl; + if (n->clones) + n = n->clones; + else if (n->next_sibling_clone) + n = n->next_sibling_clone; + else + { + while (n != first_clone && !n->next_sibling_clone) + n = n->clone_of; + if (n != first_clone) + n = n->next_sibling_clone; + } + } /* Copy the OLD_VERSION_NODE function tree to the new version. */ tree_function_versioning (node->decl, first_clone->decl, NULL, true, NULL); @@ -1603,12 +1679,136 @@ save_inline_function_body (struct cgraph_node *node) TREE_PUBLIC (first_clone->decl) = 0; DECL_COMDAT (first_clone->decl) = 0; - for (node = first_clone->next_clone; node; node = node->next_clone) - node->decl = first_clone->decl; #ifdef ENABLE_CHECKING verify_cgraph_node (first_clone); #endif return first_clone; } +/* Given virtual clone, turn it into actual clone. */ +static void +cgraph_materialize_clone (struct cgraph_node *node) +{ + bitmap_obstack_initialize (NULL); + /* Copy the OLD_VERSION_NODE function tree to the new version. */ + tree_function_versioning (node->clone_of->decl, node->decl, + node->clone.tree_map, true, + node->clone.args_to_skip); + + /* Function is no longer clone. */ + if (node->next_sibling_clone) + node->next_sibling_clone->prev_sibling_clone = node->prev_sibling_clone; + if (node->prev_sibling_clone) + node->prev_sibling_clone->next_sibling_clone = node->next_sibling_clone; + else + node->clone_of->clones = node->next_sibling_clone; + node->next_sibling_clone = NULL; + node->prev_sibling_clone = NULL; + node->clone_of = NULL; + bitmap_obstack_release (NULL); +} + +/* Once all functions from compilation unit are in memory, produce all clones + and update all calls. + We might also do this on demand if we don't want to bring all functions to + memory prior compilation, but current WHOPR implementation does that and it is + is bit easier to keep everything right in this order. */ +void +cgraph_materialize_all_clones (void) +{ + struct cgraph_node *node; + bool stabilized = false; + + if (cgraph_dump_file) + fprintf (cgraph_dump_file, "Materializing clones\n"); +#ifdef ENABLE_CHECKING + verify_cgraph (); +#endif + + /* We can also do topological order, but number of iterations should be + bounded by number of IPA passes since single IPA pass is probably not + going to create clones of clones it created itself. */ + while (!stabilized) + { + stabilized = true; + for (node = cgraph_nodes; node; node = node->next) + { + if (node->clone_of && node->decl != node->clone_of->decl + && !gimple_has_body_p (node->decl)) + { + if (gimple_has_body_p (node->clone_of->decl)) + { + if (cgraph_dump_file) + fprintf (cgraph_dump_file, " clonning %s to %s", + cgraph_node_name (node->clone_of), + cgraph_node_name (node)); + cgraph_materialize_clone (node); + } + else + stabilized = false; + } + } + } + if (cgraph_dump_file) + fprintf (cgraph_dump_file, "Updating call sites\n"); + for (node = cgraph_nodes; node; node = node->next) + if (node->analyzed && gimple_has_body_p (node->decl) + && (!node->clone_of || node->clone_of->decl != node->decl)) + { + struct cgraph_edge *e; + + current_function_decl = node->decl; + push_cfun (DECL_STRUCT_FUNCTION (node->decl)); + for (e = node->callees; e; e = e->next_callee) + { + tree decl = gimple_call_fndecl (e->call_stmt); + if (decl != e->callee->decl) + { + gimple new_stmt; + gimple_stmt_iterator gsi; + + if (cgraph_dump_file) + { + fprintf (cgraph_dump_file, "updating call of %s in %s:", + cgraph_node_name (node), + cgraph_node_name (e->callee)); + print_gimple_stmt (cgraph_dump_file, e->call_stmt, 0, dump_flags); + } + + if (e->callee->clone.args_to_skip) + new_stmt = gimple_call_copy_skip_args (e->call_stmt, + e->callee->clone.args_to_skip); + else + new_stmt = e->call_stmt; + if (gimple_vdef (new_stmt) + && TREE_CODE (gimple_vdef (new_stmt)) == SSA_NAME) + SSA_NAME_DEF_STMT (gimple_vdef (new_stmt)) = new_stmt; + gimple_call_set_fndecl (new_stmt, e->callee->decl); + + gsi = gsi_for_stmt (e->call_stmt); + gsi_replace (&gsi, new_stmt, true); + + /* Update EH information too, just in case. */ + if (!stmt_could_throw_p (new_stmt) + && lookup_stmt_eh_region (new_stmt)) + remove_stmt_from_eh_region (new_stmt); + + cgraph_set_call_stmt_including_clones (node, e->call_stmt, new_stmt); + + if (cgraph_dump_file) + { + fprintf (cgraph_dump_file, " updated to:"); + print_gimple_stmt (cgraph_dump_file, e->call_stmt, 0, dump_flags); + } + } + } + pop_cfun (); + current_function_decl = NULL; +#ifdef ENABLE_CHECKING + verify_cgraph_node (node); +#endif + } + cgraph_remove_unreachable_nodes (false, cgraph_dump_file); +} + #include "gt-cgraphunit.h" diff --git a/gcc/ipa-cp.c b/gcc/ipa-cp.c index 21dd8871e0a..d6390b089a9 100644 --- a/gcc/ipa-cp.c +++ b/gcc/ipa-cp.c @@ -186,34 +186,6 @@ ipcp_analyze_node (struct cgraph_node *node) ipa_detect_param_modifications (node); } -/* Recompute all local information since node might've got new - direct calls after cloning. */ -static void -ipcp_update_cloned_node (struct cgraph_node *new_node) -{ - /* We might've introduced new direct calls. */ - push_cfun (DECL_STRUCT_FUNCTION (new_node->decl)); - current_function_decl = new_node->decl; - rebuild_cgraph_edges (); - - /* Indirect inlinng rely on fact that we've already analyzed - the body.. */ - if (flag_indirect_inlining) - { - struct cgraph_edge *cs; - - ipcp_analyze_node (new_node); - - for (cs = new_node->callees; cs; cs = cs->next_callee) - { - ipa_count_arguments (cs); - ipa_compute_jump_functions (cs); - } - } - pop_cfun (); - current_function_decl = NULL; -} - /* Return scale for NODE. */ static inline gcov_type ipcp_get_node_scale (struct cgraph_node *node) @@ -756,98 +728,6 @@ ipcp_print_call_profile_counts (FILE * f) } } -/* Print all counts and probabilities of cfg edges of all functions. */ -static void -ipcp_print_edge_profiles (FILE * f) -{ - struct cgraph_node *node; - basic_block bb; - edge_iterator ei; - edge e; - - for (node = cgraph_nodes; node; node = node->next) - { - fprintf (f, "function %s: \n", cgraph_node_name (node)); - if (node->analyzed) - { - bb = - ENTRY_BLOCK_PTR_FOR_FUNCTION (DECL_STRUCT_FUNCTION (node->decl)); - fprintf (f, "ENTRY: "); - fprintf (f, " " HOST_WIDE_INT_PRINT_DEC - " %d\n", (HOST_WIDE_INT) bb->count, bb->frequency); - - if (bb->succs) - FOR_EACH_EDGE (e, ei, bb->succs) - { - if (e->dest == - EXIT_BLOCK_PTR_FOR_FUNCTION (DECL_STRUCT_FUNCTION - (node->decl))) - fprintf (f, "edge ENTRY -> EXIT, Count"); - else - fprintf (f, "edge ENTRY -> %d, Count", e->dest->index); - fprintf (f, " " HOST_WIDE_INT_PRINT_DEC - " Prob %d\n", (HOST_WIDE_INT) e->count, - e->probability); - } - FOR_EACH_BB_FN (bb, DECL_STRUCT_FUNCTION (node->decl)) - { - fprintf (f, "bb[%d]: ", bb->index); - fprintf (f, " " HOST_WIDE_INT_PRINT_DEC - " %d\n", (HOST_WIDE_INT) bb->count, bb->frequency); - FOR_EACH_EDGE (e, ei, bb->succs) - { - if (e->dest == - EXIT_BLOCK_PTR_FOR_FUNCTION (DECL_STRUCT_FUNCTION - (node->decl))) - fprintf (f, "edge %d -> EXIT, Count", e->src->index); - else - fprintf (f, "edge %d -> %d, Count", e->src->index, - e->dest->index); - fprintf (f, " " HOST_WIDE_INT_PRINT_DEC " Prob %d\n", - (HOST_WIDE_INT) e->count, e->probability); - } - } - } - } -} - -/* Print counts and frequencies for all basic blocks of all functions. */ -static void -ipcp_print_bb_profiles (FILE * f) -{ - basic_block bb; - struct cgraph_node *node; - - for (node = cgraph_nodes; node; node = node->next) - { - fprintf (f, "function %s: \n", cgraph_node_name (node)); - if (node->analyzed) - { - bb = - ENTRY_BLOCK_PTR_FOR_FUNCTION (DECL_STRUCT_FUNCTION (node->decl)); - fprintf (f, "ENTRY: Count"); - fprintf (f, " " HOST_WIDE_INT_PRINT_DEC - " Frequency %d\n", (HOST_WIDE_INT) bb->count, - bb->frequency); - - FOR_EACH_BB_FN (bb, DECL_STRUCT_FUNCTION (node->decl)) - { - fprintf (f, "bb[%d]: Count", bb->index); - fprintf (f, " " HOST_WIDE_INT_PRINT_DEC - " Frequency %d\n", (HOST_WIDE_INT) bb->count, - bb->frequency); - } - bb = - EXIT_BLOCK_PTR_FOR_FUNCTION (DECL_STRUCT_FUNCTION (node->decl)); - fprintf (f, "EXIT: Count"); - fprintf (f, " " HOST_WIDE_INT_PRINT_DEC - " Frequency %d\n", (HOST_WIDE_INT) bb->count, - bb->frequency); - - } - } -} - /* Print profile info for all functions. */ static void ipcp_print_profile_data (FILE * f) @@ -856,10 +736,6 @@ ipcp_print_profile_data (FILE * f) ipcp_print_func_profile_counts (f); fprintf (f, "\nCS COUNTS stage:\n"); ipcp_print_call_profile_counts (f); - fprintf (f, "\nBB COUNTS and FREQUENCIES :\n"); - ipcp_print_bb_profiles (f); - fprintf (f, "\nCFG EDGES COUNTS and PROBABILITIES :\n"); - ipcp_print_edge_profiles (f); } /* Build and initialize ipa_replace_map struct according to LAT. This struct is @@ -872,7 +748,7 @@ ipcp_create_replace_map (tree parm_tree, struct ipcp_lattice *lat) struct ipa_replace_map *replace_map; tree const_val; - replace_map = XCNEW (struct ipa_replace_map); + replace_map = GGC_NEW (struct ipa_replace_map); const_val = build_const_val (lat, TREE_TYPE (parm_tree)); if (dump_file) { @@ -959,25 +835,7 @@ ipcp_update_callgraph (void) for (cs = node->callers; cs; cs = next) { next = cs->next_caller; - if (ipcp_node_is_clone (cs->caller) || !ipcp_need_redirect_p (cs)) - { - gimple new_stmt; - gimple_stmt_iterator gsi; - - current_function_decl = cs->caller->decl; - push_cfun (DECL_STRUCT_FUNCTION (cs->caller->decl)); - - new_stmt = gimple_call_copy_skip_args (cs->call_stmt, - args_to_skip); - if (gimple_vdef (new_stmt)) - SSA_NAME_DEF_STMT (gimple_vdef (new_stmt)) = new_stmt; - gsi = gsi_for_stmt (cs->call_stmt); - gsi_replace (&gsi, new_stmt, true); - cgraph_set_call_stmt (cs, new_stmt); - pop_cfun (); - current_function_decl = NULL; - } - else + if (!ipcp_node_is_clone (cs->caller) && ipcp_need_redirect_p (cs)) { cgraph_redirect_edge_callee (cs, orig_node); gimple_call_set_fndecl (cs->call_stmt, orig_node->decl); @@ -986,29 +844,6 @@ ipcp_update_callgraph (void) } } -/* Update all cfg basic blocks in NODE according to SCALE. */ -static void -ipcp_update_bb_counts (struct cgraph_node *node, gcov_type scale) -{ - basic_block bb; - - FOR_ALL_BB_FN (bb, DECL_STRUCT_FUNCTION (node->decl)) - bb->count = bb->count * scale / REG_BR_PROB_BASE; -} - -/* Update all cfg edges in NODE according to SCALE. */ -static void -ipcp_update_edges_counts (struct cgraph_node *node, gcov_type scale) -{ - basic_block bb; - edge_iterator ei; - edge e; - - FOR_ALL_BB_FN (bb, DECL_STRUCT_FUNCTION (node->decl)) - FOR_EACH_EDGE (e, ei, bb->succs) - e->count = e->count * scale / REG_BR_PROB_BASE; -} - /* Update profiling info for versioned functions and the functions they were versioned from. */ static void @@ -1032,10 +867,6 @@ ipcp_update_profiling (void) cs->count = cs->count * scale / REG_BR_PROB_BASE; for (cs = orig_node->callees; cs; cs = cs->next_callee) cs->count = cs->count * scale_complement / REG_BR_PROB_BASE; - ipcp_update_bb_counts (node, scale); - ipcp_update_bb_counts (orig_node, scale_complement); - ipcp_update_edges_counts (node, scale); - ipcp_update_edges_counts (orig_node, scale_complement); } } } @@ -1160,13 +991,13 @@ ipcp_insert_stage (void) struct cgraph_node *node, *node1 = NULL; int i; VEC (cgraph_edge_p, heap) * redirect_callers; - varray_type replace_trees; + VEC (ipa_replace_map_p,gc)* replace_trees; int node_callers, count; tree parm_tree; struct ipa_replace_map *replace_param; fibheap_t heap; - long overall_insns = 0, new_insns = 0; - long max_new_insns; + long overall_size = 0, new_size = 0; + long max_new_size; ipa_check_create_node_params (); ipa_check_create_edge_args (); @@ -1180,13 +1011,13 @@ ipcp_insert_stage (void) { if (node->count > max_count) max_count = node->count; - overall_insns += node->local.inline_summary.self_insns; + overall_size += node->local.inline_summary.self_insns; } - max_new_insns = overall_insns; - if (max_new_insns < PARAM_VALUE (PARAM_LARGE_UNIT_INSNS)) - max_new_insns = PARAM_VALUE (PARAM_LARGE_UNIT_INSNS); - max_new_insns = max_new_insns * PARAM_VALUE (PARAM_IPCP_UNIT_GROWTH) / 100 + 1; + max_new_size = overall_size; + if (max_new_size < PARAM_VALUE (PARAM_LARGE_UNIT_INSNS)) + max_new_size = PARAM_VALUE (PARAM_LARGE_UNIT_INSNS); + max_new_size = max_new_size * PARAM_VALUE (PARAM_IPCP_UNIT_GROWTH) / 100 + 1; /* First collect all functions we proved to have constant arguments to heap. */ heap = fibheap_new (); @@ -1220,7 +1051,7 @@ ipcp_insert_stage (void) growth = ipcp_estimate_growth (node); - if (new_insns + growth > max_new_insns) + if (new_size + growth > max_new_size) break; if (growth && optimize_function_for_size_p (DECL_STRUCT_FUNCTION (node->decl))) @@ -1230,7 +1061,7 @@ ipcp_insert_stage (void) continue; } - new_insns += growth; + new_size += growth; /* Look if original function becomes dead after clonning. */ for (cs = node->callers; cs != NULL; cs = cs->next_caller) @@ -1242,9 +1073,8 @@ ipcp_insert_stage (void) info = IPA_NODE_REF (node); count = ipa_get_param_count (info); - VARRAY_GENERIC_PTR_INIT (replace_trees, ipcp_const_param_count (node), - "replace_trees"); - args_to_skip = BITMAP_ALLOC (NULL); + replace_trees = VEC_alloc (ipa_replace_map_p, gc, 1); + args_to_skip = BITMAP_GGC_ALLOC (); for (i = 0; i < count; i++) { struct ipcp_lattice *lat = ipcp_get_lattice (info, i); @@ -1263,7 +1093,7 @@ ipcp_insert_stage (void) { replace_param = ipcp_create_replace_map (parm_tree, lat); - VARRAY_PUSH_GENERIC_PTR (replace_trees, replace_param); + VEC_safe_push (ipa_replace_map_p, gc, replace_trees, replace_param); bitmap_set_bit (args_to_skip, i); } } @@ -1279,20 +1109,20 @@ ipcp_insert_stage (void) /* Redirecting all the callers of the node to the new versioned node. */ node1 = - cgraph_function_versioning (node, redirect_callers, replace_trees, - args_to_skip); - BITMAP_FREE (args_to_skip); + cgraph_create_virtual_clone (node, redirect_callers, replace_trees, + args_to_skip); + args_to_skip = NULL; VEC_free (cgraph_edge_p, heap, redirect_callers); - VARRAY_CLEAR (replace_trees); + replace_trees = NULL; + if (node1 == NULL) continue; if (dump_file) fprintf (dump_file, "versioned function %s with growth %i, overall %i\n", - cgraph_node_name (node), (int)growth, (int)new_insns); + cgraph_node_name (node), (int)growth, (int)new_size); ipcp_init_cloned_node (node, node1); - /* We've possibly introduced direct calls. */ - ipcp_update_cloned_node (node1); + /* TODO: We can use indirect inlning info to produce new calls. */ if (dump_file) dump_function_to_file (node1->decl, dump_file, dump_flags); diff --git a/gcc/ipa-inline.c b/gcc/ipa-inline.c index 15612193ace..99640bfdb55 100644 --- a/gcc/ipa-inline.c +++ b/gcc/ipa-inline.c @@ -433,7 +433,7 @@ cgraph_default_inline_p (struct cgraph_node *n, cgraph_inline_failed_t *reason) return false; } - if (!DECL_STRUCT_FUNCTION (decl)->cfg) + if (!n->analyzed) { if (reason) *reason = CIF_BODY_NOT_AVAILABLE; diff --git a/gcc/ipa-prop.c b/gcc/ipa-prop.c index 88047e476b3..f4fa37d5a05 100644 --- a/gcc/ipa-prop.c +++ b/gcc/ipa-prop.c @@ -653,7 +653,7 @@ ipa_note_param_call (struct ipa_node_params *info, int formal_id, note->formal_id = formal_id; note->stmt = stmt; note->count = bb->count; - note->frequency = compute_call_stmt_bb_frequency (bb); + note->frequency = compute_call_stmt_bb_frequency (current_function_decl, bb); note->next = info->param_calls; info->param_calls = note; diff --git a/gcc/ipa-prop.h b/gcc/ipa-prop.h index 5943a2af6f4..c4c1ccc162a 100644 --- a/gcc/ipa-prop.h +++ b/gcc/ipa-prop.h @@ -99,20 +99,6 @@ struct ipcp_lattice tree constant; }; -/* Represent which DECL tree (or reference to such tree) - will be replaced by another tree while versioning. */ -struct ipa_replace_map -{ - /* The tree that will be replaced. */ - tree old_tree; - /* The new (replacing) tree. */ - tree new_tree; - /* True when a substitution should be done, false otherwise. */ - bool replace_p; - /* True when we replace a reference to old_tree. */ - bool ref_p; -}; - /* Each instance of the following structure describes a statement that calls a function parameter. Those referring to statements within the same function are linked in a list. */ diff --git a/gcc/ipa-struct-reorg.c b/gcc/ipa-struct-reorg.c index 5b1670e0de1..6468d77f06a 100644 --- a/gcc/ipa-struct-reorg.c +++ b/gcc/ipa-struct-reorg.c @@ -3641,7 +3641,7 @@ do_reorg_1 (void) bitmap_obstack_initialize (NULL); for (node = cgraph_nodes; node; node = node->next) - if (node->analyzed && node->decl && !node->next_clone) + if (node->analyzed && node->decl) { push_cfun (DECL_STRUCT_FUNCTION (node->decl)); current_function_decl = node->decl; @@ -3809,8 +3809,7 @@ collect_data_accesses (void) { struct function *func = DECL_STRUCT_FUNCTION (c_node->decl); - if (!c_node->next_clone) - collect_accesses_in_func (func); + collect_accesses_in_func (func); exclude_alloc_and_field_accs (c_node); } } diff --git a/gcc/ipa.c b/gcc/ipa.c index b486b93d34c..686ca9e1348 100644 --- a/gcc/ipa.c +++ b/gcc/ipa.c @@ -25,6 +25,7 @@ along with GCC; see the file COPYING3. If not see #include "cgraph.h" #include "tree-pass.h" #include "timevar.h" +#include "gimple.h" #include "ggc.h" /* Fill array order with all nodes with output flag set in the reverse @@ -143,6 +144,12 @@ cgraph_remove_unreachable_nodes (bool before_inlining_p, FILE *file) e->callee->aux = first; first = e->callee; } + while (node->clone_of && !node->clone_of->aux && !gimple_has_body_p (node->decl)) + { + node = node->clone_of; + node->aux = first; + first = node; + } } /* Remove unreachable nodes. Extern inline functions need special care; @@ -168,25 +175,29 @@ cgraph_remove_unreachable_nodes (bool before_inlining_p, FILE *file) { struct cgraph_edge *e; + /* See if there is reachable caller. */ for (e = node->callers; e; e = e->next_caller) if (e->caller->aux) break; + + /* If so, we need to keep node in the callgraph. */ if (e || node->needed) { struct cgraph_node *clone; - for (clone = node->next_clone; clone; - clone = clone->next_clone) + /* If there are still clones, we must keep body around. + Otherwise we can just remove the body but keep the clone. */ + for (clone = node->clones; clone; + clone = clone->next_sibling_clone) if (clone->aux) break; if (!clone) { cgraph_release_function_body (node); + cgraph_node_remove_callees (node); node->analyzed = false; + node->local.inlinable = false; } - cgraph_node_remove_callees (node); - node->analyzed = false; - node->local.inlinable = false; } else cgraph_remove_node (node); @@ -195,7 +206,18 @@ cgraph_remove_unreachable_nodes (bool before_inlining_p, FILE *file) } } for (node = cgraph_nodes; node; node = node->next) - node->aux = NULL; + { + /* Inline clones might be kept around so their materializing allows further + cloning. If the function the clone is inlined into is removed, we need + to turn it into normal cone. */ + if (node->global.inlined_to + && !node->callers) + { + gcc_assert (node->clones); + node->global.inlined_to = false; + } + node->aux = NULL; + } #ifdef ENABLE_CHECKING verify_cgraph (); #endif diff --git a/gcc/passes.c b/gcc/passes.c index 312dfa9db72..7a39ac70bc1 100644 --- a/gcc/passes.c +++ b/gcc/passes.c @@ -846,7 +846,7 @@ do_per_function (void (*callback) (void *data), void *data) { struct cgraph_node *node; for (node = cgraph_nodes; node; node = node->next) - if (node->analyzed) + if (node->analyzed && gimple_has_body_p (node->decl)) { push_cfun (DECL_STRUCT_FUNCTION (node->decl)); current_function_decl = node->decl; diff --git a/gcc/tree-cfg.c b/gcc/tree-cfg.c index b5c67cd8a08..077f9d602d7 100644 --- a/gcc/tree-cfg.c +++ b/gcc/tree-cfg.c @@ -7074,7 +7074,7 @@ struct gimple_opt_pass pass_split_crit_edges = PROP_no_crit_edges, /* properties_provided */ 0, /* properties_destroyed */ 0, /* todo_flags_start */ - TODO_dump_func /* todo_flags_finish */ + TODO_dump_func | TODO_verify_flow /* todo_flags_finish */ } }; diff --git a/gcc/tree-inline.c b/gcc/tree-inline.c index b134ae53d15..0a383768b04 100644 --- a/gcc/tree-inline.c +++ b/gcc/tree-inline.c @@ -32,7 +32,6 @@ along with GCC; see the file COPYING3. If not see #include "params.h" #include "input.h" #include "insn-config.h" -#include "varray.h" #include "hashtab.h" #include "langhooks.h" #include "basic-block.h" @@ -1393,6 +1392,8 @@ copy_bb (copy_body_data *id, basic_block bb, int frequency_scale, need to process all of them. */ do { + tree fn; + stmt = gsi_stmt (copy_gsi); if (is_gimple_call (stmt) && gimple_call_va_arg_pack_p (stmt) @@ -1481,34 +1482,24 @@ copy_bb (copy_body_data *id, basic_block bb, int frequency_scale, callgraph edges and update or duplicate them. */ if (is_gimple_call (stmt)) { - struct cgraph_node *node; - struct cgraph_edge *edge; + struct cgraph_edge *edge = cgraph_edge (id->src_node, orig_stmt); int flags; switch (id->transform_call_graph_edges) { case CB_CGE_DUPLICATE: - edge = cgraph_edge (id->src_node, orig_stmt); - if (edge) + if (edge) cgraph_clone_edge (edge, id->dst_node, stmt, REG_BR_PROB_BASE, 1, edge->frequency, true); break; case CB_CGE_MOVE_CLONES: - for (node = id->dst_node->next_clone; - node; - node = node->next_clone) - { - edge = cgraph_edge (node, orig_stmt); - if (edge) - cgraph_set_call_stmt (edge, stmt); - } - /* FALLTHRU */ + cgraph_set_call_stmt_including_clones (id->dst_node, orig_stmt, stmt); + break; case CB_CGE_MOVE: - edge = cgraph_edge (id->dst_node, orig_stmt); - if (edge) + if (edge) cgraph_set_call_stmt (edge, stmt); break; @@ -1516,6 +1507,36 @@ copy_bb (copy_body_data *id, basic_block bb, int frequency_scale, gcc_unreachable (); } + /* Constant propagation on argument done during inlining + may create new direct call. Produce an edge for it. */ + if (!edge && is_gimple_call (stmt) + && (fn = gimple_call_fndecl (stmt)) != NULL + && !cgraph_edge (id->dst_node, stmt)) + { + struct cgraph_node *dest = cgraph_node (fn); + + /* We have missing edge in the callgraph. This can happen in one case + where previous inlining turned indirect call into direct call by + constant propagating arguments. In all other cases we hit a bug + (incorrect node sharing is most common reason for missing edges. */ + gcc_assert (dest->needed || !dest->analyzed); + if (id->transform_call_graph_edges == CB_CGE_MOVE_CLONES) + cgraph_create_edge_including_clones (id->dst_node, dest, stmt, + bb->count, CGRAPH_FREQ_BASE, + bb->loop_depth, + CIF_ORIGINALLY_INDIRECT_CALL); + else + cgraph_create_edge (id->dst_node, dest, stmt, + bb->count, CGRAPH_FREQ_BASE, + bb->loop_depth)->inline_failed + = CIF_ORIGINALLY_INDIRECT_CALL; + if (dump_file) + { + fprintf (dump_file, "Created new direct edge to %s", + cgraph_node_name (dest)); + } + } + flags = gimple_call_flags (stmt); if (flags & ECF_MAY_BE_ALLOCA) @@ -3221,29 +3242,6 @@ expand_call_inline (basic_block bb, gimple stmt, copy_body_data *id) cg_edge = cgraph_edge (id->dst_node, stmt); - /* Constant propagation on argument done during previous inlining - may create new direct call. Produce an edge for it. */ - if (!cg_edge) - { - struct cgraph_node *dest = cgraph_node (fn); - - /* We have missing edge in the callgraph. This can happen in one case - where previous inlining turned indirect call into direct call by - constant propagating arguments. In all other cases we hit a bug - (incorrect node sharing is most common reason for missing edges. */ - gcc_assert (dest->needed); - cgraph_create_edge (id->dst_node, dest, stmt, - bb->count, CGRAPH_FREQ_BASE, - bb->loop_depth)->inline_failed - = CIF_ORIGINALLY_INDIRECT_CALL; - if (dump_file) - { - fprintf (dump_file, "Created new direct edge to %s", - cgraph_node_name (dest)); - } - goto egress; - } - /* Don't try to inline functions that are not well-suited to inlining. */ if (!cgraph_inline_p (cg_edge, &reason)) @@ -4281,27 +4279,98 @@ tree_versionable_function_p (tree fndecl) return true; } -/* Create a new name for omp child function. Returns an identifier. */ - -static GTY(()) unsigned int clone_fn_id_num; +/* Delete all unreachable basic blocks and update callgraph. + Doing so is somewhat nontrivial because we need to update all clones and + remove inline function that become unreachable. */ -static tree -clone_function_name (tree decl) +static bool +delete_unreachable_blocks_update_callgraph (copy_body_data *id) { - tree name = DECL_ASSEMBLER_NAME (decl); - size_t len = IDENTIFIER_LENGTH (name); - char *tmp_name, *prefix; - - prefix = XALLOCAVEC (char, len + strlen ("_clone") + 1); - memcpy (prefix, IDENTIFIER_POINTER (name), len); - strcpy (prefix + len, "_clone"); -#ifndef NO_DOT_IN_LABEL - prefix[len] = '.'; -#elif !defined NO_DOLLAR_IN_LABEL - prefix[len] = '$'; + bool changed = false; + basic_block b, next_bb; + + find_unreachable_blocks (); + + /* Delete all unreachable basic blocks. */ + + for (b = ENTRY_BLOCK_PTR->next_bb; b != EXIT_BLOCK_PTR; b = next_bb) + { + next_bb = b->next_bb; + + if (!(b->flags & BB_REACHABLE)) + { + gimple_stmt_iterator bsi; + + for (bsi = gsi_start_bb (b); !gsi_end_p (bsi); gsi_next (&bsi)) + if (gimple_code (gsi_stmt (bsi)) == GIMPLE_CALL) + { + struct cgraph_edge *e; + struct cgraph_node *node; + + if ((e = cgraph_edge (id->dst_node, gsi_stmt (bsi))) != NULL) + { + if (!e->inline_failed) + cgraph_remove_node_and_inline_clones (e->callee); + else + cgraph_remove_edge (e); + } + if (id->transform_call_graph_edges == CB_CGE_MOVE_CLONES + && id->dst_node->clones) + for (node = id->dst_node->clones; node != id->dst_node;) + { + if ((e = cgraph_edge (node, gsi_stmt (bsi))) != NULL) + { + if (!e->inline_failed) + cgraph_remove_node_and_inline_clones (e->callee); + else + cgraph_remove_edge (e); + } + + if (node->clones) + node = node->clones; + else if (node->next_sibling_clone) + node = node->next_sibling_clone; + else + { + while (node != id->dst_node && !node->next_sibling_clone) + node = node->clone_of; + if (node != id->dst_node) + node = node->next_sibling_clone; + } + } + } + delete_basic_block (b); + changed = true; + } + } + + if (changed) + tidy_fallthru_edges (); +#ifdef ENABLE_CHECKING0 + verify_cgraph_node (id->dst_node); + if (id->transform_call_graph_edges == CB_CGE_MOVE_CLONES + && id->dst_node->clones) + { + struct cgraph_node *node; + for (node = id->dst_node->clones; node != id->dst_node;) + { + verify_cgraph_node (node); + + if (node->clones) + node = node->clones; + else if (node->next_sibling_clone) + node = node->next_sibling_clone; + else + { + while (node != id->dst_node && !node->next_sibling_clone) + node = node->clone_of; + if (node != id->dst_node) + node = node->next_sibling_clone; + } + } + } #endif - ASM_FORMAT_PRIVATE_NAME (tmp_name, prefix, clone_fn_id_num++); - return get_identifier (tmp_name); + return changed; } /* Create a copy of a function's tree. @@ -4313,7 +4382,7 @@ clone_function_name (tree decl) trees. If UPDATE_CLONES is set, the call_stmt fields of edges of clones of the function will be updated. */ void -tree_function_versioning (tree old_decl, tree new_decl, varray_type tree_map, +tree_function_versioning (tree old_decl, tree new_decl, VEC(ipa_replace_map_p,gc)* tree_map, bool update_clones, bitmap args_to_skip) { struct cgraph_node *old_version_node; @@ -4349,13 +4418,7 @@ tree_function_versioning (tree old_decl, tree new_decl, varray_type tree_map, memset (&id, 0, sizeof (id)); /* Generate a new name for the new version. */ - if (!update_clones) - { - DECL_NAME (new_decl) = clone_function_name (old_decl); - SET_DECL_ASSEMBLER_NAME (new_decl, DECL_NAME (new_decl)); - SET_DECL_RTL (new_decl, NULL_RTX); - id.statements_to_fold = pointer_set_create (); - } + id.statements_to_fold = pointer_set_create (); id.decl_map = pointer_map_create (); id.src_fn = old_decl; @@ -4388,11 +4451,10 @@ tree_function_versioning (tree old_decl, tree new_decl, varray_type tree_map, /* If there's a tree_map, prepare for substitution. */ if (tree_map) - for (i = 0; i < VARRAY_ACTIVE_SIZE (tree_map); i++) + for (i = 0; i < VEC_length (ipa_replace_map_p, tree_map); i++) { gimple init; - replace_info - = (struct ipa_replace_map *) VARRAY_GENERIC_PTR (tree_map, i); + replace_info = VEC_index (ipa_replace_map_p, tree_map, i); if (replace_info->replace_p) { tree op = replace_info->new_tree; @@ -4431,6 +4493,7 @@ tree_function_versioning (tree old_decl, tree new_decl, varray_type tree_map, number_blocks (id.dst_fn); declare_inline_vars (DECL_INITIAL (new_decl), vars); + if (DECL_STRUCT_FUNCTION (old_decl)->local_decls != NULL_TREE) /* Add local vars. */ for (t_step = DECL_STRUCT_FUNCTION (old_decl)->local_decls; @@ -4469,14 +4532,15 @@ tree_function_versioning (tree old_decl, tree new_decl, varray_type tree_map, pointer_map_destroy (id.decl_map); free_dominance_info (CDI_DOMINATORS); free_dominance_info (CDI_POST_DOMINATORS); - if (!update_clones) - { - fold_marked_statements (0, id.statements_to_fold); - pointer_set_destroy (id.statements_to_fold); - fold_cond_expr_cond (); - delete_unreachable_blocks (); - update_ssa (TODO_update_ssa); - } + + fold_marked_statements (0, id.statements_to_fold); + pointer_set_destroy (id.statements_to_fold); + fold_cond_expr_cond (); + delete_unreachable_blocks_update_callgraph (&id); + update_ssa (TODO_update_ssa); + free_dominance_info (CDI_DOMINATORS); + free_dominance_info (CDI_POST_DOMINATORS); + VEC_free (gimple, heap, init_stmts); pop_cfun (); current_function_decl = old_current_function_decl; @@ -4544,5 +4608,3 @@ tree_can_inline_p (tree caller, tree callee) /* Allow the backend to decide if inlining is ok. */ return targetm.target_option.can_inline_p (caller, callee); } - -#include "gt-tree-inline.h" diff --git a/gcc/tree-inline.h b/gcc/tree-inline.h index 2a4e38b1657..d55f84a14d4 100644 --- a/gcc/tree-inline.h +++ b/gcc/tree-inline.h @@ -21,7 +21,6 @@ along with GCC; see the file COPYING3. If not see #ifndef GCC_TREE_INLINE_H #define GCC_TREE_INLINE_H -#include "varray.h" #include "pointer-set.h" @@ -156,7 +155,6 @@ int estimate_num_insns (gimple, eni_weights *); int estimate_num_insns_fn (tree, eni_weights *); int count_insns_seq (gimple_seq, eni_weights *); bool tree_versionable_function_p (tree); -void tree_function_versioning (tree, tree, varray_type, bool, bitmap); bool tree_can_inline_p (tree, tree); extern gimple_seq remap_gimple_seq (gimple_seq, copy_body_data *); |