Reduce memory usage for storing LTO decl resolutions

With a LTO build of a large project (>11k subfiles incrementially linked)
storing the LTO resolutions took over 0.5GB memory:

lto/lto.c:1087 (lto_resolution_read)                      0: 0.0%  540398500           15903: 0.0%

The reason is that the declaration indexes are quite sparse, but every subfile
got a full continuous vector for them. Since there are so many of them the
many vectors add up.

This patch instead stores the resolutions initially in a compact (index, resolution)
format. This is only expanded into a sparse vector for fast lookup when
the subfile is actually read, but then immediately freed. This means only one
vector is allocated at a time.

This brings the overhead for this down to less than 3MB for the test case:

lto/lto.c:1087 (lto_resolution_read)                      0: 0.0%    2821456           42186: 0.0%

gcc/:

2012-09-06  Andi Kleen  <ak@linux.intel.com>

	* gcc/lto-streamer.h (res_pair): Add.
	(lto_file_decl_data): Replace resolutions with respairs.
	Add max_index.
	* gcc/lto/lto.c (lto_resolution_read): Remove max_index.  Add rp.
	Initialize respairs.
	(lto_file_finalize): Set up resolutions vector lazily from respairs.

git-svn-id: svn+ssh://gcc.gnu.org/svn/gcc/trunk@191051 138bc75d-0d04-0410-961f-82ee72b054a4

1 files changed, 22 insertions, 9 deletions

diff --git a/gcc/lto/lto.c b/gcc/lto/lto.c
index bd91c391fd1..5da54124f2d 100644
--- a/gcc/lto/lto.c
+++ b/gcc/lto/lto.c
@@ -1012,7 +1012,6 @@ lto_resolution_read (splay_tree file_ids, FILE *resolution, lto_file *file)
   unsigned int num_symbols;
   unsigned int i;
   struct lto_file_decl_data *file_data;
-  unsigned max_index = 0;
   splay_tree_node nd = NULL; 
 
   if (!resolution)
@@ -1054,13 +1053,12 @@ lto_resolution_read (splay_tree file_ids, FILE *resolution, lto_file *file)
       unsigned int j;
       unsigned int lto_resolution_str_len =
 	sizeof (lto_resolution_str) / sizeof (char *);
+      res_pair rp;
 
       t = fscanf (resolution, "%u " HOST_WIDE_INT_PRINT_HEX_PURE " %26s %*[^\n]\n", 
 		  &index, &id, r_str);
       if (t != 3)
         internal_error ("invalid line in the resolution file");
-      if (index > max_index)
-	max_index = index;
 
       for (j = 0; j < lto_resolution_str_len; j++)
 	{
@@ -1082,11 +1080,13 @@ lto_resolution_read (splay_tree file_ids, FILE *resolution, lto_file *file)
 	}
 
       file_data = (struct lto_file_decl_data *)nd->value;
-      VEC_safe_grow_cleared (ld_plugin_symbol_resolution_t, heap, 
-			     file_data->resolutions,
-			     max_index + 1);
-      VEC_replace (ld_plugin_symbol_resolution_t, 
-		   file_data->resolutions, index, r);
+      /* The indexes are very sparse. To save memory save them in a compact
+         format that is only unpacked later when the subfile is processed. */
+      rp.res = r;
+      rp.index = index;
+      VEC_safe_push (res_pair, heap, file_data->respairs, rp);
+      if (file_data->max_index < index)
+        file_data->max_index = index;
     }
 }
 
@@ -1166,6 +1166,18 @@ lto_file_finalize (struct lto_file_decl_data *file_data, lto_file *file)
 {
   const char *data;
   size_t len;
+  VEC(ld_plugin_symbol_resolution_t,heap) *resolutions = NULL;
+  int i;
+  res_pair *rp;
+
+  /* Create vector for fast access of resolution. We do this lazily
+     to save memory. */ 
+  VEC_safe_grow_cleared (ld_plugin_symbol_resolution_t, heap, 
+                            resolutions,
+                            file_data->max_index + 1);
+  for (i = 0; VEC_iterate (res_pair, file_data->respairs, i, rp); i++)
+    VEC_replace (ld_plugin_symbol_resolution_t, resolutions, rp->index, rp->res);
+  VEC_free (res_pair, heap, file_data->respairs);
 
   file_data->renaming_hash_table = lto_create_renaming_table ();
   file_data->file_name = file->filename;
@@ -1175,7 +1187,8 @@ lto_file_finalize (struct lto_file_decl_data *file_data, lto_file *file)
       internal_error ("cannot read LTO decls from %s", file_data->file_name);
       return;
     }
-  lto_read_decls (file_data, data, file_data->resolutions);
+  /* Frees resolutions */
+  lto_read_decls (file_data, data, resolutions);
   lto_free_section_data (file_data, LTO_section_decls, NULL, data, len);
 }