On Wed, 28 Aug 2013, Jan Hubicka wrote:
> Hi,
> this patch saves almost 1Gb of GGC reachable memory for firefox.
> Currently we point to all streamed in trees from global decl in state and from
> function specific states. The states are big and they dangle pointers.
> This patch gets rid of states we do not need.
>
> We run GGC collect just once during usual WPA compilation. Currently
> it is just after tree loading that introduce very little garbage. This
> patch moves it after unreachable code removal when freeing happen.
>
> Still we do not suceed in much of reuse in between GGC and heap, so peak
> memory
> use in TOP reduces by only 300MB. It may be interesting to allow random
> access
> into per-function decl in states and read them only after decl merging (and
> possibly unreachable function removal).
>
> Bootstrapped/regtested x86_64-linux, OK?
>
> * lto-symtab.c (lto_cgraph_replace_node): Free decl_in_state.
> * cgraph.c (cgraph_release_function_body): Free decl_in_state.
> * lto-section-in.c (lto_free_function_in_decl_state): New function.
> (lto_free_function_in_decl_state_for_node): New function.
>
> * lto.c (read_cgraph_and_symbols): Remove ggc_collect;
> clear section node; add comment why we do not collect.
> Index: lto-symtab.c
> ===================================================================
> --- lto-symtab.c (revision 202047)
> +++ lto-symtab.c (working copy)
> @@ -80,6 +80,8 @@ lto_cgraph_replace_node (struct cgraph_n
> /* Redirect incomming references. */
> ipa_clone_referring ((symtab_node)prevailing_node, &node->symbol.ref_list);
>
> + lto_free_function_in_decl_state_for_node ((symtab_node)node);
> +
> if (node->symbol.decl != prevailing_node->symbol.decl)
> cgraph_release_function_body (node);
>
> Index: cgraph.c
> ===================================================================
> --- cgraph.c (revision 202047)
> +++ cgraph.c (working copy)
> @@ -1663,6 +1663,8 @@ cgraph_release_function_body (struct cgr
> if (!node->used_as_abstract_origin && DECL_INITIAL (node->symbol.decl))
> DECL_INITIAL (node->symbol.decl) = error_mark_node;
> release_function_body (node->symbol.decl);
> + if (node->symbol.lto_file_data)
> + lto_free_function_in_decl_state_for_node ((symtab_node) node);
> }
>
> /* Remove the node from cgraph. */
> Index: lto-section-in.c
> ===================================================================
> --- lto-section-in.c (revision 202047)
> +++ lto-section-in.c (working copy)
> @@ -414,6 +414,41 @@ lto_get_function_in_decl_state (struct l
> return slot? ((struct lto_in_decl_state*) *slot) : NULL;
> }
>
> +/* Free decl_states. */
> +
> +void
> +lto_free_function_in_decl_state (struct lto_in_decl_state *state)
> +{
> + int i;
> + for (i = 0; i < LTO_N_DECL_STREAMS; i++)
> + ggc_free (state->streams[i].trees);
We likely also waste a lot of memory by means of GC overhead here.
Moving this array out of GC space would be better (we know the
exact size upfront) - should be doable with making lto_tree_ref_able
GTY((user)) and a custom marker.
> + ggc_free (state);
> +}
> +
> +/* Free decl_states associated with NODE. This makes it possible to furhter
> + release trees needed by the NODE's body. */
> +
> +void
> +lto_free_function_in_decl_state_for_node (symtab_node node)
> +{
> + struct lto_in_decl_state temp;
> + void **slot;
> +
> + if (!node->symbol.lto_file_data)
> + return;
> +
> + temp.fn_decl = node->symbol.decl;
> + slot = htab_find_slot (node->symbol.lto_file_data->function_decl_states,
> + &temp, NO_INSERT);
> + if (slot && *slot)
> + {
> + lto_free_function_in_decl_state ((struct lto_in_decl_state*) *slot);
> + htab_clear_slot (node->symbol.lto_file_data->function_decl_states,
> + slot);
> + }
> + node->symbol.lto_file_data = NULL;
> +}
> +
>
> /* Report read pass end of the section. */
>
> Index: ipa.c
> ===================================================================
> --- ipa.c (revision 202047)
> +++ ipa.c (working copy)
> @@ -873,6 +873,17 @@ function_and_variable_visibility (bool w
> segfault though. */
> symtab_dissolve_same_comdat_group_list ((symtab_node) node);
> }
> + if (node->symbol.externally_visible
> + && DECL_COMDAT (node->symbol.decl)
> + && comdat_can_be_unshared_p ((symtab_node) node))
> + {
> + if (dump_file
> + && DECL_VISIBILITY (node->symbol.decl) != VISIBILITY_HIDDEN)
> + fprintf (dump_file, "Promoting visibility to hidden: %s/%i\n",
> + cgraph_node_name (node), node->symbol.order);
> + DECL_VISIBILITY (node->symbol.decl) = VISIBILITY_HIDDEN;
> + DECL_VISIBILITY_SPECIFIED (node->symbol.decl) = true;
> + }
>
> if (node->thunk.thunk_p
> && TREE_PUBLIC (node->symbol.decl))
> @@ -980,6 +991,17 @@ function_and_variable_visibility (bool w
> symtab_dissolve_same_comdat_group_list ((symtab_node) vnode);
> vnode->symbol.resolution = LDPR_PREVAILING_DEF_IRONLY;
> }
> + if (vnode->symbol.externally_visible
> + && DECL_COMDAT (vnode->symbol.decl)
> + && comdat_can_be_unshared_p ((symtab_node) vnode))
> + {
> + if (dump_file
> + && DECL_VISIBILITY (vnode->symbol.decl) == VISIBILITY_HIDDEN)
> + fprintf (dump_file, "Promoting visibility to hidden: %s/%i\n",
> + varpool_node_name (vnode), vnode->symbol.order);
> + DECL_VISIBILITY (vnode->symbol.decl) = VISIBILITY_HIDDEN;
> + DECL_VISIBILITY_SPECIFIED (vnode->symbol.decl) = true;
> + }
> }
>
> if (dump_file)
Stray changes?
> Index: lto/lto.c
> ===================================================================
> --- lto/lto.c (revision 202047)
> +++ lto/lto.c (working copy)
> @@ -3429,7 +3429,6 @@ read_cgraph_and_symbols (unsigned nfiles
> tree_scc_hash.dispose ();
> obstack_free (&tree_scc_hash_obstack, NULL);
> free_gimple_type_tables ();
> - ggc_collect ();
>
> /* Set the hooks so that all of the ipa passes can read in their data. */
> lto_set_in_hooks (all_file_decl_data, get_section_data, free_section_data);
> @@ -3484,7 +3483,6 @@ read_cgraph_and_symbols (unsigned nfiles
> }
> htab_delete (tree_with_vars);
> tree_with_vars = NULL;
> - ggc_collect ();
Just leave the ggc_collect ()s there, they are free.
> timevar_pop (TV_IPA_LTO_DECL_MERGE);
> /* Each pass will set the appropriate timer. */
> @@ -3503,6 +3501,9 @@ read_cgraph_and_symbols (unsigned nfiles
> gcc_assert (all_file_decl_data[i]->symtab_node_encoder);
> lto_symtab_encoder_delete (all_file_decl_data[i]->symtab_node_encoder);
> all_file_decl_data[i]->symtab_node_encoder = NULL;
> + lto_free_function_in_decl_state
> (all_file_decl_data[i]->global_decl_state);
> + all_file_decl_data[i]->global_decl_state = NULL;
> + all_file_decl_data[i]->current_decl_state = NULL;
> }
>
> /* Finally merge the cgraph according to the decl merging decisions. */
> @@ -3513,7 +3514,12 @@ read_cgraph_and_symbols (unsigned nfiles
> dump_symtab (cgraph_dump_file);
> }
> lto_symtab_merge_symbols ();
> - ggc_collect ();
> +
> + /* Do not GGC collect here; streaming in should not produce garbage.
> + Be sure we first collect after merging symbols, setting up visibilities
> + and removing unreachable nodes. This will happen after whole program
> + visibility pass. This should release more memory back to the system
> + and possibly allow us to re-use it for heap. */
But it's not wrong to collect here, no? See above, it should be
mostly free.
Thanks,
Richard.
