Ignoring all of the other problems that partitioning might have, it does Just So Happen to work for the single most popular target. So let's see about keeping the pass limping along for a while longer.
The problem I'm attempting to solve here is the gross hack in convert_to_eh_region_ranges, which adds new basic blocks, badly, very late in the optimization pipeline. Indeed, we know that these are needed much earlier when we did the partitioning originally. Splitting the EH edge itself is not too difficult. All that's required is to generate a new landing pad. I dithered about how to fill in the new landing pad. One option is to duplicate the entire block containing the old landing pad. I worried that we might inadvertently copy a very large basic block this way. Another option is to generate a new landing pad from scratch, and branch to the post-landing pad. This requires that the post-landing pad be kept separate from the landing pad until now. It seemed easy enough to add the EDGE_PRESERVE bit to prevent blocks from being merged. And here is where things go downhill. DF generates the bb info for a block immediately. And by immediately, I mean *during* creation of the block. #0 df_bb_refs_collect (collection_rec=0x7fffffffda60, bb=0x7ffff1a4f3a8) #1 in df_bb_refs_record (bb_index=42, scan_insns=0 '\000') #2 in create_basic_block_structure (head=0x7ffff1a4ce58, ...) Which means that the bb_has_eh_pred test in df_bb_refs_collect will *always* fail, because no edges can have been added to the block yet. Which of course means we don't add the right artificial refs, which means we fail df_verify (much) later. Suggestions for something better than the df_finish_pass hack that I added at the end of partition_hot_cold_basic_blocks are very welcome. r~
diff --git a/gcc/basic-block.h b/gcc/basic-block.h index 29c1167..c03129c 100644 --- a/gcc/basic-block.h +++ b/gcc/basic-block.h @@ -65,31 +65,34 @@ DEF_VEC_P(edge); DEF_VEC_ALLOC_P(edge,gc); DEF_VEC_ALLOC_P(edge,heap); -#define EDGE_FALLTHRU 1 /* 'Straight line' flow */ -#define EDGE_ABNORMAL 2 /* Strange flow, like computed +/* Always update the table in cfg.c dump_edge_info. */ +#define EDGE_FALLTHRU 0x0001 /* 'Straight line' flow */ +#define EDGE_ABNORMAL 0x0002 /* Strange flow, like computed label, or eh */ -#define EDGE_ABNORMAL_CALL 4 /* Call with abnormal exit +#define EDGE_ABNORMAL_CALL 0x0004 /* Call with abnormal exit like an exception, or sibcall */ -#define EDGE_EH 8 /* Exception throw */ -#define EDGE_FAKE 16 /* Not a real edge (profile.c) */ -#define EDGE_DFS_BACK 32 /* A backwards edge */ -#define EDGE_CAN_FALLTHRU 64 /* Candidate for straight line +#define EDGE_EH 0x0008 /* Exception throw */ +#define EDGE_FAKE 0x0010 /* Not a real edge (profile.c) */ +#define EDGE_DFS_BACK 0x0020 /* A backwards edge */ +#define EDGE_CAN_FALLTHRU 0x0040 /* Candidate for straight line flow. */ -#define EDGE_IRREDUCIBLE_LOOP 128 /* Part of irreducible loop. */ -#define EDGE_SIBCALL 256 /* Edge from sibcall to exit. */ -#define EDGE_LOOP_EXIT 512 /* Exit of a loop. */ -#define EDGE_TRUE_VALUE 1024 /* Edge taken when controlling +#define EDGE_IRREDUCIBLE_LOOP 0x0080 /* Part of irreducible loop. */ +#define EDGE_SIBCALL 0x0100 /* Edge from sibcall to exit. */ +#define EDGE_LOOP_EXIT 0x0200 /* Exit of a loop. */ +#define EDGE_TRUE_VALUE 0x0400 /* Edge taken when controlling predicate is nonzero. */ -#define EDGE_FALSE_VALUE 2048 /* Edge taken when controlling +#define EDGE_FALSE_VALUE 0x0800 /* Edge taken when controlling predicate is zero. */ -#define EDGE_EXECUTABLE 4096 /* Edge is executable. Only +#define EDGE_EXECUTABLE 0x1000 /* Edge is executable. Only valid during SSA-CCP. */ -#define EDGE_CROSSING 8192 /* Edge crosses between hot +#define EDGE_CROSSING 0x2000 /* Edge crosses between hot and cold sections, when we do partitioning. */ -#define EDGE_ALL_FLAGS 16383 +#define EDGE_PRESERVE 0x4000 /* Never merge blocks via this edge. */ +#define EDGE_ALL_FLAGS 0x7fff -#define EDGE_COMPLEX (EDGE_ABNORMAL | EDGE_ABNORMAL_CALL | EDGE_EH) +#define EDGE_COMPLEX \ + (EDGE_ABNORMAL | EDGE_ABNORMAL_CALL | EDGE_EH | EDGE_PRESERVE) /* Counter summary from the last set of coverage counts read by profile.c. */ @@ -203,7 +206,9 @@ DEF_VEC_ALLOC_P(basic_block,heap); the compilation, so they are never cleared. All other flags may be cleared by clear_bb_flags(). It is generally - a bad idea to rely on any flags being up-to-date. */ + a bad idea to rely on any flags being up-to-date. + + Always update the table in cfg.c dump_bb_info. */ enum bb_flags { diff --git a/gcc/bb-reorder.c b/gcc/bb-reorder.c index fac5b29..c6a58d3 100644 --- a/gcc/bb-reorder.c +++ b/gcc/bb-reorder.c @@ -87,6 +87,7 @@ #include "tree-pass.h" #include "df.h" #include "bb-reorder.h" +#include "except.h" /* The number of rounds. In most cases there will only be 4 rounds, but when partitioning hot and cold basic blocks into separate sections of @@ -1208,6 +1209,79 @@ get_uncond_jump_length (void) return length; } +/* Emit a barrier into the footer of BB. */ + +static void +emit_barrier_after_bb (basic_block bb) +{ + rtx barrier = emit_barrier_after (BB_END (bb)); + bb->il.rtl->footer = unlink_insn_chain (barrier, barrier); +} + +/* The landing pad OLD_LP, in block OLD_BB, has edges from both partitions. + Duplicate the landing pad and split the edges so that no EH edge + crosses partitions. */ + +static void +fix_up_crossing_landing_pad (eh_landing_pad old_lp, basic_block old_bb) +{ + eh_landing_pad new_lp; + basic_block new_bb, last_bb, post_bb; + rtx new_label, jump, post_label; + unsigned new_partition; + edge_iterator ei; + edge e; + + /* Generate the new landing-pad structure. */ + new_lp = gen_eh_landing_pad (old_lp->region); + new_lp->post_landing_pad = old_lp->post_landing_pad; + new_lp->landing_pad = gen_label_rtx (); + LABEL_PRESERVE_P (new_lp->landing_pad) = 1; + + /* Put appropriate instructions in new bb. */ + new_label = emit_label (new_lp->landing_pad); + + expand_dw2_landing_pad_for_region (old_lp->region); + + post_bb = BLOCK_FOR_INSN (old_lp->landing_pad); + post_bb = single_succ (post_bb); + post_label = block_label (post_bb); + jump = emit_jump_insn (gen_jump (post_label)); + JUMP_LABEL (jump) = post_label; + + /* Create new basic block to be dest for lp. */ + last_bb = EXIT_BLOCK_PTR->prev_bb; + new_bb = create_basic_block (new_label, jump, last_bb); + new_bb->aux = last_bb->aux; + last_bb->aux = new_bb; + + emit_barrier_after_bb (new_bb); + + make_edge (new_bb, post_bb, 0); + + /* Make sure new bb is in the other partition. */ + new_partition = BB_PARTITION (old_bb); + new_partition ^= BB_HOT_PARTITION | BB_COLD_PARTITION; + BB_SET_PARTITION (new_bb, new_partition); + + /* Fix up the edges. */ + for (ei = ei_start (old_bb->preds); (e = ei_safe_edge (ei)) != NULL; ) + if (BB_PARTITION (e->src) == new_partition) + { + rtx insn = BB_END (e->src); + rtx note = find_reg_note (insn, REG_EH_REGION, NULL_RTX); + + gcc_assert (note != NULL); + gcc_checking_assert (INTVAL (XEXP (note, 0)) == old_lp->index); + XEXP (note, 0) = GEN_INT (new_lp->index); + + /* Adjust the edge to the new destination. */ + redirect_edge_succ (e, new_bb); + } + else + ei_next (&ei); +} + /* Find the basic blocks that are rarely executed and need to be moved to a separate section of the .o file (to cut down on paging and improve cache locality). Return a vector of all edges that cross. */ @@ -1221,7 +1295,6 @@ find_rarely_executed_basic_blocks_and_crossing_edges (void) edge_iterator ei; /* Mark which partition (hot/cold) each basic block belongs in. */ - FOR_EACH_BB (bb) { if (probably_never_executed_bb_p (bb)) @@ -1230,32 +1303,71 @@ find_rarely_executed_basic_blocks_and_crossing_edges (void) BB_SET_PARTITION (bb, BB_HOT_PARTITION); } - /* Mark every edge that crosses between sections. */ - - FOR_EACH_BB (bb) - FOR_EACH_EDGE (e, ei, bb->succs) + /* The format of .gcc_except_table does not allow landing pads to + be in a different partition as the throw. Fix this by either + moving or duplicating the landing pads. */ + if (cfun->eh->lp_array) { - if (e->src != ENTRY_BLOCK_PTR - && e->dest != EXIT_BLOCK_PTR - && BB_PARTITION (e->src) != BB_PARTITION (e->dest)) + unsigned i; + eh_landing_pad lp; + + FOR_EACH_VEC_ELT (eh_landing_pad, cfun->eh->lp_array, i, lp) { - e->flags |= EDGE_CROSSING; - VEC_safe_push (edge, heap, crossing_edges, e); + bool all_same, all_diff; + + if (lp == NULL) + continue; + + all_same = all_diff = true; + bb = BLOCK_FOR_INSN (lp->landing_pad); + FOR_EACH_EDGE (e, ei, bb->preds) + { + gcc_assert (e->flags & EDGE_EH); + if (BB_PARTITION (bb) == BB_PARTITION (e->src)) + all_diff = false; + else + all_same = false; + } + + if (all_same) + ; + else if (all_diff) + { + int which = BB_PARTITION (bb); + which ^= BB_HOT_PARTITION | BB_COLD_PARTITION; + BB_SET_PARTITION (bb, which); + } + else + fix_up_crossing_landing_pad (lp, bb); } - else - e->flags &= ~EDGE_CROSSING; } - return crossing_edges; -} + /* Mark every edge that crosses between sections. */ -/* Emit a barrier into the footer of BB. */ + FOR_EACH_BB (bb) + FOR_EACH_EDGE (e, ei, bb->succs) + { + unsigned int flags = e->flags; + + /* We should never have EDGE_CROSSING set yet. */ + gcc_checking_assert ((flags & EDGE_CROSSING) == 0); + + if (e->src != ENTRY_BLOCK_PTR + && e->dest != EXIT_BLOCK_PTR + && BB_PARTITION (e->src) != BB_PARTITION (e->dest)) + { + VEC_safe_push (edge, heap, crossing_edges, e); + flags |= EDGE_CROSSING; + } -static void -emit_barrier_after_bb (basic_block bb) -{ - rtx barrier = emit_barrier_after (BB_END (bb)); - bb->il.rtl->footer = unlink_insn_chain (barrier, barrier); + /* Now that we've split eh edges as appropriate, allow landing pads + to be merged with the post-landing pads. */ + flags &= ~EDGE_PRESERVE; + + e->flags = flags; + } + + return crossing_edges; } /* If any destination of a crossing edge does not have a label, add label; @@ -2108,6 +2220,8 @@ partition_hot_cold_basic_blocks (void) if (n_basic_blocks <= NUM_FIXED_BLOCKS + 1) return 0; + df_set_flags (DF_DEFER_INSN_RESCAN); + crossing_edges = find_rarely_executed_basic_blocks_and_crossing_edges (); if (crossing_edges == NULL) return 0; @@ -2139,6 +2253,25 @@ partition_hot_cold_basic_blocks (void) VEC_free (edge, heap, crossing_edges); + /* If we had to split landing pads, df data needs updating. */ + if (cfun->eh->lp_array) + { + /* ??? We can't re-scan the blocks with the current interface. + We have to throw away the data and start over. */ + df_finish_pass (true); + df_scan_alloc (NULL); + + /* We must re-scan the blocks because ARTIFICIAL references need + to be (re-)created for the new EH blocks. Which cannot be + done before the EH edges are added. */ + df_scan_blocks (); + /* Not all post-landing pads use all of the EH_RETURN_DATA_REGNO + data. We blindly generated all of them when creating the new + landing pad. Delete those assignments we don't use. */ + df_set_flags (DF_LR_RUN_DCE); + df_analyze (); + } + return TODO_verify_flow | TODO_verify_rtl_sharing; } diff --git a/gcc/cfg.c b/gcc/cfg.c index 00d1d5c..ca04c94 100644 --- a/gcc/cfg.c +++ b/gcc/cfg.c @@ -720,7 +720,7 @@ dump_edge_info (FILE *file, edge e, int do_succ) static const char * const bitnames[] = { "fallthru", "ab", "abcall", "eh", "fake", "dfs_back", "can_fallthru", "irreducible", "sibcall", "loop_exit", - "true", "false", "exec" + "true", "false", "exec", "crossing", "preserve" }; int comma = 0; int i, flags = e->flags; diff --git a/gcc/cfgrtl.c b/gcc/cfgrtl.c index b8843ca..076ff03 100644 --- a/gcc/cfgrtl.c +++ b/gcc/cfgrtl.c @@ -1820,18 +1820,38 @@ rtl_verify_flow_info_1 (void) } FOR_EACH_EDGE (e, ei, bb->succs) { + bool is_crossing; + if (e->flags & EDGE_FALLTHRU) + n_fallthru++, fallthru = e; + + is_crossing = (BB_PARTITION (e->src) != BB_PARTITION (e->dest) + && e->src != ENTRY_BLOCK_PTR + && e->dest != EXIT_BLOCK_PTR); + if (e->flags & EDGE_CROSSING) { - n_fallthru++, fallthru = e; - if ((e->flags & EDGE_CROSSING) - || (BB_PARTITION (e->src) != BB_PARTITION (e->dest) - && e->src != ENTRY_BLOCK_PTR - && e->dest != EXIT_BLOCK_PTR)) - { + if (!is_crossing) + { + error ("EDGE_CROSSING incorrectly set across same section"); + err = 1; + } + if (e->flags & EDGE_FALLTHRU) + { error ("fallthru edge crosses section boundary (bb %i)", e->src->index); err = 1; } + if (e->flags & EDGE_EH) + { + error ("EH edge crosses section boundary (bb %i)", + e->src->index); + err = 1; + } + } + else if (is_crossing) + { + error ("EDGE_CROSSING missing across section boundary"); + err = 1; } if ((e->flags & ~(EDGE_DFS_BACK diff --git a/gcc/except.c b/gcc/except.c index bb16036..8d56e10 100644 --- a/gcc/except.c +++ b/gcc/except.c @@ -919,6 +919,34 @@ emit_to_new_bb_before (rtx seq, rtx insn) return bb; } +/* A subroutine of dw2_build_landing_pads, also used for edge splitting + at the rtl level. Emit the code required by the target at a landing + pad for the given region. */ + +void +expand_dw2_landing_pad_for_region (eh_region region) +{ +#ifdef HAVE_exception_receiver + if (HAVE_exception_receiver) + emit_insn (gen_exception_receiver ()); + else +#endif +#ifdef HAVE_nonlocal_goto_receiver + if (HAVE_nonlocal_goto_receiver) + emit_insn (gen_nonlocal_goto_receiver ()); + else +#endif + { /* Nothing */ } + + if (region->exc_ptr_reg) + emit_move_insn (region->exc_ptr_reg, + gen_rtx_REG (ptr_mode, EH_RETURN_DATA_REGNO (0))); + if (region->filter_reg) + emit_move_insn (region->filter_reg, + gen_rtx_REG (targetm.eh_return_filter_mode (), + EH_RETURN_DATA_REGNO (1))); +} + /* Expand the extra code needed at landing pads for dwarf2 unwinding. */ static void @@ -926,10 +954,17 @@ dw2_build_landing_pads (void) { int i; eh_landing_pad lp; + int e_flags = EDGE_FALLTHRU; + + /* If we're going to partition blocks, we need to be able to add + new landing pads later, which means that we need to hold on to + the post-landing-pad block. Prevent it from being merged away. + We'll remove this bit after partitioning. */ + if (flag_reorder_blocks_and_partition) + e_flags |= EDGE_PRESERVE; for (i = 1; VEC_iterate (eh_landing_pad, cfun->eh->lp_array, i, lp); ++i) { - eh_region region; basic_block bb; rtx seq; edge e; @@ -943,32 +978,13 @@ dw2_build_landing_pads (void) emit_label (lp->landing_pad); LABEL_PRESERVE_P (lp->landing_pad) = 1; -#ifdef HAVE_exception_receiver - if (HAVE_exception_receiver) - emit_insn (gen_exception_receiver ()); - else -#endif -#ifdef HAVE_nonlocal_goto_receiver - if (HAVE_nonlocal_goto_receiver) - emit_insn (gen_nonlocal_goto_receiver ()); - else -#endif - { /* Nothing */ } - - region = lp->region; - if (region->exc_ptr_reg) - emit_move_insn (region->exc_ptr_reg, - gen_rtx_REG (ptr_mode, EH_RETURN_DATA_REGNO (0))); - if (region->filter_reg) - emit_move_insn (region->filter_reg, - gen_rtx_REG (targetm.eh_return_filter_mode (), - EH_RETURN_DATA_REGNO (1))); + expand_dw2_landing_pad_for_region (lp->region); seq = get_insns (); end_sequence (); bb = emit_to_new_bb_before (seq, label_rtx (lp->post_landing_pad)); - e = make_edge (bb, bb->next_bb, EDGE_FALLTHRU); + e = make_edge (bb, bb->next_bb, e_flags); e->count = bb->count; e->probability = REG_BR_PROB_BASE; } @@ -2388,9 +2404,6 @@ convert_to_eh_region_ranges (void) rtx section_switch_note = NULL_RTX; rtx first_no_action_insn_before_switch = NULL_RTX; rtx last_no_action_insn_before_switch = NULL_RTX; - rtx *pad_map = NULL; - sbitmap pad_loc = NULL; - int min_labelno = 0, max_labelno = 0; int saved_call_site_base = call_site_base; crtl->eh.action_record_data = VEC_alloc (uchar, gc, 64); @@ -2523,13 +2536,7 @@ convert_to_eh_region_ranges (void) gcc_assert (crtl->eh.call_site_record[cur_sec] == NULL); crtl->eh.call_site_record[cur_sec] = VEC_alloc (call_site_record, gc, 10); - max_labelno = max_label_num (); - min_labelno = get_first_label_num (); - pad_map = XCNEWVEC (rtx, max_labelno - min_labelno + 1); - pad_loc = sbitmap_alloc (max_labelno - min_labelno + 1); } - else if (LABEL_P (iter) && pad_map) - SET_BIT (pad_loc, CODE_LABEL_NUMBER (iter) - min_labelno); if (last_action >= -1 && ! first_no_action_insn) { @@ -2539,103 +2546,6 @@ convert_to_eh_region_ranges (void) call_site_base = saved_call_site_base; - if (pad_map) - { - /* When doing hot/cold partitioning, ensure landing pads are - always in the same section as the EH region, .gcc_except_table - can't express it otherwise. */ - for (cur_sec = 0; cur_sec < 2; cur_sec++) - { - int i, idx; - int n = VEC_length (call_site_record, - crtl->eh.call_site_record[cur_sec]); - basic_block prev_bb = NULL, padbb; - - for (i = 0; i < n; ++i) - { - struct call_site_record_d *cs = - VEC_index (call_site_record, - crtl->eh.call_site_record[cur_sec], i); - rtx jump, note; - - if (cs->landing_pad == NULL_RTX) - continue; - idx = CODE_LABEL_NUMBER (cs->landing_pad) - min_labelno; - /* If the landing pad is in the correct section, nothing - is needed. */ - if (TEST_BIT (pad_loc, idx) ^ (cur_sec == 0)) - continue; - /* Otherwise, if we haven't seen this pad yet, we need to - add a new label and jump to the correct section. */ - if (pad_map[idx] == NULL_RTX) - { - pad_map[idx] = gen_label_rtx (); - if (prev_bb == NULL) - for (iter = section_switch_note; - iter; iter = PREV_INSN (iter)) - if (NOTE_INSN_BASIC_BLOCK_P (iter)) - { - prev_bb = NOTE_BASIC_BLOCK (iter); - break; - } - if (cur_sec == 0) - { - note = emit_label_before (pad_map[idx], - section_switch_note); - jump = emit_jump_insn_before (gen_jump (cs->landing_pad), - section_switch_note); - } - else - { - jump = emit_jump_insn_after (gen_jump (cs->landing_pad), - section_switch_note); - note = emit_label_after (pad_map[idx], - section_switch_note); - } - JUMP_LABEL (jump) = cs->landing_pad; - add_reg_note (jump, REG_CROSSING_JUMP, NULL_RTX); - iter = NEXT_INSN (cs->landing_pad); - if (iter && NOTE_INSN_BASIC_BLOCK_P (iter)) - padbb = NOTE_BASIC_BLOCK (iter); - else - padbb = NULL; - if (padbb && prev_bb - && BB_PARTITION (padbb) != BB_UNPARTITIONED) - { - basic_block bb; - int part - = BB_PARTITION (padbb) == BB_COLD_PARTITION - ? BB_HOT_PARTITION : BB_COLD_PARTITION; - edge_iterator ei; - edge e; - - bb = create_basic_block (note, jump, prev_bb); - make_single_succ_edge (bb, padbb, EDGE_CROSSING); - BB_SET_PARTITION (bb, part); - for (ei = ei_start (padbb->preds); - (e = ei_safe_edge (ei)); ) - { - if ((e->flags & (EDGE_EH|EDGE_CROSSING)) - == (EDGE_EH|EDGE_CROSSING)) - { - redirect_edge_succ (e, bb); - e->flags &= ~EDGE_CROSSING; - } - else - ei_next (&ei); - } - if (cur_sec == 0) - prev_bb = bb; - } - } - cs->landing_pad = pad_map[idx]; - } - } - - sbitmap_free (pad_loc); - XDELETEVEC (pad_map); - } - htab_delete (ar_hash); return 0; } diff --git a/gcc/except.h b/gcc/except.h index 14eca87..5d461d7 100644 --- a/gcc/except.h +++ b/gcc/except.h @@ -253,6 +253,7 @@ extern rtx expand_builtin_dwarf_sp_column (void); extern void expand_builtin_eh_return (tree, tree); extern void expand_eh_return (void); extern rtx expand_builtin_extend_pointer (tree); +extern void expand_dw2_landing_pad_for_region (eh_region); typedef tree (*duplicate_eh_regions_map) (tree, void *); extern struct pointer_map_t *duplicate_eh_regions diff --git a/gcc/tree-cfg.c b/gcc/tree-cfg.c index bc71dd6..1207908 100644 --- a/gcc/tree-cfg.c +++ b/gcc/tree-cfg.c @@ -1421,7 +1421,7 @@ gimple_can_merge_blocks_p (basic_block a, basic_block b) if (!single_succ_p (a)) return false; - if (single_succ_edge (a)->flags & (EDGE_ABNORMAL | EDGE_EH)) + if (single_succ_edge (a)->flags & (EDGE_ABNORMAL | EDGE_EH | EDGE_PRESERVE)) return false; if (single_succ (a) != b)