https://gcc.gnu.org/g:da7dc333c4a9ef997a9f3082fba73abf0380d684
commit r17-666-gda7dc333c4a9ef997a9f3082fba73abf0380d684 Author: Konstantinos Eleftheriou <[email protected]> Date: Fri Mar 27 06:26:48 2026 -0700 avoid-store-forwarding: Continue BB analysis after complex memory ops The pass aborted analysis of the entire remaining basic block when it encountered a complex memory operation (non-simple store/load). Store-forwarding opportunities after the complex operation were silently missed. Replace the early return with a flush-and-continue: clear the pending store candidates and keep scanning. The two other flush sites (throwing insns, unknown-size memory) already use this pattern. gcc/ChangeLog: * avoid-store-forwarding.cc (store_forwarding_analyzer::avoid_store_forwarding): Replace return with flush-and-continue for complex memory operations. gcc/testsuite/ChangeLog: * gcc.target/aarch64/avoid-store-forwarding-6.c: New test. Diff: --- gcc/avoid-store-forwarding.cc | 39 ++++++++++++---------- .../gcc.target/aarch64/avoid-store-forwarding-6.c | 18 ++++++++++ 2 files changed, 40 insertions(+), 17 deletions(-) diff --git a/gcc/avoid-store-forwarding.cc b/gcc/avoid-store-forwarding.cc index 0c065ba1a75a..bb015681c458 100644 --- a/gcc/avoid-store-forwarding.cc +++ b/gcc/avoid-store-forwarding.cc @@ -691,30 +691,35 @@ store_forwarding_analyzer::avoid_store_forwarding (basic_block bb) process_store_forwarding (forwardings, insn, load_mem); } - /* Abort in case that we encounter a memory read/write that is not a - simple store/load, as we can't make safe assumptions about the - side-effects of this. */ - if ((writes_mem && !is_simple_store) - || (reads_mem && !is_simple_load)) - return; - - if (removed_count) + /* If we encounter a memory read/write that is not a simple + store/load, flush all pending store candidates and continue. + We can't make safe assumptions about the side-effects, but + store-forwarding opportunities later in the BB should still + be analyzed. */ + if ((writes_mem && !is_simple_store) + || (reads_mem && !is_simple_load)) + { + store_exprs.truncate (0); + continue; + } + + if (removed_count) { unsigned int i, j; store_fwd_info *it; VEC_ORDERED_REMOVE_IF (store_exprs, i, j, it, it->remove); } - /* Don't consider store forwarding if the RTL instruction distance is - more than PARAM_STORE_FORWARDING_MAX_DISTANCE and the cost checks - are not disabled. */ - const bool unlimited_cost = (param_store_forwarding_max_distance == 0); - if (!unlimited_cost && !store_exprs.is_empty () - && (store_exprs[0].insn_cnt - + param_store_forwarding_max_distance <= insn_cnt)) - store_exprs.ordered_remove (0); + /* Don't consider store forwarding if the RTL instruction distance is + more than PARAM_STORE_FORWARDING_MAX_DISTANCE and the cost checks + are not disabled. */ + const bool unlimited_cost = (param_store_forwarding_max_distance == 0); + if (!unlimited_cost && !store_exprs.is_empty () + && (store_exprs[0].insn_cnt + + param_store_forwarding_max_distance <= insn_cnt)) + store_exprs.ordered_remove (0); - insn_cnt++; + insn_cnt++; } } diff --git a/gcc/testsuite/gcc.target/aarch64/avoid-store-forwarding-6.c b/gcc/testsuite/gcc.target/aarch64/avoid-store-forwarding-6.c new file mode 100644 index 000000000000..150235e9649a --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/avoid-store-forwarding-6.c @@ -0,0 +1,18 @@ +/* Check that the pass continues after a complex memory op (volatile load). */ +/* { dg-do compile } */ +/* { dg-options "-O2 -favoid-store-forwarding -fdump-rtl-avoid_store_forwarding" } */ + +typedef union { + char arr_8[8]; + long long_value; +} DataUnion; + +long ssll_complex_mem (DataUnion *data, char x, volatile int *v) +{ + (void)*v; + data->arr_8[4] = x; + return data->long_value; +} + +/* { dg-final { scan-rtl-dump-times "Store forwarding detected" 1 "avoid_store_forwarding" } } */ +/* { dg-final { scan-rtl-dump-times "Store forwarding avoided" 1 "avoid_store_forwarding" } } */
