Hi Juzhe, on 2023/8/21 18:59, Juzhe-Zhong wrote: > Co-Authored-By: Kewen.Lin <li...@linux.ibm.com> > > Hi, @Richi and @Richard, base on previous disscussion, I simpily fix issuses > for > powerpc and s390 with your suggestions: > > - machine_mode len_load_mode = get_len_load_store_mode > - (loop_vinfo->vector_mode, true).require (); > - machine_mode len_store_mode = get_len_load_store_mode > - (loop_vinfo->vector_mode, false).require (); > + machine_mode len_load_mode, len_store_mode; > + if (!get_len_load_store_mode (loop_vinfo->vector_mode, true) > + .exists (&len_load_mode)) > + return false; > + if (!get_len_load_store_mode (loop_vinfo->vector_mode, false) > + .exists (&len_store_mode)) > + return false; > > Hi, @Kewen and @Stefan > > Could you test this patch again ? Thanks.
I confirmed it's bootstrapped and regress-tested on powerpc64le-linux-gnu P9/P10. Thanks! BR, Kewen > > Co-Authored-By: Kewen.Lin <li...@linux.ibm.com> > > gcc/ChangeLog: > > * tree-vect-loop.cc (vect_verify_loop_lens): Add exists check. > (vectorizable_live_operation): Add live vectorization for length loop > control. > > gcc/testsuite/ChangeLog: > > * gcc.target/riscv/rvv/autovec/partial/live-1.c: New test. > * gcc.target/riscv/rvv/autovec/partial/live_run-1.c: New test. > > --- > .../riscv/rvv/autovec/partial/live-1.c | 34 +++++++ > .../riscv/rvv/autovec/partial/live_run-1.c | 35 ++++++++ > gcc/tree-vect-loop.cc | 89 ++++++++++++++----- > 3 files changed, 138 insertions(+), 20 deletions(-) > create mode 100644 > gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live-1.c > create mode 100644 > gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live_run-1.c > > diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live-1.c > b/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live-1.c > new file mode 100644 > index 00000000000..75fa2eba8cc > --- /dev/null > +++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live-1.c > @@ -0,0 +1,34 @@ > +/* { dg-do compile } */ > +/* { dg-additional-options "-march=rv32gcv -mabi=ilp32d --param > riscv-autovec-preference=scalable -fdump-tree-optimized-details" } */ > + > +#include <stdint-gcc.h> > + > +#define EXTRACT_LAST(TYPE) > \ > + TYPE __attribute__ ((noinline, noclone)) > \ > + test_##TYPE (TYPE *x, int n, TYPE value) > \ > + { > \ > + TYPE last; > \ > + for (int j = 0; j < n; ++j) > \ > + { > \ > + last = x[j]; \ > + x[j] = last * value; \ > + } > \ > + return last; > \ > + } > + > +#define TEST_ALL(T) > \ > + T (int8_t) > \ > + T (int16_t) > \ > + T (int32_t) > \ > + T (int64_t) > \ > + T (uint8_t) > \ > + T (uint16_t) > \ > + T (uint32_t) > \ > + T (uint64_t) > \ > + T (_Float16) > \ > + T (float) > \ > + T (double) > + > +TEST_ALL (EXTRACT_LAST) > + > +/* { dg-final { scan-tree-dump-times "\.VEC_EXTRACT" 10 "optimized" } } */ > diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live_run-1.c > b/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live_run-1.c > new file mode 100644 > index 00000000000..42913a112c6 > --- /dev/null > +++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live_run-1.c > @@ -0,0 +1,35 @@ > +/* { dg-do run { target { riscv_vector } } } */ > +/* { dg-additional-options "--param riscv-autovec-preference=scalable" } */ > + > +#include "live-1.c" > + > +#define N 107 > +#define OP 70 > + > +#define TEST_LOOP(TYPE) \ > + { \ > + TYPE a[N]; \ > + for (int i = 0; i < N; ++i) \ > + { \ > + a[i] = i * 2 + (i % 3); \ > + asm volatile ("" ::: "memory"); \ > + } \ > + TYPE expected = a[N - 1]; \ > + TYPE res = test_##TYPE (a, N, OP); \ > + if (res != expected) \ > + __builtin_abort (); \ > + for (int i = 0; i < N; ++i) \ > + { \ > + TYPE old = i * 2 + (i % 3); \ > + if (a[i] != (TYPE) (old * (TYPE) OP)) \ > + __builtin_abort (); \ > + asm volatile ("" ::: "memory"); \ > + } \ > + } > + > +int __attribute__ ((optimize (1))) > +main (void) > +{ > + TEST_ALL (TEST_LOOP); > + return 0; > +} > diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc > index 1fcd8d07ea1..1cd6c291377 100644 > --- a/gcc/tree-vect-loop.cc > +++ b/gcc/tree-vect-loop.cc > @@ -32,6 +32,8 @@ along with GCC; see the file COPYING3. If not see > #include "tree-pass.h" > #include "ssa.h" > #include "optabs-tree.h" > +#include "memmodel.h" > +#include "optabs.h" > #include "diagnostic-core.h" > #include "fold-const.h" > #include "stor-layout.h" > @@ -1465,10 +1467,13 @@ vect_verify_loop_lens (loop_vec_info loop_vinfo) > if (LOOP_VINFO_LENS (loop_vinfo).is_empty ()) > return false; > > - machine_mode len_load_mode = get_len_load_store_mode > - (loop_vinfo->vector_mode, true).require (); > - machine_mode len_store_mode = get_len_load_store_mode > - (loop_vinfo->vector_mode, false).require (); > + machine_mode len_load_mode, len_store_mode; > + if (!get_len_load_store_mode (loop_vinfo->vector_mode, true) > + .exists (&len_load_mode)) > + return false; > + if (!get_len_load_store_mode (loop_vinfo->vector_mode, false) > + .exists (&len_store_mode)) > + return false; > > signed char partial_load_bias = internal_len_load_store_bias > (IFN_LEN_LOAD, len_load_mode); > @@ -10301,17 +10306,7 @@ vectorizable_live_operation (vec_info *vinfo, > stmt_vec_info stmt_info, > /* No transformation required. */ > if (loop_vinfo && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo)) > { > - if (!direct_internal_fn_supported_p (IFN_EXTRACT_LAST, vectype, > - OPTIMIZE_FOR_SPEED)) > - { > - if (dump_enabled_p ()) > - dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, > - "can't operate on partial vectors " > - "because the target doesn't support extract " > - "last reduction.\n"); > - LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false; > - } > - else if (slp_node) > + if (slp_node) > { > if (dump_enabled_p ()) > dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, > @@ -10331,9 +10326,26 @@ vectorizable_live_operation (vec_info *vinfo, > stmt_vec_info stmt_info, > else > { > gcc_assert (ncopies == 1 && !slp_node); > - vect_record_loop_mask (loop_vinfo, > - &LOOP_VINFO_MASKS (loop_vinfo), > - 1, vectype, NULL); > + if (direct_internal_fn_supported_p (IFN_EXTRACT_LAST, vectype, > + OPTIMIZE_FOR_SPEED)) > + vect_record_loop_mask (loop_vinfo, > + &LOOP_VINFO_MASKS (loop_vinfo), > + 1, vectype, NULL); > + else if (can_vec_extract_var_idx_p ( > + TYPE_MODE (vectype), TYPE_MODE (TREE_TYPE (vectype)))) > + vect_record_loop_len (loop_vinfo, > + &LOOP_VINFO_LENS (loop_vinfo), > + 1, vectype, 1); > + else > + { > + if (dump_enabled_p ()) > + dump_printf_loc ( > + MSG_MISSED_OPTIMIZATION, vect_location, > + "can't operate on partial vectors " > + "because the target doesn't support extract " > + "last reduction.\n"); > + LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false; > + } > } > } > /* ??? Enable for loop costing as well. */ > @@ -10359,7 +10371,9 @@ vectorizable_live_operation (vec_info *vinfo, > stmt_vec_info stmt_info, > gimple *vec_stmt; > if (slp_node) > { > - gcc_assert (!loop_vinfo || !LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)); > + gcc_assert (!loop_vinfo > + || (!LOOP_VINFO_FULLY_MASKED_P (loop_vinfo) > + && !LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo))); > > /* Get the correct slp vectorized stmt. */ > vec_lhs = SLP_TREE_VEC_DEFS (slp_node)[vec_entry]; > @@ -10403,7 +10417,42 @@ vectorizable_live_operation (vec_info *vinfo, > stmt_vec_info stmt_info, > > gimple_seq stmts = NULL; > tree new_tree; > - if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)) > + if (LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo)) > + { > + /* Emit: > + > + SCALAR_RES = VEC_EXTRACT <VEC_LHS, LEN + BIAS - 1> > + > + where VEC_LHS is the vectorized live-out result and MASK is > + the loop mask for the final iteration. */ > + gcc_assert (ncopies == 1 && !slp_node); > + gimple_seq tem = NULL; > + gimple_stmt_iterator gsi = gsi_last (tem); > + tree len > + = vect_get_loop_len (loop_vinfo, &gsi, > + &LOOP_VINFO_LENS (loop_vinfo), > + 1, vectype, 0, 0); > + > + /* BIAS - 1. */ > + signed char biasval = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo); > + tree bias_minus_one > + = int_const_binop (MINUS_EXPR, > + build_int_cst (TREE_TYPE (len), biasval), > + build_one_cst (TREE_TYPE (len))); > + > + /* LAST_INDEX = LEN + (BIAS - 1). */ > + tree last_index = gimple_build (&stmts, PLUS_EXPR, TREE_TYPE (len), > + len, bias_minus_one); > + > + /* SCALAR_RES = VEC_EXTRACT <VEC_LHS, LEN + BIAS - 1>. */ > + tree scalar_res > + = gimple_build (&stmts, CFN_VEC_EXTRACT, TREE_TYPE (vectype), > + vec_lhs_phi, last_index); > + > + /* Convert the extracted vector element to the scalar type. */ > + new_tree = gimple_convert (&stmts, lhs_type, scalar_res); > + } > + else if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)) > { > /* Emit: >