Hi Juzhe,

on 2023/8/21 18:59, Juzhe-Zhong wrote:
> Co-Authored-By: Kewen.Lin <li...@linux.ibm.com>
> 
> Hi, @Richi and @Richard, base on previous disscussion, I simpily fix issuses 
> for
> powerpc and s390 with your suggestions:
> 
> -  machine_mode len_load_mode = get_len_load_store_mode
> -    (loop_vinfo->vector_mode, true).require ();
> -  machine_mode len_store_mode = get_len_load_store_mode
> -    (loop_vinfo->vector_mode, false).require ();
> +  machine_mode len_load_mode, len_store_mode;
> +  if (!get_len_load_store_mode (loop_vinfo->vector_mode, true)
> +        .exists (&len_load_mode))
> +    return false;
> +  if (!get_len_load_store_mode (loop_vinfo->vector_mode, false)
> +        .exists (&len_store_mode))
> +    return false;
> 
> Hi, @Kewen and @Stefan
> 
> Could you test this patch again ? Thanks.

I confirmed it's bootstrapped and regress-tested on
powerpc64le-linux-gnu P9/P10.  Thanks!

BR,
Kewen

> 
> Co-Authored-By: Kewen.Lin <li...@linux.ibm.com>
> 
> gcc/ChangeLog:
> 
>       * tree-vect-loop.cc (vect_verify_loop_lens): Add exists check.
>       (vectorizable_live_operation): Add live vectorization for length loop 
> control.
> 
> gcc/testsuite/ChangeLog:
> 
>       * gcc.target/riscv/rvv/autovec/partial/live-1.c: New test.
>       * gcc.target/riscv/rvv/autovec/partial/live_run-1.c: New test.
> 
> ---
>  .../riscv/rvv/autovec/partial/live-1.c        | 34 +++++++
>  .../riscv/rvv/autovec/partial/live_run-1.c    | 35 ++++++++
>  gcc/tree-vect-loop.cc                         | 89 ++++++++++++++-----
>  3 files changed, 138 insertions(+), 20 deletions(-)
>  create mode 100644 
> gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live-1.c
>  create mode 100644 
> gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live_run-1.c
> 
> diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live-1.c 
> b/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live-1.c
> new file mode 100644
> index 00000000000..75fa2eba8cc
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live-1.c
> @@ -0,0 +1,34 @@
> +/* { dg-do compile } */
> +/* { dg-additional-options "-march=rv32gcv -mabi=ilp32d --param 
> riscv-autovec-preference=scalable -fdump-tree-optimized-details" } */
> +
> +#include <stdint-gcc.h>
> +
> +#define EXTRACT_LAST(TYPE)                                                   
>   \
> +  TYPE __attribute__ ((noinline, noclone))                                   
>   \
> +  test_##TYPE (TYPE *x, int n, TYPE value)                                   
>   \
> +  {                                                                          
>   \
> +    TYPE last;                                                               
>   \
> +    for (int j = 0; j < n; ++j)                                              
>   \
> +      {                                                                      
>   \
> +     last = x[j];                                                           \
> +     x[j] = last * value;                                                   \
> +      }                                                                      
>   \
> +    return last;                                                             
>   \
> +  }
> +
> +#define TEST_ALL(T)                                                          
>   \
> +  T (int8_t)                                                                 
>   \
> +  T (int16_t)                                                                
>   \
> +  T (int32_t)                                                                
>   \
> +  T (int64_t)                                                                
>   \
> +  T (uint8_t)                                                                
>   \
> +  T (uint16_t)                                                               
>   \
> +  T (uint32_t)                                                               
>   \
> +  T (uint64_t)                                                               
>   \
> +  T (_Float16)                                                               
>   \
> +  T (float)                                                                  
>   \
> +  T (double)
> +
> +TEST_ALL (EXTRACT_LAST)
> +
> +/* { dg-final { scan-tree-dump-times "\.VEC_EXTRACT" 10 "optimized" } } */
> diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live_run-1.c 
> b/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live_run-1.c
> new file mode 100644
> index 00000000000..42913a112c6
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live_run-1.c
> @@ -0,0 +1,35 @@
> +/* { dg-do run { target { riscv_vector } } } */
> +/* { dg-additional-options "--param riscv-autovec-preference=scalable" } */
> +
> +#include "live-1.c"
> +
> +#define N 107
> +#define OP 70
> +
> +#define TEST_LOOP(TYPE)                              \
> +  {                                          \
> +    TYPE a[N];                                       \
> +    for (int i = 0; i < N; ++i)                      \
> +      {                                              \
> +     a[i] = i * 2 + (i % 3);                 \
> +     asm volatile ("" ::: "memory");         \
> +      }                                              \
> +    TYPE expected = a[N - 1];                        \
> +    TYPE res = test_##TYPE (a, N, OP);               \
> +    if (res != expected)                     \
> +      __builtin_abort ();                    \
> +    for (int i = 0; i < N; ++i)                      \
> +      {                                              \
> +     TYPE old = i * 2 + (i % 3);             \
> +     if (a[i] != (TYPE) (old * (TYPE) OP))   \
> +       __builtin_abort ();                   \
> +     asm volatile ("" ::: "memory");         \
> +      }                                              \
> +  }
> +
> +int __attribute__ ((optimize (1)))
> +main (void)
> +{
> +  TEST_ALL (TEST_LOOP);
> +  return 0;
> +}
> diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
> index 1fcd8d07ea1..1cd6c291377 100644
> --- a/gcc/tree-vect-loop.cc
> +++ b/gcc/tree-vect-loop.cc
> @@ -32,6 +32,8 @@ along with GCC; see the file COPYING3.  If not see
>  #include "tree-pass.h"
>  #include "ssa.h"
>  #include "optabs-tree.h"
> +#include "memmodel.h"
> +#include "optabs.h"
>  #include "diagnostic-core.h"
>  #include "fold-const.h"
>  #include "stor-layout.h"
> @@ -1465,10 +1467,13 @@ vect_verify_loop_lens (loop_vec_info loop_vinfo)
>    if (LOOP_VINFO_LENS (loop_vinfo).is_empty ())
>      return false;
>  
> -  machine_mode len_load_mode = get_len_load_store_mode
> -    (loop_vinfo->vector_mode, true).require ();
> -  machine_mode len_store_mode = get_len_load_store_mode
> -    (loop_vinfo->vector_mode, false).require ();
> +  machine_mode len_load_mode, len_store_mode;
> +  if (!get_len_load_store_mode (loop_vinfo->vector_mode, true)
> +      .exists (&len_load_mode))
> +    return false;
> +  if (!get_len_load_store_mode (loop_vinfo->vector_mode, false)
> +      .exists (&len_store_mode))
> +    return false;
>  
>    signed char partial_load_bias = internal_len_load_store_bias
>      (IFN_LEN_LOAD, len_load_mode);
> @@ -10301,17 +10306,7 @@ vectorizable_live_operation (vec_info *vinfo, 
> stmt_vec_info stmt_info,
>        /* No transformation required.  */
>        if (loop_vinfo && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
>       {
> -       if (!direct_internal_fn_supported_p (IFN_EXTRACT_LAST, vectype,
> -                                            OPTIMIZE_FOR_SPEED))
> -         {
> -           if (dump_enabled_p ())
> -             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
> -                              "can't operate on partial vectors "
> -                              "because the target doesn't support extract "
> -                              "last reduction.\n");
> -           LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
> -         }
> -       else if (slp_node)
> +       if (slp_node)
>           {
>             if (dump_enabled_p ())
>               dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
> @@ -10331,9 +10326,26 @@ vectorizable_live_operation (vec_info *vinfo, 
> stmt_vec_info stmt_info,
>         else
>           {
>             gcc_assert (ncopies == 1 && !slp_node);
> -           vect_record_loop_mask (loop_vinfo,
> -                                  &LOOP_VINFO_MASKS (loop_vinfo),
> -                                  1, vectype, NULL);
> +           if (direct_internal_fn_supported_p (IFN_EXTRACT_LAST, vectype,
> +                                               OPTIMIZE_FOR_SPEED))
> +             vect_record_loop_mask (loop_vinfo,
> +                                    &LOOP_VINFO_MASKS (loop_vinfo),
> +                                    1, vectype, NULL);
> +           else if (can_vec_extract_var_idx_p (
> +                      TYPE_MODE (vectype), TYPE_MODE (TREE_TYPE (vectype))))
> +             vect_record_loop_len (loop_vinfo,
> +                                   &LOOP_VINFO_LENS (loop_vinfo),
> +                                   1, vectype, 1);
> +           else
> +             {
> +               if (dump_enabled_p ())
> +                 dump_printf_loc (
> +                   MSG_MISSED_OPTIMIZATION, vect_location,
> +                   "can't operate on partial vectors "
> +                   "because the target doesn't support extract "
> +                   "last reduction.\n");
> +               LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
> +             }
>           }
>       }
>        /* ???  Enable for loop costing as well.  */
> @@ -10359,7 +10371,9 @@ vectorizable_live_operation (vec_info *vinfo, 
> stmt_vec_info stmt_info,
>    gimple *vec_stmt;
>    if (slp_node)
>      {
> -      gcc_assert (!loop_vinfo || !LOOP_VINFO_FULLY_MASKED_P (loop_vinfo));
> +      gcc_assert (!loop_vinfo
> +               || (!LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
> +                   && !LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo)));
>  
>        /* Get the correct slp vectorized stmt.  */
>        vec_lhs = SLP_TREE_VEC_DEFS (slp_node)[vec_entry];
> @@ -10403,7 +10417,42 @@ vectorizable_live_operation (vec_info *vinfo, 
> stmt_vec_info stmt_info,
>  
>        gimple_seq stmts = NULL;
>        tree new_tree;
> -      if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
> +      if (LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo))
> +     {
> +       /* Emit:
> +
> +            SCALAR_RES = VEC_EXTRACT <VEC_LHS, LEN + BIAS - 1>
> +
> +          where VEC_LHS is the vectorized live-out result and MASK is
> +          the loop mask for the final iteration.  */
> +       gcc_assert (ncopies == 1 && !slp_node);
> +       gimple_seq tem = NULL;
> +       gimple_stmt_iterator gsi = gsi_last (tem);
> +       tree len
> +         = vect_get_loop_len (loop_vinfo, &gsi,
> +                              &LOOP_VINFO_LENS (loop_vinfo),
> +                              1, vectype, 0, 0);
> +
> +       /* BIAS - 1.  */
> +       signed char biasval = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
> +       tree bias_minus_one
> +         = int_const_binop (MINUS_EXPR,
> +                            build_int_cst (TREE_TYPE (len), biasval),
> +                            build_one_cst (TREE_TYPE (len)));
> +
> +       /* LAST_INDEX = LEN + (BIAS - 1).  */
> +       tree last_index = gimple_build (&stmts, PLUS_EXPR, TREE_TYPE (len),
> +                                       len, bias_minus_one);
> +
> +       /* SCALAR_RES = VEC_EXTRACT <VEC_LHS, LEN + BIAS - 1>.  */
> +       tree scalar_res
> +         = gimple_build (&stmts, CFN_VEC_EXTRACT, TREE_TYPE (vectype),
> +                         vec_lhs_phi, last_index);
> +
> +       /* Convert the extracted vector element to the scalar type.  */
> +       new_tree = gimple_convert (&stmts, lhs_type, scalar_res);
> +     }
> +      else if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
>       {
>         /* Emit:
>  

Reply via email to