[PATCH rcu v2] 05/20] srcu: Make SRCU readers use ->srcu_ctrs for counter selection

[Paul E. McKenney]

This commit causes SRCU readers to use ->srcu_ctrs for counter
selection instead of ->srcu_idx.  This takes another step towards
array-indexing-free SRCU readers.

[ paulmck: Apply kernel test robot feedback. ]

Co-developed-by: Z qiang <qiang.zhang1...@gmail.com>
Signed-off-by: Z qiang <qiang.zhang1...@gmail.com>
Signed-off-by: Paul E. McKenney <paul...@kernel.org>
Tested-by: kernel test robot <oliver.s...@intel.com>
Cc: Alexei Starovoitov <a...@kernel.org>
Cc: Andrii Nakryiko <and...@kernel.org>
Cc: Peter Zijlstra <pet...@infradead.org>
Cc: Kent Overstreet <kent.overstr...@linux.dev>
Cc: <b...@vger.kernel.org>
---
 include/linux/srcutree.h |  9 +++++----
 kernel/rcu/srcutree.c    | 23 +++++++++++++----------
 2 files changed, 18 insertions(+), 14 deletions(-)

diff --git a/include/linux/srcutree.h b/include/linux/srcutree.h
index c794d599db5c..1b01ced61a45 100644
--- a/include/linux/srcutree.h
+++ b/include/linux/srcutree.h
@@ -101,6 +101,7 @@ struct srcu_usage {
  */
 struct srcu_struct {
        unsigned int srcu_idx;                  /* Current rdr array element. */
+       struct srcu_ctr __percpu *srcu_ctrp;
        struct srcu_data __percpu *sda;         /* Per-CPU srcu_data array. */
        struct lockdep_map dep_map;
        struct srcu_usage *srcu_sup;            /* Update-side data. */
@@ -167,6 +168,7 @@ struct srcu_struct {
 #define __SRCU_STRUCT_INIT(name, usage_name, pcpu_name)                        
                        \
 {                                                                              
                \
        .sda = &pcpu_name,                                                      
                \
+       .srcu_ctrp = &pcpu_name.srcu_ctrs[0],                                   
                \
        __SRCU_STRUCT_INIT_COMMON(name, usage_name)                             
                \
 }
 
@@ -222,13 +224,12 @@ void srcu_torture_stats_print(struct srcu_struct *ssp, 
char *tt, char *tf);
  */
 static inline int __srcu_read_lock_lite(struct srcu_struct *ssp)
 {
-       int idx;
+       struct srcu_ctr __percpu *scp = READ_ONCE(ssp->srcu_ctrp);
 
        RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching 
srcu_read_lock_lite().");
-       idx = READ_ONCE(ssp->srcu_idx) & 0x1;
-       this_cpu_inc(ssp->sda->srcu_ctrs[idx].srcu_locks.counter); /* Y */
+       this_cpu_inc(scp->srcu_locks.counter); /* Y */
        barrier(); /* Avoid leaking the critical section. */
-       return idx;
+       return scp - &ssp->sda->srcu_ctrs[0];
 }
 
 /*
diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c
index d4e9cd917a69..9af86ce2dd24 100644
--- a/kernel/rcu/srcutree.c
+++ b/kernel/rcu/srcutree.c
@@ -253,8 +253,10 @@ static int init_srcu_struct_fields(struct srcu_struct 
*ssp, bool is_static)
        atomic_set(&ssp->srcu_sup->srcu_barrier_cpu_cnt, 0);
        INIT_DELAYED_WORK(&ssp->srcu_sup->work, process_srcu);
        ssp->srcu_sup->sda_is_static = is_static;
-       if (!is_static)
+       if (!is_static) {
                ssp->sda = alloc_percpu(struct srcu_data);
+               ssp->srcu_ctrp = &ssp->sda->srcu_ctrs[0];
+       }
        if (!ssp->sda)
                goto err_free_sup;
        init_srcu_struct_data(ssp);
@@ -742,12 +744,11 @@ EXPORT_SYMBOL_GPL(__srcu_check_read_flavor);
  */
 int __srcu_read_lock(struct srcu_struct *ssp)
 {
-       int idx;
+       struct srcu_ctr __percpu *scp = READ_ONCE(ssp->srcu_ctrp);
 
-       idx = READ_ONCE(ssp->srcu_idx) & 0x1;
-       this_cpu_inc(ssp->sda->srcu_ctrs[idx].srcu_locks.counter);
+       this_cpu_inc(scp->srcu_locks.counter);
        smp_mb(); /* B */  /* Avoid leaking the critical section. */
-       return idx;
+       return scp - &ssp->sda->srcu_ctrs[0];
 }
 EXPORT_SYMBOL_GPL(__srcu_read_lock);
 
@@ -772,13 +773,12 @@ EXPORT_SYMBOL_GPL(__srcu_read_unlock);
  */
 int __srcu_read_lock_nmisafe(struct srcu_struct *ssp)
 {
-       int idx;
-       struct srcu_data *sdp = raw_cpu_ptr(ssp->sda);
+       struct srcu_ctr __percpu *scpp = READ_ONCE(ssp->srcu_ctrp);
+       struct srcu_ctr *scp = raw_cpu_ptr(scpp);
 
-       idx = READ_ONCE(ssp->srcu_idx) & 0x1;
-       atomic_long_inc(&sdp->srcu_ctrs[idx].srcu_locks);
+       atomic_long_inc(&scp->srcu_locks);
        smp_mb__after_atomic(); /* B */  /* Avoid leaking the critical section. 
*/
-       return idx;
+       return scpp - &ssp->sda->srcu_ctrs[0];
 }
 EXPORT_SYMBOL_GPL(__srcu_read_lock_nmisafe);
 
@@ -1152,6 +1152,8 @@ static void srcu_flip(struct srcu_struct *ssp)
        smp_mb(); /* E */  /* Pairs with B and C. */
 
        WRITE_ONCE(ssp->srcu_idx, ssp->srcu_idx + 1); // Flip the counter.
+       WRITE_ONCE(ssp->srcu_ctrp,
+                  &ssp->sda->srcu_ctrs[!(ssp->srcu_ctrp - 
&ssp->sda->srcu_ctrs[0])]);
 
        /*
         * Ensure that if the updater misses an __srcu_read_unlock()
@@ -2004,6 +2006,7 @@ static int srcu_module_coming(struct module *mod)
                ssp->sda = alloc_percpu(struct srcu_data);
                if (WARN_ON_ONCE(!ssp->sda))
                        return -ENOMEM;
+               ssp->srcu_ctrp = &ssp->sda->srcu_ctrs[0];
        }
        return 0;
 }
-- 
2.40.1