[Intel-wired-lan] [PATCH net-next v5 2/6] net: napi: add internal ARFS rmap management

[Ahmed Zaki]

For drivers using the netif_enable_cpu_rmap(), move the IRQ rmap notifier
inside the napi_struct.

Signed-off-by: Ahmed Zaki <ahmed.z...@intel.com>
---
 include/linux/cpu_rmap.h  |  1 +
 include/linux/netdevice.h |  4 ++
 lib/cpu_rmap.c            |  2 +-
 net/core/dev.c            | 77 ++++++++++++++++++++++++++++++++++++++-
 4 files changed, 81 insertions(+), 3 deletions(-)

diff --git a/include/linux/cpu_rmap.h b/include/linux/cpu_rmap.h
index 20b5729903d7..2fd7ba75362a 100644
--- a/include/linux/cpu_rmap.h
+++ b/include/linux/cpu_rmap.h
@@ -32,6 +32,7 @@ struct cpu_rmap {
 #define CPU_RMAP_DIST_INF 0xffff
 
 extern struct cpu_rmap *alloc_cpu_rmap(unsigned int size, gfp_t flags);
+extern void cpu_rmap_get(struct cpu_rmap *rmap);
 extern int cpu_rmap_put(struct cpu_rmap *rmap);
 
 extern int cpu_rmap_add(struct cpu_rmap *rmap, void *obj);
diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index 7e95e9ee36dd..6f8b416aa32b 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -392,6 +392,10 @@ struct napi_struct {
        struct list_head        dev_list;
        struct hlist_node       napi_hash_node;
        int                     irq;
+#ifdef CONFIG_RFS_ACCEL
+       struct irq_affinity_notify notify;
+       int                     napi_rmap_idx;
+#endif
        int                     index;
        struct napi_config      *config;
 };
diff --git a/lib/cpu_rmap.c b/lib/cpu_rmap.c
index 4c348670da31..f03d9be3f06b 100644
--- a/lib/cpu_rmap.c
+++ b/lib/cpu_rmap.c
@@ -73,7 +73,7 @@ static void cpu_rmap_release(struct kref *ref)
  * cpu_rmap_get - internal helper to get new ref on a cpu_rmap
  * @rmap: reverse-map allocated with alloc_cpu_rmap()
  */
-static inline void cpu_rmap_get(struct cpu_rmap *rmap)
+void cpu_rmap_get(struct cpu_rmap *rmap)
 {
        kref_get(&rmap->refcount);
 }
diff --git a/net/core/dev.c b/net/core/dev.c
index 3ee7a514dca8..c965d947b33d 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -6748,7 +6748,20 @@ EXPORT_SYMBOL(netif_queue_set_napi);
 #ifdef CONFIG_RFS_ACCEL
 static void netif_disable_cpu_rmap(struct net_device *dev)
 {
-       free_irq_cpu_rmap(dev->rx_cpu_rmap);
+       struct cpu_rmap *rmap = dev->rx_cpu_rmap;
+       struct napi_struct *napi;
+       u16 index;
+
+       if (!rmap || !dev->rx_cpu_rmap_auto)
+               return;
+
+       for (index = 0; index < rmap->size; index++) {
+               napi = rmap->obj[index];
+               if (napi && napi->irq > 0)
+                       irq_set_affinity_notifier(napi->irq, NULL);
+       }
+
+       cpu_rmap_put(rmap);
        dev->rx_cpu_rmap = NULL;
        dev->rx_cpu_rmap_auto = false;
 }
@@ -6763,6 +6776,66 @@ int netif_enable_cpu_rmap(struct net_device *dev, 
unsigned int num_irqs)
        return 0;
 }
 EXPORT_SYMBOL(netif_enable_cpu_rmap);
+
+static void
+netif_irq_cpu_rmap_notify(struct irq_affinity_notify *notify,
+                         const cpumask_t *mask)
+{
+       struct napi_struct *napi =
+               container_of(notify, struct napi_struct, notify);
+       struct cpu_rmap *rmap = napi->dev->rx_cpu_rmap;
+       int err;
+
+       if (rmap && napi->dev->rx_cpu_rmap_auto) {
+               err = cpu_rmap_update(rmap, napi->napi_rmap_idx, mask);
+               if (err)
+                       pr_warn("%s: RMAP update failed (%d)\n",
+                               __func__, err);
+       }
+}
+
+static void
+netif_napi_affinity_release(struct kref *ref)
+{
+       struct napi_struct *napi =
+               container_of(ref, struct napi_struct, notify.kref);
+       struct cpu_rmap *rmap = napi->dev->rx_cpu_rmap;
+
+       if (!rmap || !napi->dev->rx_cpu_rmap_auto)
+               return;
+
+       rmap->obj[napi->napi_rmap_idx] = NULL;
+       cpu_rmap_put(rmap);
+}
+
+static int napi_irq_cpu_rmap_add(struct napi_struct *napi, int irq)
+{
+       struct cpu_rmap *rmap = napi->dev->rx_cpu_rmap;
+       int rc;
+
+       if (!rmap)
+               return -EINVAL;
+
+       napi->notify.notify = netif_irq_cpu_rmap_notify;
+       napi->notify.release = netif_napi_affinity_release;
+       cpu_rmap_get(rmap);
+       rc = cpu_rmap_add(rmap, napi);
+       if (rc < 0)
+               goto err_add;
+
+       napi->napi_rmap_idx = rc;
+       rc = irq_set_affinity_notifier(irq, &napi->notify);
+       if (rc)
+               goto err_set;
+
+       return 0;
+
+err_set:
+       rmap->obj[napi->napi_rmap_idx] = NULL;
+err_add:
+       cpu_rmap_put(rmap);
+       return rc;
+}
 #endif
 
 void netif_napi_set_irq(struct napi_struct *napi, int irq)
@@ -6774,7 +6847,7 @@ void netif_napi_set_irq(struct napi_struct *napi, int irq)
 
 #ifdef CONFIG_RFS_ACCEL
        if (napi->dev->rx_cpu_rmap && napi->dev->rx_cpu_rmap_auto) {
-               rc = irq_cpu_rmap_add(napi->dev->rx_cpu_rmap, irq);
+               rc = napi_irq_cpu_rmap_add(napi, irq);
                if (rc) {
                        netdev_warn(napi->dev, "Unable to update ARFS map 
(%d)\n",
                                    rc);
-- 
2.43.0