Currently, k[v]free_rcu() cannot be called in unknown context since it could lead to a deadlock when called in the middle of k[v]free_rcu().
Make users' lives easier by introducing kfree_rcu_nolock() variant, now that kfree_rcu_sheaf() is available on PREEMPT_RT and __kfree_rcu_sheaf() handles unknown context. kfree_rcu_nolock() falls back to the kvfree_rcu batching when sheaves path fails. In most cases, the sheaves path is expected to succeed and it's unnecessary to add complexity to the existing kvfree_rcu batching by teaching it how to handle unknown context. Since defer_kfree_rcu() can be called on caches without sheaves, move deferred_work_barrier() and rcu_barrier() outside the branch in kvfree_rcu_barrier_on_cache(). Signed-off-by: Harry Yoo (Oracle) <[email protected]> --- include/linux/rcupdate.h | 19 +++++++++++++++++++ mm/slab_common.c | 28 ++++++++++++++++++++++++++-- mm/slub.c | 24 +++++++++++++++++++++++- 3 files changed, 68 insertions(+), 3 deletions(-) diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index ef5bb6981133..ce0fd401352e 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -1099,6 +1099,7 @@ static inline void rcu_read_unlock_migrate(void) * In mm/slab_common.c, no suitable header to include here. */ void kvfree_call_rcu(struct kvfree_rcu_head *head, void *ptr); +void kfree_call_rcu_nolock(struct kvfree_rcu_head *head, void *ptr); /* * The BUILD_BUG_ON() makes sure the rcu_head offset can be handled. See the @@ -1124,6 +1125,24 @@ do { \ kvfree_call_rcu(NULL, (void *) (___p)); \ } while (0) +/** + * kfree_rcu_nolock() - a version of kfree_rcu() that can be called in any context. + * @ptr: pointer to kfree for double-argument invocations. + * @kvrhf: the name of the struct kvfree_rcu_head within the type of @ptr. + * + * Objects that are not allocated via kmalloc_nolock() are not supported. + * kfree_rcu_nolock() supports 2-arg variant only. + */ +#define kfree_rcu_nolock(ptr, kvrhf) \ +do { \ + typeof (ptr) ___p = (ptr); \ + \ + if (___p) { \ + BUILD_BUG_ON(offsetof(typeof(*(ptr)), kvrhf) >= 4096); \ + kfree_call_rcu_nolock(&((___p)->kvrhf), (void *) (___p)); \ + } \ +} while (0) + /* * Place this after a lock-acquisition primitive to guarantee that * an UNLOCK+LOCK pair acts as a full barrier. This guarantee applies diff --git a/mm/slab_common.c b/mm/slab_common.c index c1c32909fa52..09c457645e03 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -1263,6 +1263,29 @@ EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc); EXPORT_TRACEPOINT_SYMBOL(kfree); EXPORT_TRACEPOINT_SYMBOL(kmem_cache_free); +void kfree_call_rcu_nolock(struct kvfree_rcu_head *head, void *ptr) +{ + struct slab *slab; + struct kmem_cache *s; + + VM_WARN_ON_ONCE(is_vmalloc_addr(ptr) || !virt_to_slab(ptr)); + + slab = virt_to_slab(ptr); + s = slab->slab_cache; + + if (unlikely(IS_ENABLED(CONFIG_NUMA) && slab_nid(slab) != numa_mem_id())) + goto fallback; + + if (unlikely(!__kfree_rcu_sheaf(s, ptr, SLAB_FREE_NOLOCK))) + goto fallback; + + return; + +fallback: + defer_kfree_rcu(head); +} +EXPORT_SYMBOL_GPL(kfree_call_rcu_nolock); + #ifndef CONFIG_KVFREE_RCU_BATCHED void kvfree_call_rcu(struct rcu_head *head, void *ptr) @@ -2120,10 +2143,11 @@ void kvfree_rcu_barrier_on_cache(struct kmem_cache *s) cpus_read_lock(); flush_rcu_sheaves_on_cache(s); cpus_read_unlock(); - deferred_work_barrier(); - rcu_barrier(); } + /* kfree_rcu_nolock() might have deferred frees even without sheaves */ + deferred_work_barrier(); + rcu_barrier(); __kvfree_rcu_barrier(); } EXPORT_SYMBOL_GPL(kvfree_rcu_barrier_on_cache); diff --git a/mm/slub.c b/mm/slub.c index d0976396d703..7114451dd5d0 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -4081,6 +4081,7 @@ static void flush_all(struct kmem_cache *s) struct deferred_percpu_work { struct llist_head objects; + struct llist_head objects_by_rcu; struct llist_head rcu_sheaves; struct irq_work work; }; @@ -4089,6 +4090,7 @@ static void deferred_percpu_work_fn(struct irq_work *work); static DEFINE_PER_CPU(struct deferred_percpu_work, deferred_percpu_work) = { .objects = LLIST_HEAD_INIT(objects), + .objects_by_rcu = LLIST_HEAD_INIT(objects_by_rcu), .rcu_sheaves = LLIST_HEAD_INIT(rcu_sheaves), .work = IRQ_WORK_INIT(deferred_percpu_work_fn), }; @@ -6386,13 +6388,14 @@ static void free_to_pcs_bulk(struct kmem_cache *s, size_t size, void **p) static void deferred_percpu_work_fn(struct irq_work *work) { struct deferred_percpu_work *dpw; - struct llist_head *objs, *rcu_sheaves; + struct llist_head *objs, *objs_by_rcu, *rcu_sheaves; struct llist_node *llnode, *pos, *t; struct slab_sheaf *sheaf, *next; dpw = container_of(work, struct deferred_percpu_work, work); rcu_sheaves = &dpw->rcu_sheaves; objs = &dpw->objects; + objs_by_rcu = &dpw->objects_by_rcu; llnode = llist_del_all(objs); llist_for_each_safe(pos, t, llnode) { @@ -6417,6 +6420,14 @@ static void deferred_percpu_work_fn(struct irq_work *work) stat(s, FREE_SLOWPATH); } + llnode = llist_del_all(objs_by_rcu); + llist_for_each_safe(pos, t, llnode) { + void *head = pos; + void *objp = kvmalloc_obj_start_addr(head); + + kvfree_call_rcu(head, objp); + } + llnode = llist_del_all(rcu_sheaves); llist_for_each_entry_safe(sheaf, next, llnode, llnode) call_rcu(&sheaf->rcu_head, rcu_free_sheaf); @@ -6435,6 +6446,17 @@ static void defer_free(struct kmem_cache *s, void *head) irq_work_queue(&dpw->work); } +void defer_kfree_rcu(struct kvfree_rcu_head *head) +{ + struct deferred_percpu_work *dpw; + + guard(preempt)(); + + dpw = this_cpu_ptr(&deferred_percpu_work); + if (llist_add((struct llist_node *)head, &dpw->objects_by_rcu)) + irq_work_queue(&dpw->work); +} + void deferred_work_barrier(void) { int cpu; -- 2.53.0
