KFENCE objects are one of the reasons why kfree_nolock() cannot currently handle kmalloc() objects. They are however rare so we can simply defer their freeing to irq_work.
The only complication is where to put the llist node. We cannot use the freepointer location like in defer_free() because for some caches it may be outside the object area and KFENCE would detect writes there. Since KFENCE already solves a similar situation when freeing objects from SLAB_TYPESAFE_BY_RCU caches with an rcu_head in its internal metadata, reuse that rcu_head also for the llist node. Introduce kfence_obj_to_llnode() and kfence_llnode_to_obj() so SLAB can work with this llist node without being exposed to KFENCE internals. Signed-off-by: Vlastimil Babka (SUSE) <[email protected]> --- include/linux/kfence.h | 5 +++++ mm/kfence/core.c | 14 ++++++++++++++ mm/kfence/kfence.h | 5 ++++- mm/slub.c | 39 ++++++++++++++++++++++++++++++++++++--- 4 files changed, 59 insertions(+), 4 deletions(-) diff --git a/include/linux/kfence.h b/include/linux/kfence.h index e5822f6e7f27..00721c85258d 100644 --- a/include/linux/kfence.h +++ b/include/linux/kfence.h @@ -188,6 +188,9 @@ static __always_inline __must_check bool kfence_free(void *addr) return true; } +struct llist_node *kfence_obj_to_llnode(void *addr); +void *kfence_llnode_to_obj(struct llist_node *llnode); + /** * kfence_handle_page_fault() - perform page fault handling for KFENCE pages * @addr: faulting address @@ -235,6 +238,8 @@ static inline size_t kfence_ksize(const void *addr) { return 0; } static inline void *kfence_object_start(const void *addr) { return NULL; } static inline void __kfence_free(void *addr) { } static inline bool __must_check kfence_free(void *addr) { return false; } +static inline struct llist_node *kfence_obj_to_llnode(void *addr) { return NULL; } +static inline void *kfence_llnode_to_obj(struct llist_node *llnode) { return NULL; } static inline bool __must_check kfence_handle_page_fault(unsigned long addr, bool is_write, struct pt_regs *regs) { diff --git a/mm/kfence/core.c b/mm/kfence/core.c index 6577bd76954e..42519d24687f 100644 --- a/mm/kfence/core.c +++ b/mm/kfence/core.c @@ -1271,6 +1271,20 @@ void __kfence_free(void *addr) } } +struct llist_node *kfence_obj_to_llnode(void *addr) +{ + struct kfence_metadata *meta = addr_to_metadata((unsigned long)addr); + + return &meta->llnode; +} + +void *kfence_llnode_to_obj(struct llist_node *llnode) +{ + struct kfence_metadata *meta = container_of(llnode, struct kfence_metadata, llnode); + + return (void *)meta->addr; +} + bool kfence_handle_page_fault(unsigned long addr, bool is_write, struct pt_regs *regs) { const int page_index = (addr - (unsigned long)__kfence_pool) / PAGE_SIZE; diff --git a/mm/kfence/kfence.h b/mm/kfence/kfence.h index 1f618f9b0d12..0fca1dc2c794 100644 --- a/mm/kfence/kfence.h +++ b/mm/kfence/kfence.h @@ -58,7 +58,10 @@ struct kfence_track { /* KFENCE metadata per guarded allocation. */ struct kfence_metadata { struct list_head list __guarded_by(&kfence_freelist_lock); /* Freelist node. */ - struct rcu_head rcu_head; /* For delayed freeing. */ + union { + struct rcu_head rcu_head; /* For delayed freeing. */ + struct llist_node llnode; /* For kfree_nolock(). */ + }; /* * Lock protecting below data; to ensure consistency of the below data, diff --git a/mm/slub.c b/mm/slub.c index 044db93d64a0..2d7648b96bfa 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -4050,6 +4050,7 @@ static void flush_all(struct kmem_cache *s) struct deferred_percpu_work { struct llist_head objects; + struct llist_head objects_kfence; struct llist_head objects_by_rcu; struct llist_head rcu_sheaves; struct irq_work work; @@ -4059,6 +4060,7 @@ static void deferred_percpu_work_fn(struct irq_work *work); static DEFINE_PER_CPU(struct deferred_percpu_work, deferred_percpu_work) = { .objects = LLIST_HEAD_INIT(objects), + .objects_kfence = LLIST_HEAD_INIT(objects_kfence), .objects_by_rcu = LLIST_HEAD_INIT(objects_by_rcu), .rcu_sheaves = LLIST_HEAD_INIT(rcu_sheaves), .work = IRQ_WORK_INIT(deferred_percpu_work_fn), @@ -6399,6 +6401,13 @@ static void deferred_percpu_work_fn(struct irq_work *work) stat(s, FREE_SLOWPATH); } + llnode = llist_del_all(&dpw->objects_kfence); + llist_for_each_safe(pos, t, llnode) { + void *obj = kfence_llnode_to_obj(pos); + + __kfence_free(obj); + } + llnode = llist_del_all(&dpw->objects_by_rcu); llist_for_each_safe(pos, t, llnode) { void *head = pos; @@ -6431,6 +6440,21 @@ static void defer_free(struct kmem_cache *s, void *obj) irq_work_queue(&dpw->work); } +static void defer_free_kfence(void *obj) +{ + struct deferred_percpu_work *dpw; + struct llist_node *llnode; + + /* kasan_reset_tag() is not necessary, kfence objects are not tagged */ + llnode = kfence_obj_to_llnode(obj); + + guard(preempt)(); + + dpw = this_cpu_ptr(&deferred_percpu_work); + if (llist_add(llnode, &dpw->objects_kfence)) + irq_work_queue(&dpw->work); +} + void defer_kfree_rcu(struct kvfree_rcu_head *head) { struct deferred_percpu_work *dpw; @@ -6758,10 +6782,13 @@ EXPORT_SYMBOL(kfree); /* * Can be called while holding raw_spinlock_t or from IRQ and NMI, * but ONLY for objects allocated by kmalloc_nolock(). - * Debug checks (like kmemleak and kfence) were skipped on allocation, - * hence + * + * In case kmemleak is enabled, + * * obj = kmalloc(); kfree_nolock(obj); - * will miss kmemleak/kfence book keeping and will cause false positives. + * + * will miss kmemleak book keeping and will cause false positives. + * * large_kmalloc is not supported either. */ void kfree_nolock(const void *object) @@ -6793,6 +6820,12 @@ void kfree_nolock(const void *object) * since they take spinlocks or not safe from any context. */ kmsan_slab_free(s, x); + + if (is_kfence_address(x)) { + defer_free_kfence(x); + return; + } + /* * If KASAN finds a kernel bug it will do kasan_report_invalid_free() * which will call raw_spin_lock_irqsave() which is technically -- 2.55.0
