Module Name: src
Committed By: riastradh
Date: Sun Dec 19 12:07:55 UTC 2021
Modified Files:
src/sys/external/bsd/common/include/linux: slab.h
src/sys/external/bsd/common/linux: linux_rcu.c
src/sys/external/bsd/drm2/include/linux: mm.h vmalloc.h
Log Message:
linux: Use kmem directly for Linux kmalloc.
Take advantage of this to do LOCKDEBUG_MEM_CHECK at the point of
kfree_rcu rather than in the RCU GC thread.
To generate a diff of this commit:
cvs rdiff -u -r1.7 -r1.8 src/sys/external/bsd/common/include/linux/slab.h
cvs rdiff -u -r1.4 -r1.5 src/sys/external/bsd/common/linux/linux_rcu.c
cvs rdiff -u -r1.22 -r1.23 src/sys/external/bsd/drm2/include/linux/mm.h
cvs rdiff -u -r1.10 -r1.11 src/sys/external/bsd/drm2/include/linux/vmalloc.h
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/external/bsd/common/include/linux/slab.h
diff -u src/sys/external/bsd/common/include/linux/slab.h:1.7 src/sys/external/bsd/common/include/linux/slab.h:1.8
--- src/sys/external/bsd/common/include/linux/slab.h:1.7 Sun Dec 19 12:00:48 2021
+++ src/sys/external/bsd/common/include/linux/slab.h Sun Dec 19 12:07:55 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: slab.h,v 1.7 2021/12/19 12:00:48 riastradh Exp $ */
+/* $NetBSD: slab.h,v 1.8 2021/12/19 12:07:55 riastradh Exp $ */
/*-
* Copyright (c) 2013 The NetBSD Foundation, Inc.
@@ -33,7 +33,6 @@
#define _LINUX_SLAB_H_
#include <sys/kmem.h>
-#include <sys/malloc.h>
#include <machine/limits.h>
@@ -44,10 +43,12 @@
#define ARCH_KMALLOC_MINALIGN 4 /* XXX ??? */
-/* XXX Should use kmem, but Linux kfree doesn't take the size. */
+struct linux_malloc {
+ size_t lm_size;
+} __aligned(ALIGNBYTES + 1);
static inline int
-linux_gfp_to_malloc(gfp_t gfp)
+linux_gfp_to_kmem(gfp_t gfp)
{
int flags = 0;
@@ -62,7 +63,6 @@ linux_gfp_to_malloc(gfp_t gfp)
}
if (ISSET(gfp, __GFP_ZERO)) {
- flags |= M_ZERO;
gfp &= ~__GFP_ZERO;
}
@@ -75,31 +75,43 @@ linux_gfp_to_malloc(gfp_t gfp)
((gfp & ~__GFP_WAIT) == (GFP_KERNEL & ~__GFP_WAIT)));
if (ISSET(gfp, __GFP_WAIT)) {
- flags |= M_WAITOK;
+ flags |= KM_SLEEP;
gfp &= ~__GFP_WAIT;
} else {
- flags |= M_NOWAIT;
+ flags |= KM_NOSLEEP;
}
return flags;
}
/*
- * XXX vmalloc and kmalloc both use malloc(9). If you change this, be
- * sure to update vmalloc in <linux/vmalloc.h> and kvfree in
- * <linux/mm.h>.
+ * XXX vmalloc and kmalloc both use this. If you change that, be sure
+ * to update vmalloc in <linux/vmalloc.h> and kvfree in <linux/mm.h>.
*/
static inline void *
kmalloc(size_t size, gfp_t gfp)
{
- return malloc(size, M_TEMP, linux_gfp_to_malloc(gfp));
+ struct linux_malloc *lm;
+ int kmflags = linux_gfp_to_kmem(gfp);
+
+ KASSERTMSG(size < SIZE_MAX - sizeof(*lm), "size=%zu", size);
+
+ if (gfp & __GFP_ZERO)
+ lm = kmem_intr_zalloc(sizeof(*lm) + size, kmflags);
+ else
+ lm = kmem_intr_alloc(sizeof(*lm) + size, kmflags);
+ if (lm == NULL)
+ return NULL;
+
+ lm->lm_size = size;
+ return lm + 1;
}
static inline void *
kzalloc(size_t size, gfp_t gfp)
{
- return malloc(size, M_TEMP, (linux_gfp_to_malloc(gfp) | M_ZERO));
+ return kmalloc(size, gfp | __GFP_ZERO);
}
static inline void *
@@ -107,7 +119,7 @@ kmalloc_array(size_t n, size_t size, gfp
{
if ((size != 0) && (n > (SIZE_MAX / size)))
return NULL;
- return malloc((n * size), M_TEMP, linux_gfp_to_malloc(gfp));
+ return kmalloc(n * size, gfp);
}
static inline void *
@@ -119,14 +131,35 @@ kcalloc(size_t n, size_t size, gfp_t gfp
static inline void *
krealloc(void *ptr, size_t size, gfp_t gfp)
{
- return realloc(ptr, size, M_TEMP, linux_gfp_to_malloc(gfp));
+ struct linux_malloc *olm, *nlm;
+ int kmflags = linux_gfp_to_kmem(gfp);
+
+ if (gfp & __GFP_ZERO)
+ nlm = kmem_intr_zalloc(sizeof(*nlm) + size, kmflags);
+ else
+ nlm = kmem_intr_alloc(sizeof(*nlm) + size, kmflags);
+ if (nlm == NULL)
+ return NULL;
+
+ nlm->lm_size = size;
+ if (ptr) {
+ olm = (struct linux_malloc *)ptr - 1;
+ memcpy(nlm + 1, olm + 1, MIN(nlm->lm_size, olm->lm_size));
+ kmem_intr_free(olm, sizeof(*olm) + olm->lm_size);
+ }
+ return nlm + 1;
}
static inline void
kfree(void *ptr)
{
- if (ptr != NULL)
- free(ptr, M_TEMP);
+ struct linux_malloc *lm;
+
+ if (ptr == NULL)
+ return;
+
+ lm = (struct linux_malloc *)ptr - 1;
+ kmem_intr_free(lm, sizeof(*lm) + lm->lm_size);
}
#define SLAB_HWCACHE_ALIGN __BIT(0)
Index: src/sys/external/bsd/common/linux/linux_rcu.c
diff -u src/sys/external/bsd/common/linux/linux_rcu.c:1.4 src/sys/external/bsd/common/linux/linux_rcu.c:1.5
--- src/sys/external/bsd/common/linux/linux_rcu.c:1.4 Sun Dec 19 11:49:11 2021
+++ src/sys/external/bsd/common/linux/linux_rcu.c Sun Dec 19 12:07:55 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: linux_rcu.c,v 1.4 2021/12/19 11:49:11 riastradh Exp $ */
+/* $NetBSD: linux_rcu.c,v 1.5 2021/12/19 12:07:55 riastradh Exp $ */
/*-
* Copyright (c) 2018 The NetBSD Foundation, Inc.
@@ -30,13 +30,15 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: linux_rcu.c,v 1.4 2021/12/19 11:49:11 riastradh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: linux_rcu.c,v 1.5 2021/12/19 12:07:55 riastradh Exp $");
#include <sys/param.h>
#include <sys/types.h>
+
#include <sys/condvar.h>
#include <sys/cpu.h>
#include <sys/kthread.h>
+#include <sys/lockdebug.h>
#include <sys/mutex.h>
#include <sys/sdt.h>
#include <sys/xcall.h>
@@ -191,6 +193,8 @@ void
_kfree_rcu(struct rcu_head *head, void *obj)
{
+ LOCKDEBUG_MEM_CHECK(obj, ((struct linux_malloc *)obj - 1)->lm_size);
+
head->rcuh_u.obj = obj;
mutex_enter(&gc.lock);
Index: src/sys/external/bsd/drm2/include/linux/mm.h
diff -u src/sys/external/bsd/drm2/include/linux/mm.h:1.22 src/sys/external/bsd/drm2/include/linux/mm.h:1.23
--- src/sys/external/bsd/drm2/include/linux/mm.h:1.22 Sun Dec 19 11:46:58 2021
+++ src/sys/external/bsd/drm2/include/linux/mm.h Sun Dec 19 12:07:55 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: mm.h,v 1.22 2021/12/19 11:46:58 riastradh Exp $ */
+/* $NetBSD: mm.h,v 1.23 2021/12/19 12:07:55 riastradh Exp $ */
/*-
* Copyright (c) 2013 The NetBSD Foundation, Inc.
@@ -32,8 +32,6 @@
#ifndef _LINUX_MM_H_
#define _LINUX_MM_H_
-#include <sys/malloc.h>
-
#include <uvm/uvm_extern.h>
#include <uvm/uvm_object.h>
@@ -135,16 +133,15 @@ kvmalloc_array(size_t nelem, size_t elem
}
/*
- * XXX Requires that kmalloc in <linux/slab.h> and vmalloc in
- * <linux/vmalloc.h> both use malloc(9). If you change either of
- * those, be sure to update this.
+ * XXX kvfree must additionally work on kmalloc (linux/slab.h) and
+ * vmalloc (linux/vmalloc.h). If you change either of those, be sure
+ * to change this too.
*/
+
static inline void
kvfree(void *ptr)
{
-
- if (ptr != NULL)
- free(ptr, M_TEMP);
+ kfree(ptr);
}
static inline void
Index: src/sys/external/bsd/drm2/include/linux/vmalloc.h
diff -u src/sys/external/bsd/drm2/include/linux/vmalloc.h:1.10 src/sys/external/bsd/drm2/include/linux/vmalloc.h:1.11
--- src/sys/external/bsd/drm2/include/linux/vmalloc.h:1.10 Sun Dec 19 10:51:24 2021
+++ src/sys/external/bsd/drm2/include/linux/vmalloc.h Sun Dec 19 12:07:55 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: vmalloc.h,v 1.10 2021/12/19 10:51:24 riastradh Exp $ */
+/* $NetBSD: vmalloc.h,v 1.11 2021/12/19 12:07:55 riastradh Exp $ */
/*-
* Copyright (c) 2013, 2018 The NetBSD Foundation, Inc.
@@ -32,21 +32,21 @@
#ifndef _LINUX_VMALLOC_H_
#define _LINUX_VMALLOC_H_
-#include <sys/malloc.h>
-
#include <uvm/uvm_extern.h>
#include <linux/mm.h>
#include <linux/mm_types.h>
#include <linux/overflow.h>
+#include <linux/slab.h>
#include <asm/page.h>
struct notifier_block;
/*
- * XXX vmalloc and kmalloc both use malloc(9). If you change this, be
- * sure to update kmalloc in <linux/slab.h> and kvfree in <linux/mm.h>.
+ * XXX vmalloc and kvmalloc both use kmalloc. If you change that, be
+ * sure to update this so kvfree in <linux/mm.h> still works on vmalloc
+ * addresses.
*/
static inline bool
@@ -58,27 +58,25 @@ is_vmalloc_addr(void *addr)
static inline void *
vmalloc(unsigned long size)
{
- return malloc(size, M_TEMP, M_WAITOK);
+ return kmalloc(size, GFP_KERNEL);
}
static inline void *
vmalloc_user(unsigned long size)
{
- return malloc(size, M_TEMP, (M_WAITOK | M_ZERO));
+ return kzalloc(size, GFP_KERNEL);
}
static inline void *
vzalloc(unsigned long size)
{
- return malloc(size, M_TEMP, (M_WAITOK | M_ZERO));
+ return kzalloc(size, GFP_KERNEL);
}
static inline void
vfree(void *ptr)
{
- if (ptr == NULL)
- return;
- return free(ptr, M_TEMP);
+ return kfree(ptr);
}
#define PAGE_KERNEL UVM_PROT_RW
