Keep a per-processor stack of free kernel messages buffers.
* ipc/ipc_kmsg.h (IKM_CACHE_SIZE): New macro.
(struct ipc_mksg_cpu_cache): New type.
(ipc_kmsg_cache): Use the new type for the cache.
(ikm_cache): Drop macro.
(ikm_cache_get, ikm_cache_put): New functions.
(ikm_free): Return buffers instead of freeing them.
(_ikm_free): New version of `ikm_free' that only frees buffers.
* ipc/ipc_kmsg.c (ipc_kmsg_cache): Use new type.
(ipc_kmsg_get, ipc_kmsg_put): Use new functions.
* ipc/mach_msg.c (mach_msg_trap): Likewise.
* kern/exception.c (exception_raise): Likewise.
(exception_parse_reply): Likewise.
* kern/ipc_kobject.c (ipc_kobject_server): Likewise.
---
ipc/ipc_kmsg.c | 31 +++++---------------
ipc/ipc_kmsg.h | 83 ++++++++++++++++++++++++++++++++++++++++++++++++++++--
ipc/mach_msg.c | 14 ++++-----
kern/exception.c | 28 ++++--------------
kern/ipc_kobject.c | 6 +---
5 files changed, 99 insertions(+), 63 deletions(-)
diff --git a/ipc/ipc_kmsg.c b/ipc/ipc_kmsg.c
index 66643fd..e16709c 100644
--- a/ipc/ipc_kmsg.c
+++ b/ipc/ipc_kmsg.c
@@ -73,7 +73,9 @@
#define ptr_align(x) \
( ( ((vm_offset_t)(x)) + (sizeof(vm_offset_t)-1) ) &
~(sizeof(vm_offset_t)-1) )
-ipc_kmsg_t ipc_kmsg_cache[NCPUS];
+#include <cache.h>
+
+struct ipc_kmsg_cpu_cache ipc_kmsg_cache[NCPUS] __cacheline_aligned;
/*
* Routine: ipc_kmsg_enqueue
@@ -506,23 +508,9 @@ ipc_kmsg_get(
if ((size < sizeof(mach_msg_header_t)) || (size & 3))
return MACH_SEND_MSG_TOO_SMALL;
- if (size <= IKM_SAVED_MSG_SIZE) {
- kmsg = ikm_cache();
- if (kmsg != IKM_NULL) {
- ikm_cache() = IKM_NULL;
- ikm_check_initialized(kmsg, IKM_SAVED_KMSG_SIZE);
- } else {
- kmsg = ikm_alloc(IKM_SAVED_MSG_SIZE);
- if (kmsg == IKM_NULL)
- return MACH_SEND_NO_BUFFER;
- ikm_init(kmsg, IKM_SAVED_MSG_SIZE);
- }
- } else {
- kmsg = ikm_alloc(size);
- if (kmsg == IKM_NULL)
- return MACH_SEND_NO_BUFFER;
- ikm_init(kmsg, size);
- }
+ kmsg = ikm_cache_get (size);
+ if (kmsg == IKM_NULL)
+ return MACH_SEND_NO_BUFFER;
if (copyinmsg(msg, &kmsg->ikm_header, size)) {
ikm_free(kmsg);
@@ -599,12 +587,7 @@ ipc_kmsg_put(
else
mr = MACH_MSG_SUCCESS;
- if ((kmsg->ikm_size == IKM_SAVED_KMSG_SIZE) &&
- (ikm_cache() == IKM_NULL))
- ikm_cache() = kmsg;
- else
- ikm_free(kmsg);
-
+ ikm_cache_put (kmsg);
return mr;
}
diff --git a/ipc/ipc_kmsg.h b/ipc/ipc_kmsg.h
index 620785b..c676996 100644
--- a/ipc/ipc_kmsg.h
+++ b/ipc/ipc_kmsg.h
@@ -96,11 +96,36 @@ MACRO_END
* The per-processor cache seems to miss less than a per-thread cache,
* and it also uses less memory. Access to the cache doesn't
* require locking.
+ *
+ * The per-processor cache is a stack containing unused kernel
+ * message buffers. We choose IKM_CACHE_SIZE so that the size of
+ * the struct ipc_kmsg_cpu_cache is a multiple of a cache line,
+ * to prevent it bouncing between per-cpu caches.
+ *
+ * A kernel message buffer can be allocated using
+ * `ikm_cache_get', and returned using `ikm_cache_put'.
*/
-extern ipc_kmsg_t ipc_kmsg_cache[NCPUS];
+#define IKM_CACHE_SIZE (((1 << CPU_L1_SHIFT) - sizeof (size_t)) \
+ / sizeof (ipc_kmsg_t))
+
+struct ipc_kmsg_cpu_cache
+{
+ size_t count;
+ ipc_kmsg_t buffers[IKM_CACHE_SIZE];
+};
+
+extern struct ipc_kmsg_cpu_cache ipc_kmsg_cache[NCPUS];
+
+/* Return a kernel message buffer of at least SIZE size, preferably
+ from the cache. This functions is defined below. */
+static inline ipc_kmsg_t ikm_cache_get (mach_msg_size_t size)
+ __attribute__ ((always_inline));
-#define ikm_cache() ipc_kmsg_cache[cpu_number()]
+/* Return a kernel message buffer to the cache, or free it. This
+ functions is defined below. */
+static inline void ikm_cache_put (ipc_kmsg_t kmsg)
+ __attribute__ ((always_inline));
/*
* The size of the kernel message buffers that will be cached.
@@ -150,11 +175,18 @@ MACRO_BEGIN
\
register vm_size_t _size = (kmsg)->ikm_size; \
\
if ((integer_t)_size > 0) \
- kfree((vm_offset_t) (kmsg), _size); \
+ ikm_cache_put (kmsg); \
else \
ipc_kmsg_free(kmsg); \
MACRO_END
+#define _ikm_free(kmsg)
\
+MACRO_BEGIN \
+ vm_size_t _size = (kmsg)->ikm_size; \
+ assert (_size != IKM_SIZE_NETWORK); \
+ kfree((vm_offset_t) (kmsg), _size); \
+MACRO_END
+
/*
* struct ipc_kmsg_queue is defined in ipc/ipc_kmsg_queue.h
*/
@@ -280,5 +312,50 @@ ipc_kmsg_copyout_pseudo(ipc_kmsg_t, ipc_space_t, vm_map_t);
extern void
ipc_kmsg_copyout_dest(ipc_kmsg_t, ipc_space_t);
+
+static inline ipc_kmsg_t
+ikm_cache_get (mach_msg_size_t size)
+{
+ ipc_kmsg_t kmsg;
+ struct ipc_kmsg_cpu_cache *c = &ipc_kmsg_cache[cpu_number ()];
+
+ if (size < IKM_SAVED_MSG_SIZE)
+ size = IKM_SAVED_MSG_SIZE;
+
+ if (size > IKM_SAVED_MSG_SIZE
+ || c->count == 0) {
+ kmsg = ikm_alloc (size);
+ if (kmsg)
+ ikm_init (kmsg, size);
+ return kmsg;
+ }
+
+ c->count -= 1;
+ kmsg = c->buffers[c->count];
+
+#if MACH_IPC_TEST
+ ikm_check_initialized (kmsg, IKM_SAVED_KMSG_SIZE);
+#endif /* MACH_IPC_TEST */
+ return kmsg;
+}
+
+static inline void
+ikm_cache_put (ipc_kmsg_t kmsg)
+{
+ struct ipc_kmsg_cpu_cache *c = &ipc_kmsg_cache[cpu_number ()];
+
+ if (kmsg->ikm_size != IKM_SAVED_KMSG_SIZE
+ || c->count == IKM_CACHE_SIZE) {
+ _ikm_free (kmsg);
+ return;
+ }
+
+ c->buffers[c->count] = kmsg;
+ c->count += 1;
+
+#if MACH_IPC_TEST
+ ikm_check_initialized (kmsg, IKM_SAVED_KMSG_SIZE);
+#endif /* MACH_IPC_TEST */
+}
#endif /* _IPC_IPC_KMSG_H_ */
diff --git a/ipc/mach_msg.c b/ipc/mach_msg.c
index 1e122c7..cdf16ea 100644
--- a/ipc/mach_msg.c
+++ b/ipc/mach_msg.c
@@ -451,16 +451,14 @@ mach_msg_trap(
if ((send_size > IKM_SAVED_MSG_SIZE) ||
(send_size < sizeof(mach_msg_header_t)) ||
- (send_size & 3) ||
- ((kmsg = ikm_cache()) == IKM_NULL))
+ (send_size & 3))
goto slow_get;
- ikm_cache() = IKM_NULL;
- ikm_check_initialized(kmsg, IKM_SAVED_KMSG_SIZE);
+ kmsg = ikm_cache_get (IKM_SAVED_MSG_SIZE);
if (copyinmsg(msg, &kmsg->ikm_header,
send_size)) {
- ikm_free(kmsg);
+ ikm_cache_put (kmsg);
goto slow_get;
}
@@ -1263,18 +1261,16 @@ mach_msg_trap(
* We have the reply message data in kmsg,
* and the reply message size in reply_size.
* Just need to copy it out to the user and free kmsg.
- * We must check ikm_cache after copyoutmsg.
*/
ikm_check_initialized(kmsg, kmsg->ikm_size);
if ((kmsg->ikm_size != IKM_SAVED_KMSG_SIZE) ||
copyoutmsg(&kmsg->ikm_header, msg,
- reply_size) ||
- (ikm_cache() != IKM_NULL))
+ reply_size))
goto slow_put;
- ikm_cache() = kmsg;
+ ikm_cache_put (kmsg);
thread_syscall_return(MACH_MSG_SUCCESS);
/*NOTREACHED*/
return MACH_MSG_SUCCESS; /* help for the compiler */
diff --git a/kern/exception.c b/kern/exception.c
index 7954fba..72a4f29 100644
--- a/kern/exception.c
+++ b/kern/exception.c
@@ -348,16 +348,9 @@ exception_raise(
* and it will give the buffer back with its reply.
*/
- kmsg = ikm_cache();
- if (kmsg != IKM_NULL) {
- ikm_cache() = IKM_NULL;
- ikm_check_initialized(kmsg, IKM_SAVED_KMSG_SIZE);
- } else {
- kmsg = ikm_alloc(IKM_SAVED_MSG_SIZE);
- if (kmsg == IKM_NULL)
- panic("exception_raise");
- ikm_init(kmsg, IKM_SAVED_MSG_SIZE);
- }
+ kmsg = ikm_cache_get (IKM_SAVED_MSG_SIZE);
+ if (kmsg == IKM_NULL)
+ panic("exception_raise");
/*
* We need a reply port for the RPC.
@@ -681,22 +674,19 @@ exception_raise(
/*
* Optimized version of ipc_kmsg_put.
- * We must check ikm_cache after copyoutmsg.
*/
-
ikm_check_initialized(kmsg, kmsg->ikm_size);
assert(kmsg->ikm_size == IKM_SAVED_KMSG_SIZE);
if (copyoutmsg(&kmsg->ikm_header, receiver->ith_msg,
- sizeof(struct mach_exception)) ||
- (ikm_cache() != IKM_NULL)) {
+ sizeof(struct mach_exception))) {
mr = ipc_kmsg_put(receiver->ith_msg, kmsg,
kmsg->ikm_header.msgh_size);
thread_syscall_return(mr);
/*NOTREACHED*/
}
- ikm_cache() = kmsg;
+ ikm_cache_put (kmsg);
thread_syscall_return(MACH_MSG_SUCCESS);
/*NOTREACHED*/
#ifndef __GNUC__
@@ -809,13 +799,7 @@ exception_parse_reply(ipc_kmsg_t kmsg)
}
kr = msg->RetCode;
-
- if ((kmsg->ikm_size == IKM_SAVED_KMSG_SIZE) &&
- (ikm_cache() == IKM_NULL))
- ikm_cache() = kmsg;
- else
- ikm_free(kmsg);
-
+ ikm_cache_put (kmsg);
return kr;
}
diff --git a/kern/ipc_kobject.c b/kern/ipc_kobject.c
index bf22028..27535b0 100644
--- a/kern/ipc_kobject.c
+++ b/kern/ipc_kobject.c
@@ -236,11 +236,7 @@ ipc_kobject_server(request)
/* like ipc_kmsg_put, but without the copyout */
ikm_check_initialized(request, request->ikm_size);
- if ((request->ikm_size == IKM_SAVED_KMSG_SIZE) &&
- (ikm_cache() == IKM_NULL))
- ikm_cache() = request;
- else
- ikm_free(request);
+ ikm_cache_put (request);
} else {
/*
* The message contents of the request are intact.
--
2.1.3
