Author: kib
Date: Sun Sep 11 16:05:09 2011
New Revision: 225474
URL: http://svn.freebsd.org/changeset/base/225474
Log:
Inline the syscallenter() and syscallret(). This reduces the time measured
by the syscall entry speed microbenchmarks by ~10% on amd64.
Submitted by: jhb
Approved by: re (bz)
MFC after: 2 weeks
Added:
head/sys/kern/subr_syscall.c
- copied, changed from r225473, head/sys/kern/subr_trap.c
Modified:
head/sys/amd64/amd64/trap.c
head/sys/amd64/ia32/ia32_syscall.c
head/sys/i386/i386/trap.c
head/sys/ia64/ia32/ia32_trap.c
head/sys/ia64/ia64/trap.c
head/sys/kern/subr_trap.c
head/sys/powerpc/aim/trap.c
head/sys/powerpc/booke/trap.c
head/sys/sparc64/sparc64/trap.c
head/sys/sys/proc.h
Modified: head/sys/amd64/amd64/trap.c
==============================================================================
--- head/sys/amd64/amd64/trap.c Sun Sep 11 09:43:13 2011 (r225473)
+++ head/sys/amd64/amd64/trap.c Sun Sep 11 16:05:09 2011 (r225474)
@@ -883,6 +883,8 @@ cpu_fetch_syscall_args(struct thread *td
return (error);
}
+#include "../../kern/subr_syscall.c"
+
/*
* syscall - system call request C handler
*
Modified: head/sys/amd64/ia32/ia32_syscall.c
==============================================================================
--- head/sys/amd64/ia32/ia32_syscall.c Sun Sep 11 09:43:13 2011
(r225473)
+++ head/sys/amd64/ia32/ia32_syscall.c Sun Sep 11 16:05:09 2011
(r225474)
@@ -163,6 +163,8 @@ ia32_fetch_syscall_args(struct thread *t
return (error);
}
+#include "../../kern/subr_syscall.c"
+
void
ia32_syscall(struct trapframe *frame)
{
Modified: head/sys/i386/i386/trap.c
==============================================================================
--- head/sys/i386/i386/trap.c Sun Sep 11 09:43:13 2011 (r225473)
+++ head/sys/i386/i386/trap.c Sun Sep 11 16:05:09 2011 (r225474)
@@ -1054,6 +1054,8 @@ cpu_fetch_syscall_args(struct thread *td
return (error);
}
+#include "../../kern/subr_syscall.c"
+
/*
* syscall - system call request C handler
*
Modified: head/sys/ia64/ia32/ia32_trap.c
==============================================================================
--- head/sys/ia64/ia32/ia32_trap.c Sun Sep 11 09:43:13 2011
(r225473)
+++ head/sys/ia64/ia32/ia32_trap.c Sun Sep 11 16:05:09 2011
(r225474)
@@ -144,6 +144,8 @@ ia32_fetch_syscall_args(struct thread *t
return (error);
}
+#include "../../kern/subr_syscall.c"
+
static void
ia32_syscall(struct trapframe *tf)
{
Modified: head/sys/ia64/ia64/trap.c
==============================================================================
--- head/sys/ia64/ia64/trap.c Sun Sep 11 09:43:13 2011 (r225473)
+++ head/sys/ia64/ia64/trap.c Sun Sep 11 16:05:09 2011 (r225474)
@@ -929,6 +929,8 @@ cpu_fetch_syscall_args(struct thread *td
return (0);
}
+#include "../../kern/subr_syscall.c"
+
/*
* Process a system call.
*
Copied and modified: head/sys/kern/subr_syscall.c (from r225473,
head/sys/kern/subr_trap.c)
==============================================================================
--- head/sys/kern/subr_trap.c Sun Sep 11 09:43:13 2011 (r225473, copy
source)
+++ head/sys/kern/subr_syscall.c Sun Sep 11 16:05:09 2011
(r225474)
@@ -2,14 +2,11 @@
* Copyright (C) 1994, David Greenman
* Copyright (c) 1990, 1993
* The Regents of the University of California. All rights reserved.
- * Copyright (c) 2007 The FreeBSD Foundation
+ * Copyright (C) 2010 Konstantin Belousov <k...@freebsd.org>
*
* This code is derived from software contributed to Berkeley by
* the University of Utah, and William Jolitz.
*
- * Portions of this software were developed by A. Joseph Koshy under
- * sponsorship from the FreeBSD Foundation and Google, Inc.
- *
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
@@ -41,231 +38,21 @@
* from: @(#)trap.c 7.4 (Berkeley) 5/13/91
*/
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
#include "opt_capsicum.h"
#include "opt_ktrace.h"
#include "opt_kdtrace.h"
-#include "opt_sched.h"
-#include <sys/param.h>
-#include <sys/bus.h>
+__FBSDID("$FreeBSD$");
+
#include <sys/capability.h>
-#include <sys/kernel.h>
-#include <sys/lock.h>
-#include <sys/mutex.h>
-#include <sys/pmckern.h>
-#include <sys/proc.h>
#include <sys/ktr.h>
-#include <sys/pioctl.h>
-#include <sys/ptrace.h>
-#include <sys/resourcevar.h>
-#include <sys/sched.h>
-#include <sys/signalvar.h>
-#include <sys/syscall.h>
-#include <sys/syscallsubr.h>
-#include <sys/sysent.h>
-#include <sys/systm.h>
-#include <sys/vmmeter.h>
#ifdef KTRACE
#include <sys/uio.h>
#include <sys/ktrace.h>
#endif
#include <security/audit/audit.h>
-#include <machine/cpu.h>
-
-#ifdef VIMAGE
-#include <net/vnet.h>
-#endif
-
-#ifdef XEN
-#include <vm/vm.h>
-#include <vm/vm_param.h>
-#include <vm/pmap.h>
-#endif
-
-#include <security/mac/mac_framework.h>
-
-/*
- * Define the code needed before returning to user mode, for trap and
- * syscall.
- */
-void
-userret(struct thread *td, struct trapframe *frame)
-{
- struct proc *p = td->td_proc;
-
- CTR3(KTR_SYSC, "userret: thread %p (pid %d, %s)", td, p->p_pid,
- td->td_name);
-#if 0
-#ifdef DIAGNOSTIC
- /* Check that we called signotify() enough. */
- PROC_LOCK(p);
- thread_lock(td);
- if (SIGPENDING(td) && ((td->td_flags & TDF_NEEDSIGCHK) == 0 ||
- (td->td_flags & TDF_ASTPENDING) == 0))
- printf("failed to set signal flags properly for ast()\n");
- thread_unlock(td);
- PROC_UNLOCK(p);
-#endif
-#endif
-#ifdef KTRACE
- KTRUSERRET(td);
-#endif
- /*
- * If this thread tickled GEOM, we need to wait for the giggling to
- * stop before we return to userland
- */
- if (td->td_pflags & TDP_GEOM)
- g_waitidle();
-
- /*
- * Charge system time if profiling.
- */
- if (p->p_flag & P_PROFIL)
- addupc_task(td, TRAPF_PC(frame), td->td_pticks * psratio);
- /*
- * Let the scheduler adjust our priority etc.
- */
- sched_userret(td);
- KASSERT(td->td_locks == 0,
- ("userret: Returning with %d locks held.", td->td_locks));
-#ifdef VIMAGE
- /* Unfortunately td_vnet_lpush needs VNET_DEBUG. */
- VNET_ASSERT(curvnet == NULL,
- ("%s: Returning on td %p (pid %d, %s) with vnet %p set in %s",
- __func__, td, p->p_pid, td->td_name, curvnet,
- (td->td_vnet_lpush != NULL) ? td->td_vnet_lpush : "N/A"));
-#endif
-#ifdef XEN
- PT_UPDATES_FLUSH();
-#endif
-}
-
-/*
- * Process an asynchronous software trap.
- * This is relatively easy.
- * This function will return with preemption disabled.
- */
-void
-ast(struct trapframe *framep)
-{
- struct thread *td;
- struct proc *p;
- int flags;
- int sig;
-
- td = curthread;
- p = td->td_proc;
-
- CTR3(KTR_SYSC, "ast: thread %p (pid %d, %s)", td, p->p_pid,
- p->p_comm);
- KASSERT(TRAPF_USERMODE(framep), ("ast in kernel mode"));
- WITNESS_WARN(WARN_PANIC, NULL, "Returning to user mode");
- mtx_assert(&Giant, MA_NOTOWNED);
- THREAD_LOCK_ASSERT(td, MA_NOTOWNED);
- td->td_frame = framep;
- td->td_pticks = 0;
-
- /*
- * This updates the td_flag's for the checks below in one
- * "atomic" operation with turning off the astpending flag.
- * If another AST is triggered while we are handling the
- * AST's saved in flags, the astpending flag will be set and
- * ast() will be called again.
- */
- thread_lock(td);
- flags = td->td_flags;
- td->td_flags &= ~(TDF_ASTPENDING | TDF_NEEDSIGCHK | TDF_NEEDSUSPCHK |
- TDF_NEEDRESCHED | TDF_ALRMPEND | TDF_PROFPEND | TDF_MACPEND);
- thread_unlock(td);
- PCPU_INC(cnt.v_trap);
-
- if (td->td_ucred != p->p_ucred)
- cred_update_thread(td);
- if (td->td_pflags & TDP_OWEUPC && p->p_flag & P_PROFIL) {
- addupc_task(td, td->td_profil_addr, td->td_profil_ticks);
- td->td_profil_ticks = 0;
- td->td_pflags &= ~TDP_OWEUPC;
- }
- if (flags & TDF_ALRMPEND) {
- PROC_LOCK(p);
- psignal(p, SIGVTALRM);
- PROC_UNLOCK(p);
- }
- if (flags & TDF_PROFPEND) {
- PROC_LOCK(p);
- psignal(p, SIGPROF);
- PROC_UNLOCK(p);
- }
-#ifdef MAC
- if (flags & TDF_MACPEND)
- mac_thread_userret(td);
-#endif
- if (flags & TDF_NEEDRESCHED) {
-#ifdef KTRACE
- if (KTRPOINT(td, KTR_CSW))
- ktrcsw(1, 1);
-#endif
- thread_lock(td);
- sched_prio(td, td->td_user_pri);
- mi_switch(SW_INVOL | SWT_NEEDRESCHED, NULL);
- thread_unlock(td);
-#ifdef KTRACE
- if (KTRPOINT(td, KTR_CSW))
- ktrcsw(0, 1);
-#endif
- }
-
- /*
- * Check for signals. Unlocked reads of p_pendingcnt or
- * p_siglist might cause process-directed signal to be handled
- * later.
- */
- if (flags & TDF_NEEDSIGCHK || p->p_pendingcnt > 0 ||
- !SIGISEMPTY(p->p_siglist)) {
- PROC_LOCK(p);
- mtx_lock(&p->p_sigacts->ps_mtx);
- while ((sig = cursig(td, SIG_STOP_ALLOWED)) != 0)
- postsig(sig);
- mtx_unlock(&p->p_sigacts->ps_mtx);
- PROC_UNLOCK(p);
- }
- /*
- * We need to check to see if we have to exit or wait due to a
- * single threading requirement or some other STOP condition.
- */
- if (flags & TDF_NEEDSUSPCHK) {
- PROC_LOCK(p);
- thread_suspend_check(0);
- PROC_UNLOCK(p);
- }
-
- if (td->td_pflags & TDP_OLDMASK) {
- td->td_pflags &= ~TDP_OLDMASK;
- kern_sigprocmask(td, SIG_SETMASK, &td->td_oldsigmask, NULL, 0);
- }
-
- userret(td, framep);
- mtx_assert(&Giant, MA_NOTOWNED);
-}
-
-#ifdef HAVE_SYSCALL_ARGS_DEF
-const char *
-syscallname(struct proc *p, u_int code)
-{
- static const char unknown[] = "unknown";
- struct sysentvec *sv;
-
- sv = p->p_sysent;
- if (sv->sv_syscallnames == NULL || code >= sv->sv_size)
- return (unknown);
- return (sv->sv_syscallnames[code]);
-}
-
-int
+static inline int
syscallenter(struct thread *td, struct syscall_args *sa)
{
struct proc *p;
@@ -371,7 +158,7 @@ syscallenter(struct thread *td, struct s
return (error);
}
-void
+static inline void
syscallret(struct thread *td, int error, struct syscall_args *sa __unused)
{
struct proc *p;
@@ -424,4 +211,3 @@ syscallret(struct thread *td, int error,
PROC_UNLOCK(p);
}
}
-#endif /* HAVE_SYSCALL_ARGS_DEF */
Modified: head/sys/kern/subr_trap.c
==============================================================================
--- head/sys/kern/subr_trap.c Sun Sep 11 09:43:13 2011 (r225473)
+++ head/sys/kern/subr_trap.c Sun Sep 11 16:05:09 2011 (r225474)
@@ -252,7 +252,6 @@ ast(struct trapframe *framep)
mtx_assert(&Giant, MA_NOTOWNED);
}
-#ifdef HAVE_SYSCALL_ARGS_DEF
const char *
syscallname(struct proc *p, u_int code)
{
@@ -264,164 +263,3 @@ syscallname(struct proc *p, u_int code)
return (unknown);
return (sv->sv_syscallnames[code]);
}
-
-int
-syscallenter(struct thread *td, struct syscall_args *sa)
-{
- struct proc *p;
- int error, traced;
-
- PCPU_INC(cnt.v_syscall);
- p = td->td_proc;
-
- td->td_pticks = 0;
- if (td->td_ucred != p->p_ucred)
- cred_update_thread(td);
- if (p->p_flag & P_TRACED) {
- traced = 1;
- PROC_LOCK(p);
- td->td_dbgflags &= ~TDB_USERWR;
- td->td_dbgflags |= TDB_SCE;
- PROC_UNLOCK(p);
- } else
- traced = 0;
- error = (p->p_sysent->sv_fetch_syscall_args)(td, sa);
-#ifdef KTRACE
- if (KTRPOINT(td, KTR_SYSCALL))
- ktrsyscall(sa->code, sa->narg, sa->args);
-#endif
-
- CTR6(KTR_SYSC,
-"syscall: td=%p pid %d %s (%#lx, %#lx, %#lx)",
- td, td->td_proc->p_pid, syscallname(p, sa->code),
- sa->args[0], sa->args[1], sa->args[2]);
-
- if (error == 0) {
- STOPEVENT(p, S_SCE, sa->narg);
- PTRACESTOP_SC(p, td, S_PT_SCE);
- if (td->td_dbgflags & TDB_USERWR) {
- /*
- * Reread syscall number and arguments if
- * debugger modified registers or memory.
- */
- error = (p->p_sysent->sv_fetch_syscall_args)(td, sa);
-#ifdef KTRACE
- if (KTRPOINT(td, KTR_SYSCALL))
- ktrsyscall(sa->code, sa->narg, sa->args);
-#endif
- if (error != 0)
- goto retval;
- }
-
-#ifdef CAPABILITY_MODE
- /*
- * In capability mode, we only allow access to system calls
- * flagged with SYF_CAPENABLED.
- */
- if (IN_CAPABILITY_MODE(td) &&
- !(sa->callp->sy_flags & SYF_CAPENABLED)) {
- error = ECAPMODE;
- goto retval;
- }
-#endif
-
- error = syscall_thread_enter(td, sa->callp);
- if (error != 0)
- goto retval;
-
-#ifdef KDTRACE_HOOKS
- /*
- * If the systrace module has registered it's probe
- * callback and if there is a probe active for the
- * syscall 'entry', process the probe.
- */
- if (systrace_probe_func != NULL && sa->callp->sy_entry != 0)
- (*systrace_probe_func)(sa->callp->sy_entry, sa->code,
- sa->callp, sa->args, 0);
-#endif
-
- AUDIT_SYSCALL_ENTER(sa->code, td);
- error = (sa->callp->sy_call)(td, sa->args);
- AUDIT_SYSCALL_EXIT(error, td);
-
- /* Save the latest error return value. */
- td->td_errno = error;
-
-#ifdef KDTRACE_HOOKS
- /*
- * If the systrace module has registered it's probe
- * callback and if there is a probe active for the
- * syscall 'return', process the probe.
- */
- if (systrace_probe_func != NULL && sa->callp->sy_return != 0)
- (*systrace_probe_func)(sa->callp->sy_return, sa->code,
- sa->callp, NULL, (error) ? -1 : td->td_retval[0]);
-#endif
- syscall_thread_exit(td, sa->callp);
- CTR4(KTR_SYSC, "syscall: p=%p error=%d return %#lx %#lx",
- p, error, td->td_retval[0], td->td_retval[1]);
- }
- retval:
- if (traced) {
- PROC_LOCK(p);
- td->td_dbgflags &= ~TDB_SCE;
- PROC_UNLOCK(p);
- }
- (p->p_sysent->sv_set_syscall_retval)(td, error);
- return (error);
-}
-
-void
-syscallret(struct thread *td, int error, struct syscall_args *sa __unused)
-{
- struct proc *p;
- int traced;
-
- p = td->td_proc;
-
- /*
- * Check for misbehavior.
- */
- WITNESS_WARN(WARN_PANIC, NULL, "System call %s returning",
- syscallname(p, sa->code));
- KASSERT(td->td_critnest == 0,
- ("System call %s returning in a critical section",
- syscallname(p, sa->code)));
- KASSERT(td->td_locks == 0,
- ("System call %s returning with %d locks held",
- syscallname(p, sa->code), td->td_locks));
-
- /*
- * Handle reschedule and other end-of-syscall issues
- */
- userret(td, td->td_frame);
-
- CTR4(KTR_SYSC, "syscall %s exit thread %p pid %d proc %s",
- syscallname(p, sa->code), td, td->td_proc->p_pid, td->td_name);
-
-#ifdef KTRACE
- if (KTRPOINT(td, KTR_SYSRET))
- ktrsysret(sa->code, error, td->td_retval[0]);
-#endif
-
- if (p->p_flag & P_TRACED) {
- traced = 1;
- PROC_LOCK(p);
- td->td_dbgflags |= TDB_SCX;
- PROC_UNLOCK(p);
- } else
- traced = 0;
- /*
- * This works because errno is findable through the
- * register set. If we ever support an emulation where this
- * is not the case, this code will need to be revisited.
- */
- STOPEVENT(p, S_SCX, sa->code);
- PTRACESTOP_SC(p, td, S_PT_SCX);
- if (traced || (td->td_dbgflags & (TDB_EXEC | TDB_FORK)) != 0) {
- PROC_LOCK(p);
- td->td_dbgflags &= ~(TDB_SCX | TDB_EXEC | TDB_FORK);
- PROC_UNLOCK(p);
- }
-}
-#endif /* HAVE_SYSCALL_ARGS_DEF */
Modified: head/sys/powerpc/aim/trap.c
==============================================================================
--- head/sys/powerpc/aim/trap.c Sun Sep 11 09:43:13 2011 (r225473)
+++ head/sys/powerpc/aim/trap.c Sun Sep 11 16:05:09 2011 (r225474)
@@ -445,6 +445,8 @@ cpu_fetch_syscall_args(struct thread *td
return (error);
}
+#include "../../kern/subr_syscall.c"
+
void
syscall(struct trapframe *frame)
{
Modified: head/sys/powerpc/booke/trap.c
==============================================================================
--- head/sys/powerpc/booke/trap.c Sun Sep 11 09:43:13 2011
(r225473)
+++ head/sys/powerpc/booke/trap.c Sun Sep 11 16:05:09 2011
(r225474)
@@ -375,6 +375,8 @@ cpu_fetch_syscall_args(struct thread *td
return (error);
}
+#include "../../kern/subr_syscall.c"
+
void
syscall(struct trapframe *frame)
{
Modified: head/sys/sparc64/sparc64/trap.c
==============================================================================
--- head/sys/sparc64/sparc64/trap.c Sun Sep 11 09:43:13 2011
(r225473)
+++ head/sys/sparc64/sparc64/trap.c Sun Sep 11 16:05:09 2011
(r225474)
@@ -606,6 +606,8 @@ cpu_fetch_syscall_args(struct thread *td
return (error);
}
+#include "../../kern/subr_syscall.c"
+
/*
* Syscall handler
* The arguments to the syscall are passed in the out registers by the caller,
Modified: head/sys/sys/proc.h
==============================================================================
--- head/sys/sys/proc.h Sun Sep 11 09:43:13 2011 (r225473)
+++ head/sys/sys/proc.h Sun Sep 11 16:05:09 2011 (r225474)
@@ -873,9 +873,6 @@ void cpu_switch(struct thread *, struct
void cpu_throw(struct thread *, struct thread *) __dead2;
void unsleep(struct thread *);
void userret(struct thread *, struct trapframe *);
-struct syscall_args;
-int syscallenter(struct thread *, struct syscall_args *);
-void syscallret(struct thread *, int, struct syscall_args *);
void cpu_exit(struct thread *);
void exit1(struct thread *, int) __dead2;
_______________________________________________
svn-src-all@freebsd.org mailing list
http://lists.freebsd.org/mailman/listinfo/svn-src-all
To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"
