Author: kib
Date: Thu Sep 29 00:39:56 2011
New Revision: 225855
URL: http://svn.freebsd.org/changeset/base/225855
Log:
Merge the optimizations for the syscall entry and leave.
MFC r225474:
Inline the syscallenter() and syscallret(). This reduces the time measured
by the syscall entry speed microbenchmarks by ~10% on amd64.
MFC r225475:
Perform amd64-specific microoptimizations for native syscall entry
sequence. The effect is ~1% on the microbenchmark.
In particular, do not restore registers which are preserved by the
C calling sequence. Align the jump target. Avoid unneeded memory
accesses by calculating some data in syscall entry trampoline.
MFC r225483:
The jump target shall be after the padding, not into it.
MFC r225575:
Microoptimize the return path for the fast syscalls on amd64. Arrange
the code to have the fall-through path to follow the likely target.
Do not use intermediate register to reload user %rsp.
MFC r225576:
Put amd64_syscall() prototype in md_var.h.
Tested by: Alexandr Kovalenko <never nevermind kiev ua>
Added:
stable/8/sys/kern/subr_syscall.c
- copied, changed from r225474, head/sys/kern/subr_syscall.c
Modified:
stable/8/sys/amd64/amd64/exception.S
stable/8/sys/amd64/amd64/genassym.c
stable/8/sys/amd64/amd64/trap.c
stable/8/sys/amd64/ia32/ia32_syscall.c
stable/8/sys/amd64/include/md_var.h
stable/8/sys/i386/i386/trap.c
stable/8/sys/ia64/ia32/ia32_trap.c
stable/8/sys/ia64/ia64/trap.c
stable/8/sys/kern/subr_trap.c
stable/8/sys/powerpc/aim/trap.c
stable/8/sys/powerpc/booke/trap.c
stable/8/sys/sparc64/sparc64/trap.c
stable/8/sys/sun4v/sun4v/trap.c
stable/8/sys/sys/proc.h
Directory Properties:
stable/8/sys/ (props changed)
stable/8/sys/amd64/include/xen/ (props changed)
stable/8/sys/cddl/contrib/opensolaris/ (props changed)
stable/8/sys/contrib/dev/acpica/ (props changed)
stable/8/sys/contrib/pf/ (props changed)
Modified: stable/8/sys/amd64/amd64/exception.S
==============================================================================
--- stable/8/sys/amd64/amd64/exception.S Wed Sep 28 21:42:13 2011
(r225854)
+++ stable/8/sys/amd64/amd64/exception.S Thu Sep 29 00:39:56 2011
(r225855)
@@ -380,8 +380,11 @@ IDTVEC(fast_syscall)
movl $TF_HASSEGS,TF_FLAGS(%rsp)
cld
FAKE_MCOUNT(TF_RIP(%rsp))
- movq %rsp,%rdi
- call syscall
+ movq PCPU(CURTHREAD),%rdi
+ movq %rsp,TD_FRAME(%rdi)
+ movl TF_RFLAGS(%rsp),%esi
+ andl $PSL_T,%esi
+ call amd64_syscall
1: movq PCPU(CURPCB),%rax
/* Disable interrupts before testing PCB_FULL_IRET. */
cli
@@ -390,29 +393,25 @@ IDTVEC(fast_syscall)
/* Check for and handle AST's on return to userland. */
movq PCPU(CURTHREAD),%rax
testl $TDF_ASTPENDING | TDF_NEEDRESCHED,TD_FLAGS(%rax)
- je 2f
- sti
- movq %rsp, %rdi
- call ast
- jmp 1b
-2: /* Restore preserved registers. */
+ jne 2f
+ /* Restore preserved registers. */
MEXITCOUNT
movq TF_RDI(%rsp),%rdi /* bonus; preserve arg 1 */
movq TF_RSI(%rsp),%rsi /* bonus: preserve arg 2 */
movq TF_RDX(%rsp),%rdx /* return value 2 */
movq TF_RAX(%rsp),%rax /* return value 1 */
- movq TF_RBX(%rsp),%rbx /* C preserved */
- movq TF_RBP(%rsp),%rbp /* C preserved */
- movq TF_R12(%rsp),%r12 /* C preserved */
- movq TF_R13(%rsp),%r13 /* C preserved */
- movq TF_R14(%rsp),%r14 /* C preserved */
- movq TF_R15(%rsp),%r15 /* C preserved */
movq TF_RFLAGS(%rsp),%r11 /* original %rflags */
movq TF_RIP(%rsp),%rcx /* original %rip */
- movq TF_RSP(%rsp),%r9 /* user stack pointer */
- movq %r9,%rsp /* original %rsp */
+ movq TF_RSP(%rsp),%rsp /* user stack pointer */
swapgs
sysretq
+
+2: /* AST scheduled. */
+ sti
+ movq %rsp,%rdi
+ call ast
+ jmp 1b
+
3: /* Requested full context restore, use doreti for that. */
MEXITCOUNT
jmp doreti
Modified: stable/8/sys/amd64/amd64/genassym.c
==============================================================================
--- stable/8/sys/amd64/amd64/genassym.c Wed Sep 28 21:42:13 2011
(r225854)
+++ stable/8/sys/amd64/amd64/genassym.c Thu Sep 29 00:39:56 2011
(r225855)
@@ -87,6 +87,7 @@ ASSYM(TD_PCB, offsetof(struct thread, td
ASSYM(TD_PFLAGS, offsetof(struct thread, td_pflags));
ASSYM(TD_PROC, offsetof(struct thread, td_proc));
ASSYM(TD_TID, offsetof(struct thread, td_tid));
+ASSYM(TD_FRAME, offsetof(struct thread, td_frame));
ASSYM(TDF_ASTPENDING, TDF_ASTPENDING);
ASSYM(TDF_NEEDRESCHED, TDF_NEEDRESCHED);
Modified: stable/8/sys/amd64/amd64/trap.c
==============================================================================
--- stable/8/sys/amd64/amd64/trap.c Wed Sep 28 21:42:13 2011
(r225854)
+++ stable/8/sys/amd64/amd64/trap.c Thu Sep 29 00:39:56 2011
(r225855)
@@ -889,41 +889,37 @@ cpu_fetch_syscall_args(struct thread *td
return (error);
}
+#include "../../kern/subr_syscall.c"
+
/*
* syscall - system call request C handler
*
* A system call is essentially treated as a trap.
*/
void
-syscall(struct trapframe *frame)
+amd64_syscall(struct thread *td, int traced)
{
- struct thread *td;
struct syscall_args sa;
- register_t orig_tf_rflags;
int error;
ksiginfo_t ksi;
#ifdef DIAGNOSTIC
- if (ISPL(frame->tf_cs) != SEL_UPL) {
+ if (ISPL(td->td_frame->tf_cs) != SEL_UPL) {
panic("syscall");
/* NOT REACHED */
}
#endif
- orig_tf_rflags = frame->tf_rflags;
- td = curthread;
- td->td_frame = frame;
-
error = syscallenter(td, &sa);
/*
* Traced syscall.
*/
- if (orig_tf_rflags & PSL_T) {
- frame->tf_rflags &= ~PSL_T;
+ if (__predict_false(traced)) {
+ td->td_frame->tf_rflags &= ~PSL_T;
ksiginfo_init_trap(&ksi);
ksi.ksi_signo = SIGTRAP;
ksi.ksi_code = TRAP_TRACE;
- ksi.ksi_addr = (void *)frame->tf_rip;
+ ksi.ksi_addr = (void *)td->td_frame->tf_rip;
trapsignal(td, &ksi);
}
Modified: stable/8/sys/amd64/ia32/ia32_syscall.c
==============================================================================
--- stable/8/sys/amd64/ia32/ia32_syscall.c Wed Sep 28 21:42:13 2011
(r225854)
+++ stable/8/sys/amd64/ia32/ia32_syscall.c Thu Sep 29 00:39:56 2011
(r225855)
@@ -156,6 +156,8 @@ ia32_fetch_syscall_args(struct thread *t
return (error);
}
+#include "../../kern/subr_syscall.c"
+
void
ia32_syscall(struct trapframe *frame)
{
Modified: stable/8/sys/amd64/include/md_var.h
==============================================================================
--- stable/8/sys/amd64/include/md_var.h Wed Sep 28 21:42:13 2011
(r225854)
+++ stable/8/sys/amd64/include/md_var.h Thu Sep 29 00:39:56 2011
(r225855)
@@ -75,6 +75,7 @@ struct fpreg;
struct dbreg;
struct dumperinfo;
+void amd64_syscall(struct thread *td, int traced);
void busdma_swi(void);
void cpu_setregs(void);
void doreti_iret(void) __asm(__STRING(doreti_iret));
Modified: stable/8/sys/i386/i386/trap.c
==============================================================================
--- stable/8/sys/i386/i386/trap.c Wed Sep 28 21:42:13 2011
(r225854)
+++ stable/8/sys/i386/i386/trap.c Thu Sep 29 00:39:56 2011
(r225855)
@@ -1060,6 +1060,8 @@ cpu_fetch_syscall_args(struct thread *td
return (error);
}
+#include "../../kern/subr_syscall.c"
+
/*
* syscall - system call request C handler
*
Modified: stable/8/sys/ia64/ia32/ia32_trap.c
==============================================================================
--- stable/8/sys/ia64/ia32/ia32_trap.c Wed Sep 28 21:42:13 2011
(r225854)
+++ stable/8/sys/ia64/ia32/ia32_trap.c Thu Sep 29 00:39:56 2011
(r225855)
@@ -144,6 +144,8 @@ ia32_fetch_syscall_args(struct thread *t
return (error);
}
+#include "../../kern/subr_syscall.c"
+
static void
ia32_syscall(struct trapframe *tf)
{
Modified: stable/8/sys/ia64/ia64/trap.c
==============================================================================
--- stable/8/sys/ia64/ia64/trap.c Wed Sep 28 21:42:13 2011
(r225854)
+++ stable/8/sys/ia64/ia64/trap.c Thu Sep 29 00:39:56 2011
(r225855)
@@ -936,6 +936,8 @@ cpu_fetch_syscall_args(struct thread *td
return (0);
}
+#include "../../kern/subr_syscall.c"
+
/*
* Process a system call.
*
Copied and modified: stable/8/sys/kern/subr_syscall.c (from r225474,
head/sys/kern/subr_syscall.c)
==============================================================================
--- head/sys/kern/subr_syscall.c Sun Sep 11 16:05:09 2011
(r225474, copy source)
+++ stable/8/sys/kern/subr_syscall.c Thu Sep 29 00:39:56 2011
(r225855)
@@ -38,13 +38,11 @@
* from: @(#)trap.c 7.4 (Berkeley) 5/13/91
*/
-#include "opt_capsicum.h"
#include "opt_ktrace.h"
#include "opt_kdtrace.h"
__FBSDID("$FreeBSD$");
-#include <sys/capability.h>
#include <sys/ktr.h>
#ifdef KTRACE
#include <sys/uio.h>
@@ -100,22 +98,6 @@ syscallenter(struct thread *td, struct s
goto retval;
}
-#ifdef CAPABILITY_MODE
- /*
- * In capability mode, we only allow access to system calls
- * flagged with SYF_CAPENABLED.
- */
- if (IN_CAPABILITY_MODE(td) &&
- !(sa->callp->sy_flags & SYF_CAPENABLED)) {
- error = ECAPMODE;
- goto retval;
- }
-#endif
-
- error = syscall_thread_enter(td, sa->callp);
- if (error != 0)
- goto retval;
-
#ifdef KDTRACE_HOOKS
/*
* If the systrace module has registered it's probe
@@ -144,7 +126,6 @@ syscallenter(struct thread *td, struct s
(*systrace_probe_func)(sa->callp->sy_return, sa->code,
sa->callp, NULL, (error) ? -1 : td->td_retval[0]);
#endif
- syscall_thread_exit(td, sa->callp);
CTR4(KTR_SYSC, "syscall: p=%p error=%d return %#lx %#lx",
p, error, td->td_retval[0], td->td_retval[1]);
}
Modified: stable/8/sys/kern/subr_trap.c
==============================================================================
--- stable/8/sys/kern/subr_trap.c Wed Sep 28 21:42:13 2011
(r225854)
+++ stable/8/sys/kern/subr_trap.c Thu Sep 29 00:39:56 2011
(r225855)
@@ -249,7 +249,6 @@ ast(struct trapframe *framep)
mtx_assert(&Giant, MA_NOTOWNED);
}
-#ifdef HAVE_SYSCALL_ARGS_DEF
const char *
syscallname(struct proc *p, u_int code)
{
@@ -261,148 +260,3 @@ syscallname(struct proc *p, u_int code)
return (unknown);
return (sv->sv_syscallnames[code]);
}
-
-int
-syscallenter(struct thread *td, struct syscall_args *sa)
-{
- struct proc *p;
- int error, traced;
-
- PCPU_INC(cnt.v_syscall);
- p = td->td_proc;
- td->td_syscalls++;
-
- td->td_pticks = 0;
- if (td->td_ucred != p->p_ucred)
- cred_update_thread(td);
- if (p->p_flag & P_TRACED) {
- traced = 1;
- PROC_LOCK(p);
- td->td_dbgflags &= ~TDB_USERWR;
- td->td_dbgflags |= TDB_SCE;
- PROC_UNLOCK(p);
- } else
- traced = 0;
- error = (p->p_sysent->sv_fetch_syscall_args)(td, sa);
-#ifdef KTRACE
- if (KTRPOINT(td, KTR_SYSCALL))
- ktrsyscall(sa->code, sa->narg, sa->args);
-#endif
-
- CTR6(KTR_SYSC,
-"syscall: td=%p pid %d %s (%#lx, %#lx, %#lx)",
- td, td->td_proc->p_pid, syscallname(p, sa->code),
- sa->args[0], sa->args[1], sa->args[2]);
-
- if (error == 0) {
- STOPEVENT(p, S_SCE, sa->narg);
- PTRACESTOP_SC(p, td, S_PT_SCE);
- if (td->td_dbgflags & TDB_USERWR) {
- /*
- * Reread syscall number and arguments if
- * debugger modified registers or memory.
- */
- error = (p->p_sysent->sv_fetch_syscall_args)(td, sa);
-#ifdef KTRACE
- if (KTRPOINT(td, KTR_SYSCALL))
- ktrsyscall(sa->code, sa->narg, sa->args);
-#endif
- if (error != 0)
- goto retval;
- }
-
-#ifdef KDTRACE_HOOKS
- /*
- * If the systrace module has registered it's probe
- * callback and if there is a probe active for the
- * syscall 'entry', process the probe.
- */
- if (systrace_probe_func != NULL && sa->callp->sy_entry != 0)
- (*systrace_probe_func)(sa->callp->sy_entry, sa->code,
- sa->callp, sa->args, 0);
-#endif
-
- AUDIT_SYSCALL_ENTER(sa->code, td);
- error = (sa->callp->sy_call)(td, sa->args);
- AUDIT_SYSCALL_EXIT(error, td);
-
- /* Save the latest error return value. */
- td->td_errno = error;
-
-#ifdef KDTRACE_HOOKS
- /*
- * If the systrace module has registered it's probe
- * callback and if there is a probe active for the
- * syscall 'return', process the probe.
- */
- if (systrace_probe_func != NULL && sa->callp->sy_return != 0)
- (*systrace_probe_func)(sa->callp->sy_return, sa->code,
- sa->callp, NULL, (error) ? -1 : td->td_retval[0]);
-#endif
- CTR4(KTR_SYSC, "syscall: p=%p error=%d return %#lx %#lx",
- p, error, td->td_retval[0], td->td_retval[1]);
- }
- retval:
- if (traced) {
- PROC_LOCK(p);
- td->td_dbgflags &= ~TDB_SCE;
- PROC_UNLOCK(p);
- }
- (p->p_sysent->sv_set_syscall_retval)(td, error);
- return (error);
-}
-
-void
-syscallret(struct thread *td, int error, struct syscall_args *sa __unused)
-{
- struct proc *p;
- int traced;
-
- p = td->td_proc;
-
- /*
- * Check for misbehavior.
- */
- WITNESS_WARN(WARN_PANIC, NULL, "System call %s returning",
- syscallname(p, sa->code));
- KASSERT(td->td_critnest == 0,
- ("System call %s returning in a critical section",
- syscallname(p, sa->code)));
- KASSERT(td->td_locks == 0,
- ("System call %s returning with %d locks held",
- syscallname(p, sa->code), td->td_locks));
-
- /*
- * Handle reschedule and other end-of-syscall issues
- */
- userret(td, td->td_frame);
-
- CTR4(KTR_SYSC, "syscall %s exit thread %p pid %d proc %s",
- syscallname(p, sa->code), td, td->td_proc->p_pid, td->td_name);
-
-#ifdef KTRACE
- if (KTRPOINT(td, KTR_SYSRET))
- ktrsysret(sa->code, error, td->td_retval[0]);
-#endif
-
- if (p->p_flag & P_TRACED) {
- traced = 1;
- PROC_LOCK(p);
- td->td_dbgflags |= TDB_SCX;
- PROC_UNLOCK(p);
- } else
- traced = 0;
- /*
- * This works because errno is findable through the
- * register set. If we ever support an emulation where this
- * is not the case, this code will need to be revisited.
- */
- STOPEVENT(p, S_SCX, sa->code);
- PTRACESTOP_SC(p, td, S_PT_SCX);
- if (traced || (td->td_dbgflags & (TDB_EXEC | TDB_FORK)) != 0) {
- PROC_LOCK(p);
- td->td_dbgflags &= ~(TDB_SCX | TDB_EXEC | TDB_FORK);
- PROC_UNLOCK(p);
- }
-}
-#endif /* HAVE_SYSCALL_ARGS_DEF */
Modified: stable/8/sys/powerpc/aim/trap.c
==============================================================================
--- stable/8/sys/powerpc/aim/trap.c Wed Sep 28 21:42:13 2011
(r225854)
+++ stable/8/sys/powerpc/aim/trap.c Thu Sep 29 00:39:56 2011
(r225855)
@@ -394,6 +394,8 @@ cpu_fetch_syscall_args(struct thread *td
return (error);
}
+#include "../../kern/subr_syscall.c"
+
void
syscall(struct trapframe *frame)
{
Modified: stable/8/sys/powerpc/booke/trap.c
==============================================================================
--- stable/8/sys/powerpc/booke/trap.c Wed Sep 28 21:42:13 2011
(r225854)
+++ stable/8/sys/powerpc/booke/trap.c Thu Sep 29 00:39:56 2011
(r225855)
@@ -379,6 +379,8 @@ cpu_fetch_syscall_args(struct thread *td
return (error);
}
+#include "../../kern/subr_syscall.c"
+
void
syscall(struct trapframe *frame)
{
Modified: stable/8/sys/sparc64/sparc64/trap.c
==============================================================================
--- stable/8/sys/sparc64/sparc64/trap.c Wed Sep 28 21:42:13 2011
(r225854)
+++ stable/8/sys/sparc64/sparc64/trap.c Thu Sep 29 00:39:56 2011
(r225855)
@@ -614,6 +614,8 @@ cpu_fetch_syscall_args(struct thread *td
return (error);
}
+#include "../../kern/subr_syscall.c"
+
/*
* Syscall handler
* The arguments to the syscall are passed in the out registers by the caller,
Modified: stable/8/sys/sun4v/sun4v/trap.c
==============================================================================
--- stable/8/sys/sun4v/sun4v/trap.c Wed Sep 28 21:42:13 2011
(r225854)
+++ stable/8/sys/sun4v/sun4v/trap.c Thu Sep 29 00:39:56 2011
(r225855)
@@ -610,6 +610,8 @@ cpu_fetch_syscall_args(struct thread *td
return (error);
}
+#include "../../kern/subr_syscall.c"
+
/*
* Syscall handler. The arguments to the syscall are passed in the o registers
* by the caller, and are saved in the trap frame. The syscall number is passed
Modified: stable/8/sys/sys/proc.h
==============================================================================
--- stable/8/sys/sys/proc.h Wed Sep 28 21:42:13 2011 (r225854)
+++ stable/8/sys/sys/proc.h Thu Sep 29 00:39:56 2011 (r225855)
@@ -853,9 +853,6 @@ void cpu_switch(struct thread *, struct
void cpu_throw(struct thread *, struct thread *) __dead2;
void unsleep(struct thread *);
void userret(struct thread *, struct trapframe *);
-struct syscall_args;
-int syscallenter(struct thread *, struct syscall_args *);
-void syscallret(struct thread *, int, struct syscall_args *);
void cpu_exit(struct thread *);
void exit1(struct thread *, int) __dead2;
_______________________________________________
svn-src-all@freebsd.org mailing list
http://lists.freebsd.org/mailman/listinfo/svn-src-all
To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"
