Author: mjg
Date: Fri Feb 14 13:08:46 2020
New Revision: 357911
URL: https://svnweb.freebsd.org/changeset/base/357911
Log:
Annotate branches in the syscall path
This in particular significantly shortens amd64_syscall, which otherwise
keeps jumping forward over 2KB of code in total.
Note some of these branches should be either eliminated altogether or
coalesced.
Modified:
head/sys/kern/subr_syscall.c
head/sys/kern/subr_trap.c
head/sys/security/audit/audit.h
head/sys/sys/ktrace.h
Modified: head/sys/kern/subr_syscall.c
==============================================================================
--- head/sys/kern/subr_syscall.c Fri Feb 14 12:59:27 2020
(r357910)
+++ head/sys/kern/subr_syscall.c Fri Feb 14 13:08:46 2020
(r357911)
@@ -69,7 +69,7 @@ syscallenter(struct thread *td)
if (__predict_false(td->td_cowgen != p->p_cowgen))
thread_cow_update(td);
traced = (p->p_flag & P_TRACED) != 0;
- if (traced || td->td_dbgflags & TDB_USERWR) {
+ if (__predict_false(traced || td->td_dbgflags & TDB_USERWR)) {
PROC_LOCK(p);
td->td_dbgflags &= ~TDB_USERWR;
if (traced)
@@ -85,19 +85,19 @@ syscallenter(struct thread *td)
(uintptr_t)td, "pid:%d", td->td_proc->p_pid, "arg0:%p", sa->args[0],
"arg1:%p", sa->args[1], "arg2:%p", sa->args[2]);
- if (error != 0) {
+ if (__predict_false(error != 0)) {
td->td_errno = error;
goto retval;
}
STOPEVENT(p, S_SCE, sa->narg);
- if ((p->p_flag & P_TRACED) != 0) {
+ if (__predict_false((p->p_flag & P_TRACED) != 0)) {
PROC_LOCK(p);
if (p->p_ptevents & PTRACE_SCE)
ptracestop((td), SIGTRAP, NULL);
PROC_UNLOCK(p);
}
- if ((td->td_dbgflags & TDB_USERWR) != 0) {
+ if (__predict_false((td->td_dbgflags & TDB_USERWR) != 0)) {
/*
* Reread syscall number and arguments if debugger
* modified registers or memory.
@@ -118,8 +118,8 @@ syscallenter(struct thread *td)
* In capability mode, we only allow access to system calls
* flagged with SYF_CAPENABLED.
*/
- if (IN_CAPABILITY_MODE(td) &&
- !(sa->callp->sy_flags & SYF_CAPENABLED)) {
+ if (__predict_false(IN_CAPABILITY_MODE(td) &&
+ !(sa->callp->sy_flags & SYF_CAPENABLED))) {
td->td_errno = error = ECAPMODE;
goto retval;
}
@@ -152,7 +152,7 @@ syscallenter(struct thread *td)
AUDIT_SYSCALL_EXIT(error, td);
/* Save the latest error return value. */
- if ((td->td_pflags & TDP_NERRNO) == 0)
+ if (__predict_false((td->td_pflags & TDP_NERRNO) == 0))
td->td_errno = error;
#ifdef KDTRACE_HOOKS
@@ -168,7 +168,7 @@ syscallenter(struct thread *td)
(uintptr_t)td, "pid:%d", td->td_proc->p_pid, "error:%d", error,
"retval0:%#lx", td->td_retval[0], "retval1:%#lx",
td->td_retval[1]);
- if (traced) {
+ if (__predict_false(traced)) {
PROC_LOCK(p);
td->td_dbgflags &= ~TDB_SCE;
PROC_UNLOCK(p);
@@ -189,9 +189,10 @@ syscallret(struct thread *td)
p = td->td_proc;
sa = &td->td_sa;
- if ((trap_enotcap || (p->p_flag2 & P2_TRAPCAP) != 0) &&
- IN_CAPABILITY_MODE(td)) {
- if (td->td_errno == ENOTCAPABLE || td->td_errno == ECAPMODE) {
+ if (__predict_false(td->td_errno == ENOTCAPABLE ||
+ td->td_errno == ECAPMODE)) {
+ if ((trap_enotcap ||
+ (p->p_flag2 & P2_TRAPCAP) != 0) && IN_CAPABILITY_MODE(td)) {
ksiginfo_init_trap(&ksi);
ksi.ksi_signo = SIGTRAP;
ksi.ksi_errno = td->td_errno;
@@ -211,20 +212,21 @@ syscallret(struct thread *td)
}
#endif
- if (p->p_flag & P_TRACED) {
+ traced = 0;
+ if (__predict_false(p->p_flag & P_TRACED)) {
traced = 1;
PROC_LOCK(p);
td->td_dbgflags |= TDB_SCX;
PROC_UNLOCK(p);
- } else
- traced = 0;
+ }
/*
* This works because errno is findable through the
* register set. If we ever support an emulation where this
* is not the case, this code will need to be revisited.
*/
STOPEVENT(p, S_SCX, sa->code);
- if (traced || (td->td_dbgflags & (TDB_EXEC | TDB_FORK)) != 0) {
+ if (__predict_false(traced ||
+ (td->td_dbgflags & (TDB_EXEC | TDB_FORK)) != 0)) {
PROC_LOCK(p);
/*
* If tracing the execed process, trap to the debugger
Modified: head/sys/kern/subr_trap.c
==============================================================================
--- head/sys/kern/subr_trap.c Fri Feb 14 12:59:27 2020 (r357910)
+++ head/sys/kern/subr_trap.c Fri Feb 14 13:08:46 2020 (r357911)
@@ -141,13 +141,13 @@ userret(struct thread *td, struct trapframe *frame)
* If this thread tickled GEOM, we need to wait for the giggling to
* stop before we return to userland
*/
- if (td->td_pflags & TDP_GEOM)
+ if (__predict_false(td->td_pflags & TDP_GEOM))
g_waitidle();
/*
* Charge system time if profiling.
*/
- if (p->p_flag & P_PROFIL)
+ if (__predict_false(p->p_flag & P_PROFIL))
addupc_task(td, TRAPF_PC(frame), td->td_pticks * psratio);
#ifdef HWPMC_HOOKS
Modified: head/sys/security/audit/audit.h
==============================================================================
--- head/sys/security/audit/audit.h Fri Feb 14 12:59:27 2020
(r357910)
+++ head/sys/security/audit/audit.h Fri Feb 14 13:08:46 2020
(r357911)
@@ -378,7 +378,7 @@ void audit_thread_free(struct thread *td);
} while (0)
#define AUDIT_SYSCALL_ENTER(code, td) do {
\
- if (audit_syscalls_enabled) { \
+ if (__predict_false(audit_syscalls_enabled)) { \
audit_syscall_enter(code, td); \
} \
} while (0)
Modified: head/sys/sys/ktrace.h
==============================================================================
--- head/sys/sys/ktrace.h Fri Feb 14 12:59:27 2020 (r357910)
+++ head/sys/sys/ktrace.h Fri Feb 14 13:08:46 2020 (r357911)
@@ -73,7 +73,7 @@ struct ktr_header {
#define KTRPOINT(td, type) (__predict_false(KTRCHECK((td), (type))))
#define KTRCHECKDRAIN(td) (!(STAILQ_EMPTY(&(td)->td_proc->p_ktr)))
#define KTRUSERRET(td) do {
\
- if (KTRCHECKDRAIN(td)) \
+ if (__predict_false(KTRCHECKDRAIN(td))) \
ktruserret(td); \
} while (0)
_______________________________________________
svn-src-head@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-head
To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org"
