Annotate generated type checks with static branch prediction
optimizing well-formed messages.
* utils.c (WriteBogusDefines): Define `mig_unlikely' if not defined.
* server.c: Use `mig_unlikely' in generated type checks.
* user.c: Likewise.
---
server.c | 21 ++++++++++++---------
user.c | 27 +++++++++++++++------------
utils.c | 9 +++++++--
3 files changed, 34 insertions(+), 23 deletions(-)
diff --git a/server.c b/server.c
index bcdeb0b..56ccfc7 100644
--- a/server.c
+++ b/server.c
@@ -402,16 +402,16 @@ WriteCheckHead(FILE *file, const routine_t *rt)
fprintf(file, "\tmsgh_simple = !(In0P->Head.msgh_bits &
MACH_MSGH_BITS_COMPLEX);\n");
if (rt->rtNumRequestVar > 0)
- fprintf(file, "\tif ((msgh_size < %d)",
+ fprintf(file, "\tif (mig_unlikely ((msgh_size < %d)",
rt->rtRequestSize);
else
- fprintf(file, "\tif ((In0P->Head.msgh_size != %d)",
+ fprintf(file, "\tif (mig_unlikely ((In0P->Head.msgh_size != %d)",
rt->rtRequestSize);
if (rt->rtSimpleCheckRequest)
fprintf(file, " ||\n\t %s(In0P->Head.msgh_bits &
MACH_MSGH_BITS_COMPLEX)",
rt->rtSimpleReceiveRequest ? "" : "!");
- fprintf(file, ")\n");
+ fprintf(file, "))\n");
WriteMsgError(file, "MIG_BAD_ARGUMENTS");
fprintf(file, "#endif\t/* TypeCheck */\n");
fprintf(file, "\n");
@@ -429,7 +429,7 @@ WriteTypeCheck(FILE *file, const argument_t *arg)
arg->argRequestPos, arg->argTTName, arg->argVarName);
else
{
- fprintf(file, "\tif (");
+ fprintf(file, "\tif (mig_unlikely (");
if (!it->itIndefinite) {
fprintf(file, "(In%dP->%s%s.msgt_inline != %s) ||\n\t ",
arg->argRequestPos, arg->argTTName,
@@ -457,7 +457,7 @@ WriteTypeCheck(FILE *file, const argument_t *arg)
arg->argRequestPos, arg->argTTName,
arg->argLongForm ? "l" : "",
it->itNumber);
- fprintf(file, "\t (In%dP->%s.msgt%s_size != %d))\n",
+ fprintf(file, "\t (In%dP->%s.msgt%s_size != %d)))\n",
arg->argRequestPos, arg->argTTName,
arg->argLongForm ? "l" : "",
it->itSize);
@@ -515,9 +515,10 @@ WriteCheckMsgSize(FILE *file, const argument_t *arg)
if (arg->argRequestPos == rt->rtMaxRequestPos)
{
fprintf(file, "#if\tTypeCheck\n");
- fprintf(file, "\tif (msgh_size != %d + (", rt->rtRequestSize);
+ fprintf(file, "\tif (mig_unlikely (msgh_size != %d + (",
+ rt->rtRequestSize);
WriteCheckArgSize(file, arg);
- fprintf(file, "))\n");
+ fprintf(file, ")))\n");
WriteMsgError(file, "MIG_BAD_ARGUMENTS");
fprintf(file, "#endif\t/* TypeCheck */\n");
@@ -545,10 +546,12 @@ WriteCheckMsgSize(FILE *file, const argument_t *arg)
it won't underflow. */
if (LastVarArg)
- fprintf(file, "\tif (msgh_size != %d + msgh_size_delta)\n",
+ fprintf(file,
+ "\tif (mig_unlikely (msgh_size != %d + msgh_size_delta))\n",
rt->rtRequestSize);
else
- fprintf(file, "\tif (msgh_size < %d + msgh_size_delta)\n",
+ fprintf(file,
+ "\tif (mig_unlikely (msgh_size < %d + msgh_size_delta))\n",
rt->rtRequestSize);
WriteMsgError(file, "MIG_BAD_ARGUMENTS");
diff --git a/user.c b/user.c
index f4a6cd5..5451a03 100644
--- a/user.c
+++ b/user.c
@@ -686,7 +686,7 @@ WriteRequestArgs(FILE *file, const routine_t *rt)
static void
WriteCheckIdentity(FILE *file, const routine_t *rt)
{
- fprintf(file, "\tif (OutP->Head.msgh_id != %d) {\n",
+ fprintf(file, "\tif (mig_unlikely (OutP->Head.msgh_id != %d)) {\n",
rt->rtNumber + SubsystemBase + 100);
fprintf(file, "\t\tif (OutP->Head.msgh_id == MACH_NOTIFY_SEND_ONCE)\n\t");
WriteMsgError(file, rt, "MIG_SERVER_DIED");
@@ -708,16 +708,17 @@ WriteCheckIdentity(FILE *file, const routine_t *rt)
fprintf(file, "\tmsgh_size = OutP->Head.msgh_size;\n\n");
fprintf(file,
- "\tif ((OutP->Head.msgh_bits & MACH_MSGH_BITS_COMPLEX) ||\n");
+ "\tif (mig_unlikely ("
+ "(OutP->Head.msgh_bits & MACH_MSGH_BITS_COMPLEX) ||\n");
if (rt->rtNoReplyArgs)
- fprintf(file, "\t (OutP->Head.msgh_size != %d))\n",
+ fprintf(file, "\t (OutP->Head.msgh_size != %d)))\n",
rt->rtReplySize);
else {
fprintf(file, "\t ((msgh_size %s %d) &&\n",
(rt->rtNumReplyVar > 0) ? "<" : "!=",
rt->rtReplySize);
fprintf(file, "\t ((msgh_size != sizeof(mig_reply_header_t))
||\n");
- fprintf(file, "\t (OutP->RetCode == KERN_SUCCESS))))\n");
+ fprintf(file, "\t (OutP->RetCode == KERN_SUCCESS)))))\n");
}
}
else {
@@ -727,7 +728,7 @@ WriteCheckIdentity(FILE *file, const routine_t *rt)
fprintf(file, "\tmsgh_simple = !(OutP->Head.msgh_bits &
MACH_MSGH_BITS_COMPLEX);\n");
fprintf(file, "\n");
- fprintf(file, "\tif (((msgh_size %s %d)",
+ fprintf(file, "\tif (mig_unlikely (((msgh_size %s %d)",
(rt->rtNumReplyVar > 0) ? "<" : "!=",
rt->rtReplySize);
@@ -740,7 +741,7 @@ WriteCheckIdentity(FILE *file, const routine_t *rt)
fprintf(file, "\t ((msgh_size != sizeof(mig_reply_header_t)) ||\n");
fprintf(file, "\t !msgh_simple ||\n");
- fprintf(file, "\t (OutP->RetCode == KERN_SUCCESS)))\n");
+ fprintf(file, "\t (OutP->RetCode == KERN_SUCCESS))))\n");
}
WriteMsgError(file, rt, "MIG_TYPE_ERROR");
fprintf(file, "#endif\t/* TypeCheck */\n");
@@ -778,7 +779,7 @@ WriteTypeCheck(FILE *file, const argument_t *arg)
}
else
{
- fprintf(file, "\tif (");
+ fprintf(file, "\tif (mig_unlikely (");
if (!it->itIndefinite) {
fprintf(file, "(OutP->%s%s.msgt_inline != %s) ||\n\t ",
arg->argTTName,
@@ -806,7 +807,7 @@ WriteTypeCheck(FILE *file, const argument_t *arg)
arg->argTTName,
arg->argLongForm ? "l" : "",
it->itNumber);
- fprintf(file, "\t (OutP->%s.msgt%s_size != %d))\n",
+ fprintf(file, "\t (OutP->%s.msgt%s_size != %d)))\n",
arg->argTTName,
arg->argLongForm ? "l" : "",
it->itSize);
@@ -861,10 +862,10 @@ WriteCheckMsgSize(FILE *file, const argument_t *arg)
if (arg->argReplyPos == rt->rtMaxReplyPos)
{
fprintf(file, "#if\tTypeCheck\n");
- fprintf(file, "\tif (msgh_size != %d + (",
+ fprintf(file, "\tif (mig_unlikely (msgh_size != %d + (",
rt->rtReplySize);
WriteCheckArgSize(file, arg);
- fprintf(file, "))\n");
+ fprintf(file, ")))\n");
WriteMsgError(file, rt, "MIG_TYPE_ERROR");
fprintf(file, "#endif\t/* TypeCheck */\n");
@@ -892,10 +893,12 @@ WriteCheckMsgSize(FILE *file, const argument_t *arg)
it won't underflow. */
if (LastVarArg)
- fprintf(file, "\tif (msgh_size != %d + msgh_size_delta)\n",
+ fprintf(file,
+ "\tif (mig_unlikely (msgh_size != %d + msgh_size_delta))\n",
rt->rtReplySize);
else
- fprintf(file, "\tif (msgh_size < %d + msgh_size_delta)\n",
+ fprintf(file,
+ "\tif (mig_unlikely (msgh_size < %d + msgh_size_delta))\n",
rt->rtReplySize);
WriteMsgError(file, rt, "MIG_TYPE_ERROR");
diff --git a/utils.c b/utils.c
index 6c42bc3..396e743 100644
--- a/utils.c
+++ b/utils.c
@@ -60,6 +60,11 @@ WriteBogusDefines(FILE *file)
fprintf(file, "#endif\n");
fprintf(file, "\n");
+ fprintf(file, "#ifndef\tmig_unlikely\n");
+ fprintf(file, "#define\tmig_unlikely(X)\t__builtin_expect (!! (X), 0)\n");
+ fprintf(file, "#endif\n");
+ fprintf(file, "\n");
+
fprintf(file, "#ifndef\tTypeCheck\n");
fprintf(file, "#define\tTypeCheck 1\n");
fprintf(file, "#endif\n");
@@ -70,11 +75,11 @@ WriteBogusDefines(FILE *file)
fprintf(file, "#endif\n");
fprintf(file, "\n");
- fprintf(file, "#define BAD_TYPECHECK(type, check) ({\\\n");
+ fprintf(file, "#define BAD_TYPECHECK(type, check) mig_unlikely (({\\\n");
fprintf(file,
" union { mach_msg_type_t t; unsigned32_t w; } _t, _c;\\\n");
fprintf(file,
- " _t.t = *(type); _c.t = *(check); _t.w != _c.w; })\n");
+ " _t.t = *(type); _c.t = *(check);_t.w != _c.w; }))\n");
}
void
--
2.1.3
