Linus,
Please pull the latest x86-alternatives-for-linus git tree from:
git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git
x86-alternatives-for-linus
# HEAD: 093ae8f9a86a974c920b613860f1f7fd5bbd70ab x86/TSC: Use RDTSCP
Small RDTSCP opimization, enabled by the newly added ALTERNATIVE_3(), and
other small improvements.
Thanks,
Ingo
------------------>
Borislav Petkov (4):
x86/alternatives: Add macro comments
x86/alternatives: Print containing function
x86/alternatives: Add an ALTERNATIVE_3() macro
x86/TSC: Use RDTSCP
arch/x86/include/asm/alternative.h | 39 +++++++++++++++++++++++++++++++-------
arch/x86/include/asm/msr.h | 16 ++++++++++++++--
arch/x86/kernel/alternative.c | 4 ++--
3 files changed, 48 insertions(+), 11 deletions(-)
diff --git a/arch/x86/include/asm/alternative.h
b/arch/x86/include/asm/alternative.h
index 0660e14690c8..4c74073a19cc 100644
--- a/arch/x86/include/asm/alternative.h
+++ b/arch/x86/include/asm/alternative.h
@@ -94,13 +94,12 @@ static inline int alternatives_text_reserved(void *start,
void *end)
#define alt_total_slen alt_end_marker"b-661b"
#define alt_rlen(num) e_replacement(num)"f-"b_replacement(num)"f"
-#define __OLDINSTR(oldinstr, num) \
+#define OLDINSTR(oldinstr, num)
\
+ "# ALT: oldnstr\n" \
"661:\n\t" oldinstr "\n662:\n" \
+ "# ALT: padding\n" \
".skip -(((" alt_rlen(num) ")-(" alt_slen ")) > 0) * " \
- "((" alt_rlen(num) ")-(" alt_slen ")),0x90\n"
-
-#define OLDINSTR(oldinstr, num)
\
- __OLDINSTR(oldinstr, num) \
+ "((" alt_rlen(num) ")-(" alt_slen ")),0x90\n" \
alt_end_marker ":\n"
/*
@@ -116,11 +115,23 @@ static inline int alternatives_text_reserved(void *start,
void *end)
* additionally longer than the first replacement alternative.
*/
#define OLDINSTR_2(oldinstr, num1, num2) \
+ "# ALT: oldinstr2\n"
\
"661:\n\t" oldinstr "\n662:\n"
\
+ "# ALT: padding2\n"
\
".skip -((" alt_max_short(alt_rlen(num1), alt_rlen(num2)) " - ("
alt_slen ")) > 0) * " \
"(" alt_max_short(alt_rlen(num1), alt_rlen(num2)) " - ("
alt_slen ")), 0x90\n" \
alt_end_marker ":\n"
+#define OLDINSTR_3(oldinsn, n1, n2, n3)
\
+ "# ALT: oldinstr3\n"
\
+ "661:\n\t" oldinsn "\n662:\n"
\
+ "# ALT: padding3\n"
\
+ ".skip -((" alt_max_short(alt_max_short(alt_rlen(n1), alt_rlen(n2)),
alt_rlen(n3)) \
+ " - (" alt_slen ")) > 0) * "
\
+ "(" alt_max_short(alt_max_short(alt_rlen(n1), alt_rlen(n2)),
alt_rlen(n3)) \
+ " - (" alt_slen ")), 0x90\n"
\
+ alt_end_marker ":\n"
+
#define ALTINSTR_ENTRY(feature, num) \
" .long 661b - .\n" /* label */ \
" .long " b_replacement(num)"f - .\n" /* new instruction */ \
@@ -129,8 +140,9 @@ static inline int alternatives_text_reserved(void *start,
void *end)
" .byte " alt_rlen(num) "\n" /* replacement len */ \
" .byte " alt_pad_len "\n" /* pad len */
-#define ALTINSTR_REPLACEMENT(newinstr, feature, num) /* replacement */ \
- b_replacement(num)":\n\t" newinstr "\n" e_replacement(num) ":\n\t"
+#define ALTINSTR_REPLACEMENT(newinstr, feature, num) /* replacement */
\
+ "# ALT: replacement " #num "\n"
\
+ b_replacement(num)":\n\t" newinstr "\n" e_replacement(num) ":\n"
/* alternative assembly primitive: */
#define ALTERNATIVE(oldinstr, newinstr, feature) \
@@ -153,6 +165,19 @@ static inline int alternatives_text_reserved(void *start,
void *end)
ALTINSTR_REPLACEMENT(newinstr2, feature2, 2) \
".popsection\n"
+#define ALTERNATIVE_3(oldinsn, newinsn1, feat1, newinsn2, feat2, newinsn3,
feat3) \
+ OLDINSTR_3(oldinsn, 1, 2, 3)
\
+ ".pushsection .altinstructions,\"a\"\n"
\
+ ALTINSTR_ENTRY(feat1, 1)
\
+ ALTINSTR_ENTRY(feat2, 2)
\
+ ALTINSTR_ENTRY(feat3, 3)
\
+ ".popsection\n"
\
+ ".pushsection .altinstr_replacement, \"ax\"\n"
\
+ ALTINSTR_REPLACEMENT(newinsn1, feat1, 1)
\
+ ALTINSTR_REPLACEMENT(newinsn2, feat2, 2)
\
+ ALTINSTR_REPLACEMENT(newinsn3, feat3, 3)
\
+ ".popsection\n"
+
/*
* Alternative instructions for different CPU types or capabilities.
*
diff --git a/arch/x86/include/asm/msr.h b/arch/x86/include/asm/msr.h
index 91e4cf189914..5cc3930cb465 100644
--- a/arch/x86/include/asm/msr.h
+++ b/arch/x86/include/asm/msr.h
@@ -217,6 +217,8 @@ static __always_inline unsigned long long rdtsc(void)
*/
static __always_inline unsigned long long rdtsc_ordered(void)
{
+ DECLARE_ARGS(val, low, high);
+
/*
* The RDTSC instruction is not ordered relative to memory
* access. The Intel SDM and the AMD APM are both vague on this
@@ -227,9 +229,19 @@ static __always_inline unsigned long long
rdtsc_ordered(void)
* ordering guarantees as reading from a global memory location
* that some other imaginary CPU is updating continuously with a
* time stamp.
+ *
+ * Thus, use the preferred barrier on the respective CPU, aiming for
+ * RDTSCP as the default.
*/
- barrier_nospec();
- return rdtsc();
+ asm volatile(ALTERNATIVE_3("rdtsc",
+ "mfence; rdtsc", X86_FEATURE_MFENCE_RDTSC,
+ "lfence; rdtsc", X86_FEATURE_LFENCE_RDTSC,
+ "rdtscp", X86_FEATURE_RDTSCP)
+ : EAX_EDX_RET(val, low, high)
+ /* RDTSCP clobbers ECX with MSR_TSC_AUX. */
+ :: "ecx");
+
+ return EAX_EDX_VAL(val, low, high);
}
static inline unsigned long long native_read_pmc(int counter)
diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c
index ebeac487a20c..d458c7973c56 100644
--- a/arch/x86/kernel/alternative.c
+++ b/arch/x86/kernel/alternative.c
@@ -393,10 +393,10 @@ void __init_or_module noinline apply_alternatives(struct
alt_instr *start,
continue;
}
- DPRINTK("feat: %d*32+%d, old: (%px len: %d), repl: (%px, len:
%d), pad: %d",
+ DPRINTK("feat: %d*32+%d, old: (%pS (%px) len: %d), repl: (%px,
len: %d), pad: %d",
a->cpuid >> 5,
a->cpuid & 0x1f,
- instr, a->instrlen,
+ instr, instr, a->instrlen,
replacement, a->replacementlen, a->padlen);
DUMP_BYTES(instr, a->instrlen, "%px: old_insn: ", instr);
