Author: kib Date: Sat Oct 20 23:17:24 2018 New Revision: 339507 URL: https://svnweb.freebsd.org/changeset/base/339507
Log: amd64: flush L1 data cache on syscall return with an error. The knob allows to select the flushing mode or turn it off/on. The idea, as well as the list of the ignored syscall errors, were taken from https://www.openwall.com/lists/kernel-hardening/2018/10/11/10 . I was not able to measure statistically significant difference between flush enabled vs disabled using syscall_timing getuid. Reviewed by: bwidawsk Sponsored by: The FreeBSD Foundation MFC after: 1 week Differential revision: https://reviews.freebsd.org/D17536 Modified: head/sys/amd64/amd64/initcpu.c head/sys/amd64/amd64/machdep.c head/sys/amd64/amd64/support.S head/sys/amd64/amd64/trap.c head/sys/amd64/ia32/ia32_syscall.c head/sys/amd64/include/md_var.h head/sys/dev/cpuctl/cpuctl.c Modified: head/sys/amd64/amd64/initcpu.c ============================================================================== --- head/sys/amd64/amd64/initcpu.c Sat Oct 20 23:08:04 2018 (r339506) +++ head/sys/amd64/amd64/initcpu.c Sat Oct 20 23:17:24 2018 (r339507) @@ -253,6 +253,7 @@ initializecpu(void) } hw_ibrs_recalculate(); hw_ssb_recalculate(false); + amd64_syscall_ret_flush_l1d_recalc(); switch (cpu_vendor_id) { case CPU_VENDOR_AMD: init_amd(); Modified: head/sys/amd64/amd64/machdep.c ============================================================================== --- head/sys/amd64/amd64/machdep.c Sat Oct 20 23:08:04 2018 (r339506) +++ head/sys/amd64/amd64/machdep.c Sat Oct 20 23:17:24 2018 (r339507) @@ -1722,6 +1722,11 @@ hammer_time(u_int64_t modulep, u_int64_t physfree) != NULL) vty_set_preferred(VTY_VT); + TUNABLE_INT_FETCH("hw.ibrs_disable", &hw_ibrs_disable); + TUNABLE_INT_FETCH("hw.spec_store_bypass_disable", &hw_ssb_disable); + TUNABLE_INT_FETCH("machdep.syscall_ret_l1d_flush", + &syscall_ret_l1d_flush_mode); + finishidentcpu(); /* Final stage of CPU initialization */ initializecpu(); /* Initialize CPU registers */ @@ -1864,9 +1869,6 @@ hammer_time(u_int64_t modulep, u_int64_t physfree) x86_init_fdt(); #endif thread0.td_critnest = 0; - - TUNABLE_INT_FETCH("hw.ibrs_disable", &hw_ibrs_disable); - TUNABLE_INT_FETCH("hw.spec_store_bypass_disable", &hw_ssb_disable); TSEXIT(); Modified: head/sys/amd64/amd64/support.S ============================================================================== --- head/sys/amd64/amd64/support.S Sat Oct 20 23:08:04 2018 (r339506) +++ head/sys/amd64/amd64/support.S Sat Oct 20 23:17:24 2018 (r339507) @@ -1556,3 +1556,10 @@ ENTRY(flush_l1d_sw) ret #undef L1D_FLUSH_SIZE END(flush_l1d_sw) + +ENTRY(flush_l1d_sw_abi) + pushq %rbx + call flush_l1d_sw + popq %rbx + ret +END(flush_l1d_sw_abi) Modified: head/sys/amd64/amd64/trap.c ============================================================================== --- head/sys/amd64/amd64/trap.c Sat Oct 20 23:08:04 2018 (r339506) +++ head/sys/amd64/amd64/trap.c Sat Oct 20 23:17:24 2018 (r339507) @@ -1056,6 +1056,84 @@ cpu_fetch_syscall_args(struct thread *td) #include "../../kern/subr_syscall.c" +static void (*syscall_ret_l1d_flush)(void); +int syscall_ret_l1d_flush_mode; + +static void +flush_l1d_hw(void) +{ + + wrmsr(MSR_IA32_FLUSH_CMD, IA32_FLUSH_CMD_L1D); +} + +static void __inline +amd64_syscall_ret_flush_l1d_inline(int error) +{ + void (*p)(void); + + if (error != 0 && error != EEXIST && error != EAGAIN && + error != EXDEV && error != ENOENT && error != ENOTCONN && + error != EINPROGRESS) { + p = syscall_ret_l1d_flush; + if (p != NULL) + p(); + } +} + +void +amd64_syscall_ret_flush_l1d(int error) +{ + + amd64_syscall_ret_flush_l1d_inline(error); +} + +void +amd64_syscall_ret_flush_l1d_recalc(void) +{ + bool l1d_hw; + + l1d_hw = (cpu_stdext_feature3 & CPUID_STDEXT3_L1D_FLUSH) != 0; +again: + switch (syscall_ret_l1d_flush_mode) { + case 0: + syscall_ret_l1d_flush = NULL; + break; + case 1: + syscall_ret_l1d_flush = l1d_hw ? flush_l1d_hw : + flush_l1d_sw_abi; + break; + case 2: + syscall_ret_l1d_flush = l1d_hw ? flush_l1d_hw : NULL; + break; + case 3: + syscall_ret_l1d_flush = flush_l1d_sw_abi; + break; + default: + syscall_ret_l1d_flush_mode = 1; + goto again; + } +} + +static int +machdep_syscall_ret_flush_l1d(SYSCTL_HANDLER_ARGS) +{ + int error, val; + + val = syscall_ret_l1d_flush_mode; + error = sysctl_handle_int(oidp, &val, 0, req); + if (error != 0 || req->newptr == NULL) + return (error); + syscall_ret_l1d_flush_mode = val; + amd64_syscall_ret_flush_l1d_recalc(); + return (0); +} +SYSCTL_PROC(_machdep, OID_AUTO, syscall_ret_flush_l1d, CTLTYPE_INT | + CTLFLAG_RWTUN | CTLFLAG_NOFETCH | CTLFLAG_MPSAFE, NULL, 0, + machdep_syscall_ret_flush_l1d, "I", + "Flush L1D on syscall return with error (0 - off, 1 - on, " + "2 - use hw only, 3 - use sw only"); + + /* * System call handler for native binaries. The trap frame is already * set up by the assembler trampoline and a pointer to it is saved in @@ -1110,4 +1188,6 @@ amd64_syscall(struct thread *td, int traced) */ if (__predict_false(td->td_frame->tf_rip >= VM_MAXUSER_ADDRESS)) set_pcb_flags(td->td_pcb, PCB_FULL_IRET); + + amd64_syscall_ret_flush_l1d_inline(error); } Modified: head/sys/amd64/ia32/ia32_syscall.c ============================================================================== --- head/sys/amd64/ia32/ia32_syscall.c Sat Oct 20 23:08:04 2018 (r339506) +++ head/sys/amd64/ia32/ia32_syscall.c Sat Oct 20 23:17:24 2018 (r339507) @@ -231,6 +231,7 @@ ia32_syscall(struct trapframe *frame) } syscallret(td, error); + amd64_syscall_ret_flush_l1d(error); } static void Modified: head/sys/amd64/include/md_var.h ============================================================================== --- head/sys/amd64/include/md_var.h Sat Oct 20 23:08:04 2018 (r339506) +++ head/sys/amd64/include/md_var.h Sat Oct 20 23:17:24 2018 (r339507) @@ -41,6 +41,7 @@ extern int hw_lower_amd64_sharedpage; extern int hw_ibrs_disable; extern int hw_ssb_disable; extern int nmi_flush_l1d_sw; +extern int syscall_ret_l1d_flush_mode; /* * The file "conf/ldscript.amd64" defines the symbol "kernphys". Its @@ -55,8 +56,11 @@ void amd64_conf_fast_syscall(void); void amd64_db_resume_dbreg(void); void amd64_lower_shared_page(struct sysentvec *); void amd64_syscall(struct thread *td, int traced); +void amd64_syscall_ret_flush_l1d(int error); +void amd64_syscall_ret_flush_l1d_recalc(void); void doreti_iret(void) __asm(__STRING(doreti_iret)); void doreti_iret_fault(void) __asm(__STRING(doreti_iret_fault)); +void flush_l1d_sw_abi(void); void ld_ds(void) __asm(__STRING(ld_ds)); void ld_es(void) __asm(__STRING(ld_es)); void ld_fs(void) __asm(__STRING(ld_fs)); Modified: head/sys/dev/cpuctl/cpuctl.c ============================================================================== --- head/sys/dev/cpuctl/cpuctl.c Sat Oct 20 23:08:04 2018 (r339506) +++ head/sys/dev/cpuctl/cpuctl.c Sat Oct 20 23:17:24 2018 (r339507) @@ -521,6 +521,9 @@ cpuctl_do_eval_cpu_features(int cpu, struct thread *td hw_ibrs_recalculate(); restore_cpu(oldcpu, is_bound, td); hw_ssb_recalculate(true); +#ifdef __amd64__ + amd64_syscall_ret_flush_l1d_recalc(); +#endif printcpuinfo(); return (0); } _______________________________________________ svn-src-all@freebsd.org mailing list https://lists.freebsd.org/mailman/listinfo/svn-src-all To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"
