Author: tychon Date: Fri Apr 27 12:44:20 2018 New Revision: 333059 URL: https://svnweb.freebsd.org/changeset/base/333059
Log: Expand the checks for UCR3 == PMAP_NO_CR3 to enable processes to be excluded from PTI. Reviewed by: kib Sponsored by: Dell EMC Isilon Differential Revision: https://reviews.freebsd.org/D15100 Modified: head/sys/amd64/amd64/cpu_switch.S head/sys/amd64/amd64/exception.S head/sys/amd64/amd64/genassym.c head/sys/amd64/amd64/machdep.c head/sys/amd64/amd64/mp_machdep.c head/sys/amd64/amd64/pmap.c head/sys/amd64/amd64/trap.c head/sys/amd64/include/asmacros.h head/sys/amd64/include/pcpu.h Modified: head/sys/amd64/amd64/cpu_switch.S ============================================================================== --- head/sys/amd64/amd64/cpu_switch.S Fri Apr 27 12:03:27 2018 (r333058) +++ head/sys/amd64/amd64/cpu_switch.S Fri Apr 27 12:44:20 2018 (r333059) @@ -215,10 +215,13 @@ done_tss: movq %r8,PCPU(RSP0) movq %r8,PCPU(CURPCB) /* Update the TSS_RSP0 pointer for the next interrupt */ - cmpb $0,pti(%rip) - jne 1f - movq %r8,TSS_RSP0(%rdx) -1: movq %r12,PCPU(CURTHREAD) /* into next thread */ + cmpq $~0,PCPU(UCR3) + je 1f + movq PCPU(PTI_RSP0),%rax + movq %rax,TSS_RSP0(%rdx) + jmp 2f +1: movq %r8,TSS_RSP0(%rdx) +2: movq %r12,PCPU(CURTHREAD) /* into next thread */ /* Test if debug registers should be restored. */ testl $PCB_DBREGS,PCB_FLAGS(%r8) @@ -295,12 +298,7 @@ do_tss: movq %rdx,PCPU(TSSP) shrq $8,%rcx movl %ecx,8(%rax) movb $0x89,5(%rax) /* unset busy */ - cmpb $0,pti(%rip) - je 1f - movq PCPU(PRVSPACE),%rax - addq $PC_PTI_STACK+PC_PTI_STACK_SZ*8,%rax - movq %rax,TSS_RSP0(%rdx) -1: movl $TSSSEL,%eax + movl $TSSSEL,%eax ltr %ax jmp done_tss Modified: head/sys/amd64/amd64/exception.S ============================================================================== --- head/sys/amd64/amd64/exception.S Fri Apr 27 12:03:27 2018 (r333058) +++ head/sys/amd64/amd64/exception.S Fri Apr 27 12:44:20 2018 (r333059) @@ -298,11 +298,15 @@ IDTVEC(page_pti) jz Xpage swapgs pushq %rax - pushq %rdx movq %cr3,%rax movq %rax,PCPU(SAVED_UCR3) + cmpq $~0,PCPU(UCR3) + jne 1f + popq %rax + jmp 2f +1: pushq %rdx PTI_UUENTRY has_err=1 - subq $TF_ERR,%rsp +2: subq $TF_ERR,%rsp movq %rdi,TF_RDI(%rsp) movq %rax,TF_RAX(%rsp) movq %rdx,TF_RDX(%rsp) @@ -347,9 +351,11 @@ page_cr2: */ .macro PROTF_ENTRY name,trapno \name\()_pti_doreti: + swapgs + cmpq $~0,PCPU(UCR3) + je 1f pushq %rax pushq %rdx - swapgs movq PCPU(KCR3),%rax movq %rax,%cr3 movq PCPU(RSP0),%rax @@ -362,7 +368,7 @@ page_cr2: movq %rax,%rsp popq %rdx popq %rax - swapgs +1: swapgs jmp X\name IDTVEC(\name\()_pti) cmpq $doreti_iret,PTI_RIP-2*8(%rsp) @@ -438,6 +444,8 @@ prot_addrf: IDTVEC(fast_syscall_pti) swapgs movq %rax,PCPU(SCRATCH_RAX) + cmpq $~0,PCPU(UCR3) + je fast_syscall_common movq PCPU(KCR3),%rax movq %rax,%cr3 jmp fast_syscall_common @@ -503,7 +511,7 @@ fast_syscall_common: movq TF_RFLAGS(%rsp),%r11 /* original %rflags */ movq TF_RIP(%rsp),%rcx /* original %rip */ movq TF_RSP(%rsp),%rsp /* user stack pointer */ - cmpb $0,pti + cmpq $~0,PCPU(UCR3) je 2f movq PCPU(UCR3),%r9 movq %r9,%cr3 @@ -1011,11 +1019,11 @@ ld_regs: jz 2f /* keep running with kernel GS.base */ cli call handle_ibrs_exit_rs - cmpb $0,pti + cmpq $~0,PCPU(UCR3) je 1f pushq %rdx - movq PCPU(PRVSPACE),%rdx - addq $PC_PTI_STACK+PC_PTI_STACK_SZ*8-PTI_SIZE,%rdx + movq PCPU(PTI_RSP0),%rdx + subq $PTI_SIZE,%rdx movq %rax,PTI_RAX(%rdx) popq %rax movq %rax,PTI_RDX(%rdx) Modified: head/sys/amd64/amd64/genassym.c ============================================================================== --- head/sys/amd64/amd64/genassym.c Fri Apr 27 12:03:27 2018 (r333058) +++ head/sys/amd64/amd64/genassym.c Fri Apr 27 12:44:20 2018 (r333059) @@ -227,6 +227,7 @@ ASSYM(PC_UCR3, offsetof(struct pcpu, pc_ucr3)); ASSYM(PC_SAVED_UCR3, offsetof(struct pcpu, pc_saved_ucr3)); ASSYM(PC_PTI_STACK, offsetof(struct pcpu, pc_pti_stack)); ASSYM(PC_PTI_STACK_SZ, PC_PTI_STACK_SZ); +ASSYM(PC_PTI_RSP0, offsetof(struct pcpu, pc_pti_rsp0)); ASSYM(PC_IBPB_SET, offsetof(struct pcpu, pc_ibpb_set)); ASSYM(LA_EOI, LAPIC_EOI * LAPIC_MEM_MUL); Modified: head/sys/amd64/amd64/machdep.c ============================================================================== --- head/sys/amd64/amd64/machdep.c Fri Apr 27 12:03:27 2018 (r333058) +++ head/sys/amd64/amd64/machdep.c Fri Apr 27 12:44:20 2018 (r333059) @@ -1795,9 +1795,10 @@ hammer_time(u_int64_t modulep, u_int64_t physfree) rsp0 = (vm_offset_t)thread0.td_pcb; /* Ensure the stack is aligned to 16 bytes */ rsp0 &= ~0xFul; - common_tss[0].tss_rsp0 = pti ? ((vm_offset_t)PCPU_PTR(pti_stack) + - PC_PTI_STACK_SZ * sizeof(uint64_t)) & ~0xful : rsp0; + common_tss[0].tss_rsp0 = rsp0; PCPU_SET(rsp0, rsp0); + PCPU_SET(pti_rsp0, ((vm_offset_t)PCPU_PTR(pti_stack) + + PC_PTI_STACK_SZ * sizeof(uint64_t)) & ~0xful); PCPU_SET(curpcb, thread0.td_pcb); /* transfer to user mode */ Modified: head/sys/amd64/amd64/mp_machdep.c ============================================================================== --- head/sys/amd64/amd64/mp_machdep.c Fri Apr 27 12:03:27 2018 (r333058) +++ head/sys/amd64/amd64/mp_machdep.c Fri Apr 27 12:44:20 2018 (r333059) @@ -276,6 +276,8 @@ init_secondary(void) pc->pc_tssp = &common_tss[cpu]; pc->pc_commontssp = &common_tss[cpu]; pc->pc_rsp0 = 0; + pc->pc_pti_rsp0 = ((vm_offset_t)&pc->pc_pti_stack + + PC_PTI_STACK_SZ * sizeof(uint64_t) & ~0xful); pc->pc_tss = (struct system_segment_descriptor *)&gdt[NGDT * cpu + GPROC0_SEL]; pc->pc_fs32p = &gdt[NGDT * cpu + GUFS32_SEL]; @@ -285,8 +287,7 @@ init_secondary(void) pc->pc_curpmap = kernel_pmap; pc->pc_pcid_gen = 1; pc->pc_pcid_next = PMAP_PCID_KERN + 1; - common_tss[cpu].tss_rsp0 = pti ? ((vm_offset_t)&pc->pc_pti_stack + - PC_PTI_STACK_SZ * sizeof(uint64_t)) & ~0xful : 0; + common_tss[cpu].tss_rsp0 = 0; /* Save the per-cpu pointer for use by the NMI handler. */ np = ((struct nmi_pcpu *) &nmi_stack[PAGE_SIZE]) - 1; Modified: head/sys/amd64/amd64/pmap.c ============================================================================== --- head/sys/amd64/amd64/pmap.c Fri Apr 27 12:03:27 2018 (r333058) +++ head/sys/amd64/amd64/pmap.c Fri Apr 27 12:44:20 2018 (r333059) @@ -2607,8 +2607,10 @@ pmap_pinit0(pmap_t pmap) CPU_FOREACH(i) { pmap->pm_pcids[i].pm_pcid = PMAP_PCID_NONE; pmap->pm_pcids[i].pm_gen = 0; - if (!pti) + if (!pti) { __pcpu[i].pc_kcr3 = PMAP_NO_CR3; + __pcpu[i].pc_ucr3 = PMAP_NO_CR3; + } } PCPU_SET(curpmap, kernel_pmap); pmap_activate(curthread); @@ -2783,7 +2785,8 @@ _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, str * the kernel-mode page table active on return * to user space. */ - *pml4 |= pg_nx; + if (pmap->pm_ucr3 != PMAP_NO_CR3) + *pml4 |= pg_nx; pml4u = &pmap->pm_pml4u[pml4index]; *pml4u = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | @@ -7359,9 +7362,10 @@ pmap_activate_sw(struct thread *td) { pmap_t oldpmap, pmap; struct invpcid_descr d; - uint64_t cached, cr3, kcr3, kern_pti_cached, ucr3; + uint64_t cached, cr3, kcr3, kern_pti_cached, rsp0, ucr3; register_t rflags; u_int cpuid; + struct amd64tss *tssp; oldpmap = PCPU_GET(curpmap); pmap = vmspace_pmap(td->td_proc->p_vmspace); @@ -7451,6 +7455,12 @@ pmap_activate_sw(struct thread *td) PCPU_SET(kcr3, pmap->pm_cr3); PCPU_SET(ucr3, pmap->pm_ucr3); } + } + if (pmap->pm_ucr3 != PMAP_NO_CR3) { + rsp0 = ((vm_offset_t)PCPU_PTR(pti_stack) + + PC_PTI_STACK_SZ * sizeof(uint64_t)) & ~0xful; + tssp = PCPU_GET(tssp); + tssp->tss_rsp0 = rsp0; } #ifdef SMP CPU_CLR_ATOMIC(cpuid, &oldpmap->pm_active); Modified: head/sys/amd64/amd64/trap.c ============================================================================== --- head/sys/amd64/amd64/trap.c Fri Apr 27 12:03:27 2018 (r333058) +++ head/sys/amd64/amd64/trap.c Fri Apr 27 12:44:20 2018 (r333059) @@ -461,11 +461,13 @@ trap(struct trapframe *frame) */ if (frame->tf_rip == (long)doreti_iret) { frame->tf_rip = (long)doreti_iret_fault; - if (pti && frame->tf_rsp == (uintptr_t)PCPU_PTR( - pti_stack) + (PC_PTI_STACK_SZ - 5) * - sizeof(register_t)) + if ((PCPU_GET(curpmap)->pm_ucr3 != + PMAP_NO_CR3) && + (frame->tf_rsp == (uintptr_t)PCPU_GET( + pti_rsp0) - 5 * sizeof(register_t))) { frame->tf_rsp = PCPU_GET(rsp0) - 5 * sizeof(register_t); + } return; } if (frame->tf_rip == (long)ld_ds) { Modified: head/sys/amd64/include/asmacros.h ============================================================================== --- head/sys/amd64/include/asmacros.h Fri Apr 27 12:03:27 2018 (r333058) +++ head/sys/amd64/include/asmacros.h Fri Apr 27 12:44:20 2018 (r333059) @@ -196,9 +196,12 @@ .macro PTI_UENTRY has_err swapgs + cmpq $~0,PCPU(UCR3) + je 1f pushq %rax pushq %rdx PTI_UUENTRY \has_err +1: .endm .macro PTI_ENTRY name, cont, has_err=0 Modified: head/sys/amd64/include/pcpu.h ============================================================================== --- head/sys/amd64/include/pcpu.h Fri Apr 27 12:03:27 2018 (r333058) +++ head/sys/amd64/include/pcpu.h Fri Apr 27 12:44:20 2018 (r333059) @@ -68,14 +68,15 @@ uint64_t pc_pm_save_cnt; \ u_int pc_cmci_mask; /* MCx banks for CMCI */ \ uint64_t pc_dbreg[16]; /* ddb debugging regs */ \ - uint64_t pc_pti_stack[PC_PTI_STACK_SZ]; \ + uint64_t pc_pti_stack[PC_PTI_STACK_SZ]; \ + register_t pc_pti_rsp0; \ int pc_dbreg_cmd; /* ddb debugging reg cmd */ \ u_int pc_vcpu_id; /* Xen vCPU ID */ \ uint32_t pc_pcid_next; \ uint32_t pc_pcid_gen; \ uint32_t pc_smp_tlb_done; /* TLB op acknowledgement */ \ uint32_t pc_ibpb_set; \ - char __pad[224] /* be divisor of PAGE_SIZE \ + char __pad[216] /* be divisor of PAGE_SIZE \ after cache alignment */ #define PC_DBREG_CMD_NONE 0 _______________________________________________ svn-src-head@freebsd.org mailing list https://lists.freebsd.org/mailman/listinfo/svn-src-head To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org"
