Author: kib Date: Sun Nov 10 09:28:18 2019 New Revision: 354589 URL: https://svnweb.freebsd.org/changeset/base/354589
Log: amd64: move common_tss into pcpu. This saves some memory, around 256K I think. It removes some code, e.g. KPTI does not need to specially map common_tss anymore. Also, common_tss become domain-local. Reviewed by: jhb Tested by: pho Sponsored by: The FreeBSD Foundation MFC after: 1 week Differential revision: https://reviews.freebsd.org/D22231 Modified: head/sys/amd64/amd64/cpu_switch.S head/sys/amd64/amd64/db_interface.c head/sys/amd64/amd64/genassym.c head/sys/amd64/amd64/machdep.c head/sys/amd64/amd64/mp_machdep.c head/sys/amd64/amd64/pmap.c head/sys/amd64/amd64/sys_machdep.c head/sys/amd64/include/pcpu.h head/sys/amd64/include/tss.h Modified: head/sys/amd64/amd64/cpu_switch.S ============================================================================== --- head/sys/amd64/amd64/cpu_switch.S Sun Nov 10 09:25:19 2019 (r354588) +++ head/sys/amd64/amd64/cpu_switch.S Sun Nov 10 09:28:18 2019 (r354589) @@ -188,8 +188,10 @@ do_kthread: /* Do we need to reload tss ? */ movq PCPU(TSSP),%rax movq PCB_TSSP(%r8),%rdx + movq PCPU(PRVSPACE),%r13 + addq $PC_COMMONTSS,%r13 testq %rdx,%rdx - cmovzq PCPU(COMMONTSSP),%rdx + cmovzq %r13,%rdx cmpq %rax,%rdx jne do_tss done_tss: Modified: head/sys/amd64/amd64/db_interface.c ============================================================================== --- head/sys/amd64/amd64/db_interface.c Sun Nov 10 09:25:19 2019 (r354588) +++ head/sys/amd64/amd64/db_interface.c Sun Nov 10 09:28:18 2019 (r354589) @@ -97,9 +97,9 @@ void db_show_mdpcpu(struct pcpu *pc) { + db_printf("self = %p\n", pc->pc_prvspace); db_printf("curpmap = %p\n", pc->pc_curpmap); db_printf("tssp = %p\n", pc->pc_tssp); - db_printf("commontssp = %p\n", pc->pc_commontssp); db_printf("rsp0 = 0x%lx\n", pc->pc_rsp0); db_printf("kcr3 = 0x%lx\n", pc->pc_kcr3); db_printf("ucr3 = 0x%lx\n", pc->pc_ucr3); Modified: head/sys/amd64/amd64/genassym.c ============================================================================== --- head/sys/amd64/amd64/genassym.c Sun Nov 10 09:25:19 2019 (r354588) +++ head/sys/amd64/amd64/genassym.c Sun Nov 10 09:28:18 2019 (r354589) @@ -225,7 +225,7 @@ ASSYM(PC_RSP0, offsetof(struct pcpu, pc_rsp0)); ASSYM(PC_FS32P, offsetof(struct pcpu, pc_fs32p)); ASSYM(PC_GS32P, offsetof(struct pcpu, pc_gs32p)); ASSYM(PC_LDT, offsetof(struct pcpu, pc_ldt)); -ASSYM(PC_COMMONTSSP, offsetof(struct pcpu, pc_commontssp)); +ASSYM(PC_COMMONTSS, offsetof(struct pcpu, pc_common_tss)); ASSYM(PC_TSS, offsetof(struct pcpu, pc_tss)); ASSYM(PC_PM_SAVE_CNT, offsetof(struct pcpu, pc_pm_save_cnt)); ASSYM(PC_KCR3, offsetof(struct pcpu, pc_kcr3)); Modified: head/sys/amd64/amd64/machdep.c ============================================================================== --- head/sys/amd64/amd64/machdep.c Sun Nov 10 09:25:19 2019 (r354588) +++ head/sys/amd64/amd64/machdep.c Sun Nov 10 09:28:18 2019 (r354589) @@ -669,8 +669,6 @@ static char nmi0_stack[PAGE_SIZE] __aligned(16); static char dbg0_stack[PAGE_SIZE] __aligned(16); CTASSERT(sizeof(struct nmi_pcpu) == 16); -struct amd64tss common_tss[MAXCPU]; - /* * Software prototypes -- in more palatable form. * @@ -1550,8 +1548,7 @@ amd64_bsp_pcpu_init1(struct pcpu *pc) PCPU_SET(prvspace, pc); PCPU_SET(curthread, &thread0); - PCPU_SET(tssp, &common_tss[0]); - PCPU_SET(commontssp, &common_tss[0]); + PCPU_SET(tssp, PCPU_PTR(common_tss)); PCPU_SET(tss, (struct system_segment_descriptor *)&gdt[GPROC0_SEL]); PCPU_SET(ldt, (struct system_segment_descriptor *)&gdt[GUSERLDT_SEL]); PCPU_SET(fs32p, &gdt[GUFS32_SEL]); @@ -1572,9 +1569,12 @@ void amd64_bsp_ist_init(struct pcpu *pc) { struct nmi_pcpu *np; + struct amd64tss *tssp; + tssp = &pc->pc_common_tss; + /* doublefault stack space, runs on ist1 */ - common_tss[0].tss_ist1 = (long)&dblfault_stack[sizeof(dblfault_stack)]; + tssp->tss_ist1 = (long)&dblfault_stack[sizeof(dblfault_stack)]; /* * NMI stack, runs on ist2. The pcpu pointer is stored just @@ -1582,7 +1582,7 @@ amd64_bsp_ist_init(struct pcpu *pc) */ np = ((struct nmi_pcpu *)&nmi0_stack[sizeof(nmi0_stack)]) - 1; np->np_pcpu = (register_t)pc; - common_tss[0].tss_ist2 = (long)np; + tssp->tss_ist2 = (long)np; /* * MC# stack, runs on ist3. The pcpu pointer is stored just @@ -1590,14 +1590,14 @@ amd64_bsp_ist_init(struct pcpu *pc) */ np = ((struct nmi_pcpu *)&mce0_stack[sizeof(mce0_stack)]) - 1; np->np_pcpu = (register_t)pc; - common_tss[0].tss_ist3 = (long)np; + tssp->tss_ist3 = (long)np; /* * DB# stack, runs on ist4. */ np = ((struct nmi_pcpu *)&dbg0_stack[sizeof(dbg0_stack)]) - 1; np->np_pcpu = (register_t)pc; - common_tss[0].tss_ist4 = (long)np; + tssp->tss_ist4 = (long)np; } u_int64_t @@ -1664,6 +1664,8 @@ hammer_time(u_int64_t modulep, u_int64_t physfree) */ pmap_thread_init_invl_gen(&thread0); + pc = &temp_bsp_pcpu; + /* * make gdt memory segments */ @@ -1672,14 +1674,13 @@ hammer_time(u_int64_t modulep, u_int64_t physfree) x != GUSERLDT_SEL && x != (GUSERLDT_SEL) + 1) ssdtosd(&gdt_segs[x], &gdt[x]); } - gdt_segs[GPROC0_SEL].ssd_base = (uintptr_t)&common_tss[0]; + gdt_segs[GPROC0_SEL].ssd_base = (uintptr_t)&pc->pc_common_tss; ssdtosyssd(&gdt_segs[GPROC0_SEL], (struct system_segment_descriptor *)&gdt[GPROC0_SEL]); r_gdt.rd_limit = NGDT * sizeof(gdt[0]) - 1; r_gdt.rd_base = (long) gdt; lgdt(&r_gdt); - pc = &temp_bsp_pcpu; wrmsr(MSR_FSBASE, 0); /* User value */ wrmsr(MSR_GSBASE, (u_int64_t)pc); @@ -1781,7 +1782,8 @@ hammer_time(u_int64_t modulep, u_int64_t physfree) amd64_bsp_ist_init(pc); /* Set the IO permission bitmap (empty due to tss seg limit) */ - common_tss[0].tss_iobase = sizeof(struct amd64tss) + IOPERM_BITMAP_SIZE; + pc->pc_common_tss.tss_iobase = sizeof(struct amd64tss) + + IOPERM_BITMAP_SIZE; gsel_tss = GSEL(GPROC0_SEL, SEL_KPL); ltr(gsel_tss); @@ -1865,7 +1867,7 @@ hammer_time(u_int64_t modulep, u_int64_t physfree) rsp0 = thread0.td_md.md_stack_base; /* Ensure the stack is aligned to 16 bytes */ rsp0 &= ~0xFul; - common_tss[0].tss_rsp0 = rsp0; + __pcpu[0].pc_common_tss.tss_rsp0 = rsp0; amd64_bsp_pcpu_init2(rsp0); /* transfer to user mode */ Modified: head/sys/amd64/amd64/mp_machdep.c ============================================================================== --- head/sys/amd64/amd64/mp_machdep.c Sun Nov 10 09:25:19 2019 (r354588) +++ head/sys/amd64/amd64/mp_machdep.c Sun Nov 10 09:28:18 2019 (r354589) @@ -285,26 +285,51 @@ init_secondary(void) /* Update microcode before doing anything else. */ ucode_load_ap(cpu); + /* Get per-cpu data and save */ + pc = &__pcpu[cpu]; + + /* prime data page for it to use */ + pcpu_init(pc, cpu, sizeof(struct pcpu)); + dpcpu_init(dpcpu, cpu); + pc->pc_apic_id = cpu_apic_ids[cpu]; + pc->pc_prvspace = pc; + pc->pc_curthread = 0; + pc->pc_tssp = &pc->pc_common_tss; + pc->pc_rsp0 = 0; + pc->pc_pti_rsp0 = (((vm_offset_t)&pc->pc_pti_stack + + PC_PTI_STACK_SZ * sizeof(uint64_t)) & ~0xful); + pc->pc_tss = (struct system_segment_descriptor *)&gdt[NGDT * cpu + + GPROC0_SEL]; + pc->pc_fs32p = &gdt[NGDT * cpu + GUFS32_SEL]; + pc->pc_gs32p = &gdt[NGDT * cpu + GUGS32_SEL]; + pc->pc_ldt = (struct system_segment_descriptor *)&gdt[NGDT * cpu + + GUSERLDT_SEL]; + /* See comment in pmap_bootstrap(). */ + pc->pc_pcid_next = PMAP_PCID_KERN + 2; + pc->pc_pcid_gen = 1; + /* Init tss */ - common_tss[cpu] = common_tss[0]; - common_tss[cpu].tss_iobase = sizeof(struct amd64tss) + + pc->pc_common_tss = __pcpu[0].pc_common_tss; + pc->pc_common_tss.tss_iobase = sizeof(struct amd64tss) + IOPERM_BITMAP_SIZE; - common_tss[cpu].tss_ist1 = (long)&doublefault_stack[PAGE_SIZE]; + pc->pc_common_tss.tss_rsp0 = 0; + pc->pc_common_tss.tss_ist1 = (long)&doublefault_stack[PAGE_SIZE]; + /* The NMI stack runs on IST2. */ np = ((struct nmi_pcpu *) &nmi_stack[PAGE_SIZE]) - 1; - common_tss[cpu].tss_ist2 = (long) np; + pc->pc_common_tss.tss_ist2 = (long)np; /* The MC# stack runs on IST3. */ np = ((struct nmi_pcpu *) &mce_stack[PAGE_SIZE]) - 1; - common_tss[cpu].tss_ist3 = (long) np; + pc->pc_common_tss.tss_ist3 = (long)np; /* The DB# stack runs on IST4. */ np = ((struct nmi_pcpu *) &dbg_stack[PAGE_SIZE]) - 1; - common_tss[cpu].tss_ist4 = (long) np; + pc->pc_common_tss.tss_ist4 = (long)np; /* Prepare private GDT */ - gdt_segs[GPROC0_SEL].ssd_base = (long) &common_tss[cpu]; + gdt_segs[GPROC0_SEL].ssd_base = (long)&pc->pc_common_tss; for (x = 0; x < NGDT; x++) { if (x != GPROC0_SEL && x != (GPROC0_SEL + 1) && x != GUSERLDT_SEL && x != (GUSERLDT_SEL + 1)) @@ -313,45 +338,20 @@ init_secondary(void) ssdtosyssd(&gdt_segs[GPROC0_SEL], (struct system_segment_descriptor *)&gdt[NGDT * cpu + GPROC0_SEL]); ap_gdt.rd_limit = NGDT * sizeof(gdt[0]) - 1; - ap_gdt.rd_base = (long) &gdt[NGDT * cpu]; + ap_gdt.rd_base = (u_long)&gdt[NGDT * cpu]; lgdt(&ap_gdt); /* does magic intra-segment return */ - /* Get per-cpu data */ - pc = &__pcpu[cpu]; - - /* prime data page for it to use */ - pcpu_init(pc, cpu, sizeof(struct pcpu)); - dpcpu_init(dpcpu, cpu); - pc->pc_apic_id = cpu_apic_ids[cpu]; - pc->pc_prvspace = pc; - pc->pc_curthread = 0; - pc->pc_tssp = &common_tss[cpu]; - pc->pc_commontssp = &common_tss[cpu]; - pc->pc_rsp0 = 0; - pc->pc_pti_rsp0 = (((vm_offset_t)&pc->pc_pti_stack + - PC_PTI_STACK_SZ * sizeof(uint64_t)) & ~0xful); - pc->pc_tss = (struct system_segment_descriptor *)&gdt[NGDT * cpu + - GPROC0_SEL]; - pc->pc_fs32p = &gdt[NGDT * cpu + GUFS32_SEL]; - pc->pc_gs32p = &gdt[NGDT * cpu + GUGS32_SEL]; - pc->pc_ldt = (struct system_segment_descriptor *)&gdt[NGDT * cpu + - GUSERLDT_SEL]; - /* See comment in pmap_bootstrap(). */ - pc->pc_pcid_next = PMAP_PCID_KERN + 2; - pc->pc_pcid_gen = 1; - common_tss[cpu].tss_rsp0 = 0; - /* Save the per-cpu pointer for use by the NMI handler. */ np = ((struct nmi_pcpu *) &nmi_stack[PAGE_SIZE]) - 1; - np->np_pcpu = (register_t) pc; + np->np_pcpu = (register_t)pc; /* Save the per-cpu pointer for use by the MC# handler. */ np = ((struct nmi_pcpu *) &mce_stack[PAGE_SIZE]) - 1; - np->np_pcpu = (register_t) pc; + np->np_pcpu = (register_t)pc; /* Save the per-cpu pointer for use by the DB# handler. */ np = ((struct nmi_pcpu *) &dbg_stack[PAGE_SIZE]) - 1; - np->np_pcpu = (register_t) pc; + np->np_pcpu = (register_t)pc; wrmsr(MSR_FSBASE, 0); /* User value */ wrmsr(MSR_GSBASE, (u_int64_t)pc); Modified: head/sys/amd64/amd64/pmap.c ============================================================================== --- head/sys/amd64/amd64/pmap.c Sun Nov 10 09:25:19 2019 (r354588) +++ head/sys/amd64/amd64/pmap.c Sun Nov 10 09:28:18 2019 (r354589) @@ -1765,6 +1765,10 @@ pmap_bootstrap(vm_paddr_t *firstaddr) pcpu_init(&__pcpu[0], 0, sizeof(struct pcpu)); amd64_bsp_pcpu_init1(&__pcpu[0]); amd64_bsp_ist_init(&__pcpu[0]); + gdt_segs[GPROC0_SEL].ssd_base = (uintptr_t)&__pcpu[0].pc_common_tss; + ssdtosyssd(&gdt_segs[GPROC0_SEL], + (struct system_segment_descriptor *)&gdt[GPROC0_SEL]); + ltr(GSEL(GPROC0_SEL, SEL_KPL)); __pcpu[0].pc_dynamic = temp_bsp_pcpu.pc_dynamic; __pcpu[0].pc_acpi_id = temp_bsp_pcpu.pc_acpi_id; @@ -9719,20 +9723,19 @@ pmap_pti_init(void) sizeof(struct user_segment_descriptor) * NGDT * MAXCPU, false); pmap_pti_add_kva_locked((vm_offset_t)idt, (vm_offset_t)idt + sizeof(struct gate_descriptor) * NIDT, false); - pmap_pti_add_kva_locked((vm_offset_t)common_tss, - (vm_offset_t)common_tss + sizeof(struct amd64tss) * MAXCPU, false); CPU_FOREACH(i) { /* Doublefault stack IST 1 */ - va = common_tss[i].tss_ist1; + va = __pcpu[i].pc_common_tss.tss_ist1; pmap_pti_add_kva_locked(va - PAGE_SIZE, va, false); /* NMI stack IST 2 */ - va = common_tss[i].tss_ist2 + sizeof(struct nmi_pcpu); + va = __pcpu[i].pc_common_tss.tss_ist2 + sizeof(struct nmi_pcpu); pmap_pti_add_kva_locked(va - PAGE_SIZE, va, false); /* MC# stack IST 3 */ - va = common_tss[i].tss_ist3 + sizeof(struct nmi_pcpu); + va = __pcpu[i].pc_common_tss.tss_ist3 + + sizeof(struct nmi_pcpu); pmap_pti_add_kva_locked(va - PAGE_SIZE, va, false); /* DB# stack IST 4 */ - va = common_tss[i].tss_ist4 + sizeof(struct nmi_pcpu); + va = __pcpu[i].pc_common_tss.tss_ist4 + sizeof(struct nmi_pcpu); pmap_pti_add_kva_locked(va - PAGE_SIZE, va, false); } pmap_pti_add_kva_locked((vm_offset_t)kernphys + KERNBASE, Modified: head/sys/amd64/amd64/sys_machdep.c ============================================================================== --- head/sys/amd64/amd64/sys_machdep.c Sun Nov 10 09:25:19 2019 (r354588) +++ head/sys/amd64/amd64/sys_machdep.c Sun Nov 10 09:28:18 2019 (r354589) @@ -426,8 +426,7 @@ amd64_set_ioperm(td, uap) memset(iomap, 0xff, IOPERM_BITMAP_SIZE); critical_enter(); /* Takes care of tss_rsp0. */ - memcpy(tssp, &common_tss[PCPU_GET(cpuid)], - sizeof(struct amd64tss)); + memcpy(tssp, PCPU_PTR(common_tss), sizeof(struct amd64tss)); tssp->tss_iobase = sizeof(*tssp); pcb->pcb_tssp = tssp; tss_sd = PCPU_GET(tss); Modified: head/sys/amd64/include/pcpu.h ============================================================================== --- head/sys/amd64/include/pcpu.h Sun Nov 10 09:25:19 2019 (r354588) +++ head/sys/amd64/include/pcpu.h Sun Nov 10 09:28:18 2019 (r354589) @@ -35,6 +35,8 @@ #error "sys/cdefs.h is a prerequisite for this file" #endif +#include <machine/tss.h> + #define PC_PTI_STACK_SZ 16 struct monitorbuf { @@ -56,7 +58,7 @@ _Static_assert(sizeof(struct monitorbuf) == 128, "2x c struct pcpu *pc_prvspace; /* Self-reference */ \ struct pmap *pc_curpmap; \ struct amd64tss *pc_tssp; /* TSS segment active on CPU */ \ - struct amd64tss *pc_commontssp;/* Common TSS for the CPU */ \ + void *pc_pad0; \ uint64_t pc_kcr3; \ uint64_t pc_ucr3; \ uint64_t pc_saved_ucr3; \ @@ -89,7 +91,8 @@ _Static_assert(sizeof(struct monitorbuf) == 128, "2x c uint32_t pc_pad[2]; \ uint8_t pc_mds_tmp[64]; \ u_int pc_ipi_bitmap; \ - char __pad[3172] /* pad to UMA_PCPU_ALLOC_SIZE */ + struct amd64tss pc_common_tss; \ + char __pad[3068] /* pad to UMA_PCPU_ALLOC_SIZE */ #define PC_DBREG_CMD_NONE 0 #define PC_DBREG_CMD_LOAD 1 Modified: head/sys/amd64/include/tss.h ============================================================================== --- head/sys/amd64/include/tss.h Sun Nov 10 09:25:19 2019 (r354588) +++ head/sys/amd64/include/tss.h Sun Nov 10 09:28:18 2019 (r354589) @@ -65,8 +65,4 @@ struct amd64tss { u_int16_t tss_iobase; /* io bitmap offset */ }; -#ifdef _KERNEL -extern struct amd64tss common_tss[]; -#endif - #endif /* _MACHINE_TSS_H_ */ _______________________________________________ svn-src-head@freebsd.org mailing list https://lists.freebsd.org/mailman/listinfo/svn-src-head To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org"
