The branch main has been updated by kib:
URL:
https://cgit.FreeBSD.org/src/commit/?id=47656cc1ef1cac307f24de88a4fe23a1389af44e
commit 47656cc1ef1cac307f24de88a4fe23a1389af44e
Author: Konstantin Belousov <k...@freebsd.org>
AuthorDate: 2024-05-13 22:42:08 +0000
Commit: Konstantin Belousov <k...@freebsd.org>
CommitDate: 2024-08-21 16:35:15 +0000
amd64: use INVLPGB for kernel pmap invalidations
avoiding broadcast IPIs.
Reviewed by: alc, markj
Tested by: pho
Sponsored by: The FreeBSD Foundation
MFC after: 1 week
Differential revision: https://reviews.freebsd.org/D45191
---
sys/amd64/amd64/mp_machdep.c | 54 ++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 54 insertions(+)
diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c
index 91737637b714..12abb8b6bf8b 100644
--- a/sys/amd64/amd64/mp_machdep.c
+++ b/sys/amd64/amd64/mp_machdep.c
@@ -679,6 +679,20 @@ local_cb:
void
smp_masked_invltlb(pmap_t pmap, smp_invl_cb_t curcpu_cb)
{
+ if (invlpgb_works && pmap == kernel_pmap) {
+ invlpgb(INVLPGB_GLOB, 0, 0);
+
+ /*
+ * TLBSYNC syncs only against INVLPGB executed on the
+ * same CPU. Since current thread is pinned by
+ * caller, we do not need to enter critical section to
+ * prevent migration.
+ */
+ tlbsync();
+ sched_unpin();
+ return;
+ }
+
smp_targeted_tlb_shootdown(pmap, 0, 0, curcpu_cb, invl_op_tlb);
#ifdef COUNT_XINVLTLB_HITS
ipi_global++;
@@ -688,6 +702,13 @@ smp_masked_invltlb(pmap_t pmap, smp_invl_cb_t curcpu_cb)
void
smp_masked_invlpg(vm_offset_t addr, pmap_t pmap, smp_invl_cb_t curcpu_cb)
{
+ if (invlpgb_works && pmap == kernel_pmap) {
+ invlpgb(INVLPGB_GLOB | INVLPGB_VA | trunc_page(addr), 0, 0);
+ tlbsync();
+ sched_unpin();
+ return;
+ }
+
smp_targeted_tlb_shootdown(pmap, addr, 0, curcpu_cb, invl_op_pg);
#ifdef COUNT_XINVLTLB_HITS
ipi_page++;
@@ -698,6 +719,39 @@ void
smp_masked_invlpg_range(vm_offset_t addr1, vm_offset_t addr2, pmap_t pmap,
smp_invl_cb_t curcpu_cb)
{
+ if (invlpgb_works && pmap == kernel_pmap) {
+ vm_offset_t va;
+ uint64_t cnt, total;
+
+ addr1 = trunc_page(addr1);
+ addr2 = round_page(addr2);
+ total = atop(addr2 - addr1);
+ for (va = addr1; total > 0;) {
+ if ((va & PDRMASK) != 0 || total < NPDEPG) {
+ cnt = atop(NBPDR - (va & PDRMASK));
+ if (cnt > total)
+ cnt = total;
+ if (cnt > invlpgb_maxcnt + 1)
+ cnt = invlpgb_maxcnt + 1;
+ invlpgb(INVLPGB_GLOB | INVLPGB_VA | va, 0,
+ cnt - 1);
+ va += ptoa(cnt);
+ total -= cnt;
+ } else {
+ cnt = total / NPTEPG;
+ if (cnt > invlpgb_maxcnt + 1)
+ cnt = invlpgb_maxcnt + 1;
+ invlpgb(INVLPGB_GLOB | INVLPGB_VA | va, 0,
+ INVLPGB_2M_CNT | (cnt - 1));
+ va += cnt << PDRSHIFT;
+ total -= cnt * NPTEPG;
+ }
+ }
+ tlbsync();
+ sched_unpin();
+ return;
+ }
+
smp_targeted_tlb_shootdown(pmap, addr1, addr2, curcpu_cb,
invl_op_pgrng);
#ifdef COUNT_XINVLTLB_HITS
