Author: markj Date: Wed Sep 23 19:33:47 2020 New Revision: 366089 URL: https://svnweb.freebsd.org/changeset/base/366089
Log: Add largepage support to the arm64 pmap. Reviewed by: alc, kib Sponsored by: Juniper Networks, Inc., Klara, Inc. Differential Revision: https://reviews.freebsd.org/D26466 Modified: head/sys/arm64/arm64/pmap.c head/sys/arm64/include/param.h Modified: head/sys/arm64/arm64/pmap.c ============================================================================== --- head/sys/arm64/arm64/pmap.c Wed Sep 23 19:18:53 2020 (r366088) +++ head/sys/arm64/arm64/pmap.c Wed Sep 23 19:33:47 2020 (r366089) @@ -177,6 +177,8 @@ __FBSDID("$FreeBSD$"); #define PV_STAT(x) do { } while (0) #endif +#define pmap_l0_pindex(v) (NUL2E + NUL1E + ((v) >> L0_SHIFT)) +#define pmap_l1_pindex(v) (NUL2E + ((v) >> L1_SHIFT)) #define pmap_l2_pindex(v) ((v) >> L2_SHIFT) static struct md_page * @@ -1087,6 +1089,9 @@ pmap_init(void) KASSERT(MAXPAGESIZES > 1 && pagesizes[1] == 0, ("pmap_init: can't assign to pagesizes[1]")); pagesizes[1] = L2_SIZE; + KASSERT(MAXPAGESIZES > 2 && pagesizes[2] == 0, + ("pmap_init: can't assign to pagesizes[2]")); + pagesizes[2] = L1_SIZE; } /* @@ -1337,7 +1342,7 @@ pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_ use = true; if (use) { - switch(lvl) { + switch (lvl) { case 1: off = va & L1_OFFSET; break; @@ -1349,7 +1354,7 @@ pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_ off = 0; } m = PHYS_TO_VM_PAGE((tpte & ~ATTR_MASK) | off); - if (!vm_page_wire_mapped(m)) + if (m != NULL && !vm_page_wire_mapped(m)) m = NULL; } } @@ -2968,12 +2973,24 @@ pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t continue; } + va_next = (sva + L1_SIZE) & ~L1_OFFSET; + if (va_next < sva) + va_next = eva; l1 = pmap_l0_to_l1(l0, sva); - if (pmap_load(l1) == 0) { - va_next = (sva + L1_SIZE) & ~L1_OFFSET; - if (va_next < sva) - va_next = eva; + if (pmap_load(l1) == 0) continue; + if ((pmap_load(l1) & ATTR_DESCR_MASK) == L1_BLOCK) { + KASSERT(va_next <= eva, + ("partial update of non-transparent 1G page " + "l1 %#lx sva %#lx eva %#lx va_next %#lx", + pmap_load(l1), sva, eva, va_next)); + MPASS(pmap != kernel_pmap); + MPASS((pmap_load(l1) & ATTR_SW_MANAGED) == 0); + pmap_clear(l1); + pmap_invalidate_page(pmap, sva); + pmap_resident_count_dec(pmap, L1_SIZE / PAGE_SIZE); + pmap_unuse_pt(pmap, sva, pmap_load(l0), &free); + continue; } /* @@ -3217,12 +3234,23 @@ pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t continue; } + va_next = (sva + L1_SIZE) & ~L1_OFFSET; + if (va_next < sva) + va_next = eva; l1 = pmap_l0_to_l1(l0, sva); - if (pmap_load(l1) == 0) { - va_next = (sva + L1_SIZE) & ~L1_OFFSET; - if (va_next < sva) - va_next = eva; + if (pmap_load(l1) == 0) continue; + if ((pmap_load(l1) & ATTR_DESCR_MASK) == L1_BLOCK) { + KASSERT(va_next <= eva, + ("partial update of non-transparent 1G page " + "l1 %#lx sva %#lx eva %#lx va_next %#lx", + pmap_load(l1), sva, eva, va_next)); + MPASS((pmap_load(l1) & ATTR_SW_MANAGED) == 0); + if ((pmap_load(l1) & mask) != nbits) { + pmap_store(l1, (pmap_load(l1) & ~mask) | nbits); + pmap_invalidate_page(pmap, sva); + } + continue; } va_next = (sva + L2_SIZE) & ~L2_OFFSET; @@ -3485,6 +3513,96 @@ setl3: } #endif /* VM_NRESERVLEVEL > 0 */ +static int +pmap_enter_largepage(pmap_t pmap, vm_offset_t va, pt_entry_t newpte, int flags, + int psind) +{ + pd_entry_t *l0p, *l1p, *l2p, origpte; + vm_page_t mp; + + PMAP_LOCK_ASSERT(pmap, MA_OWNED); + KASSERT(psind > 0 && psind < MAXPAGESIZES, + ("psind %d unexpected", psind)); + KASSERT(((newpte & ~ATTR_MASK) & (pagesizes[psind] - 1)) == 0, + ("unaligned phys address %#lx newpte %#lx psind %d", + (newpte & ~ATTR_MASK), newpte, psind)); + +restart: + if (psind == 2) { + l0p = pmap_l0(pmap, va); + if ((pmap_load(l0p) & ATTR_DESCR_VALID) == 0) { + mp = _pmap_alloc_l3(pmap, pmap_l0_pindex(va), NULL); + if (mp == NULL) { + if ((flags & PMAP_ENTER_NOSLEEP) != 0) + return (KERN_RESOURCE_SHORTAGE); + PMAP_UNLOCK(pmap); + vm_wait(NULL); + PMAP_LOCK(pmap); + goto restart; + } + l1p = pmap_l0_to_l1(l0p, va); + KASSERT(l1p != NULL, ("va %#lx lost l1 entry", va)); + origpte = pmap_load(l1p); + } else { + l1p = pmap_l0_to_l1(l0p, va); + KASSERT(l1p != NULL, ("va %#lx lost l1 entry", va)); + origpte = pmap_load(l1p); + if ((origpte & ATTR_DESCR_VALID) == 0) { + mp = PHYS_TO_VM_PAGE(pmap_load(l0p) & + ~ATTR_MASK); + mp->ref_count++; + } + } + KASSERT((origpte & ATTR_DESCR_VALID) == 0 || + ((origpte & ATTR_DESCR_MASK) == L1_BLOCK && + (origpte & ~ATTR_MASK) == (newpte & ~ATTR_MASK)), + ("va %#lx changing 1G phys page l1 %#lx newpte %#lx", + va, origpte, newpte)); + pmap_store(l1p, newpte); + } else /* (psind == 1) */ { + l2p = pmap_l2(pmap, va); + if (l2p == NULL) { + mp = _pmap_alloc_l3(pmap, pmap_l1_pindex(va), NULL); + if (mp == NULL) { + if ((flags & PMAP_ENTER_NOSLEEP) != 0) + return (KERN_RESOURCE_SHORTAGE); + PMAP_UNLOCK(pmap); + vm_wait(NULL); + PMAP_LOCK(pmap); + goto restart; + } + l2p = (pd_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mp)); + l2p = &l2p[pmap_l2_index(va)]; + origpte = pmap_load(l2p); + } else { + l1p = pmap_l1(pmap, va); + origpte = pmap_load(l2p); + if ((origpte & ATTR_DESCR_VALID) == 0) { + mp = PHYS_TO_VM_PAGE(pmap_load(l1p) & + ~ATTR_MASK); + mp->ref_count++; + } + } + KASSERT((origpte & ATTR_DESCR_VALID) == 0 || + ((origpte & ATTR_DESCR_MASK) == L2_BLOCK && + (origpte & ~ATTR_MASK) == (newpte & ~ATTR_MASK)), + ("va %#lx changing 2M phys page l2 %#lx newpte %#lx", + va, origpte, newpte)); + pmap_store(l2p, newpte); + } + dsb(ishst); + + if ((origpte & ATTR_DESCR_VALID) == 0) + pmap_resident_count_inc(pmap, pagesizes[psind] / PAGE_SIZE); + if ((newpte & ATTR_SW_WIRED) != 0 && (origpte & ATTR_SW_WIRED) == 0) + pmap->pm_stats.wired_count += pagesizes[psind] / PAGE_SIZE; + else if ((newpte & ATTR_SW_WIRED) == 0 && + (origpte & ATTR_SW_WIRED) != 0) + pmap->pm_stats.wired_count -= pagesizes[psind] / PAGE_SIZE; + + return (KERN_SUCCESS); +} + /* * Insert the given physical page (p) at * the specified virtual address (v) in the @@ -3560,6 +3678,17 @@ pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, v lock = NULL; PMAP_LOCK(pmap); + if ((flags & PMAP_ENTER_LARGEPAGE) != 0) { + KASSERT((m->oflags & VPO_UNMANAGED) != 0, + ("managed largepage va %#lx flags %#x", va, flags)); + new_l3 &= ~L3_PAGE; + if (psind == 2) + new_l3 |= L1_BLOCK; + else /* (psind == 1) */ + new_l3 |= L2_BLOCK; + rv = pmap_enter_largepage(pmap, va, new_l3, flags, psind); + goto out; + } if (psind == 1) { /* Assert the required virtual and physical alignment. */ KASSERT((va & L2_OFFSET) == 0, ("pmap_enter: va unaligned")); @@ -4209,11 +4338,23 @@ pmap_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t } l1 = pmap_l0_to_l1(l0, sva); - if (pmap_load(l1) == 0) { - va_next = (sva + L1_SIZE) & ~L1_OFFSET; - if (va_next < sva) - va_next = eva; + va_next = (sva + L1_SIZE) & ~L1_OFFSET; + if (va_next < sva) + va_next = eva; + if (pmap_load(l1) == 0) continue; + + if ((pmap_load(l1) & ATTR_DESCR_MASK) == L1_BLOCK) { + KASSERT(va_next <= eva, + ("partial update of non-transparent 1G page " + "l1 %#lx sva %#lx eva %#lx va_next %#lx", + pmap_load(l1), sva, eva, va_next)); + MPASS(pmap != kernel_pmap); + MPASS((pmap_load(l1) & (ATTR_SW_MANAGED | + ATTR_SW_WIRED)) == ATTR_SW_WIRED); + pmap_clear_bits(l1, ATTR_SW_WIRED); + pmap->pm_stats.wired_count -= L1_SIZE / PAGE_SIZE; + continue; } va_next = (sva + L2_SIZE) & ~L2_OFFSET; @@ -4284,7 +4425,7 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_ pd_entry_t *l0, *l1, *l2, srcptepaddr; pt_entry_t *dst_pte, mask, nbits, ptetemp, *src_pte; vm_offset_t addr, end_addr, va_next; - vm_page_t dst_l2pg, dstmpte, srcmpte; + vm_page_t dst_m, dstmpte, srcmpte; PMAP_ASSERT_STAGE1(dst_pmap); PMAP_ASSERT_STAGE1(src_pmap); @@ -4308,13 +4449,40 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_ va_next = end_addr; continue; } + + va_next = (addr + L1_SIZE) & ~L1_OFFSET; + if (va_next < addr) + va_next = end_addr; l1 = pmap_l0_to_l1(l0, addr); - if (pmap_load(l1) == 0) { - va_next = (addr + L1_SIZE) & ~L1_OFFSET; - if (va_next < addr) - va_next = end_addr; + if (pmap_load(l1) == 0) continue; + if ((pmap_load(l1) & ATTR_DESCR_MASK) == L1_BLOCK) { + KASSERT(va_next <= end_addr, + ("partial update of non-transparent 1G page " + "l1 %#lx addr %#lx end_addr %#lx va_next %#lx", + pmap_load(l1), addr, end_addr, va_next)); + srcptepaddr = pmap_load(l1); + l1 = pmap_l1(dst_pmap, addr); + if (l1 == NULL) { + if (_pmap_alloc_l3(dst_pmap, + pmap_l0_pindex(addr), NULL) == NULL) + break; + l1 = pmap_l1(dst_pmap, addr); + } else { + l0 = pmap_l0(dst_pmap, addr); + dst_m = PHYS_TO_VM_PAGE(pmap_load(l0) & + ~ATTR_MASK); + dst_m->ref_count++; + } + KASSERT(pmap_load(l1) == 0, + ("1G mapping present in dst pmap " + "l1 %#lx addr %#lx end_addr %#lx va_next %#lx", + pmap_load(l1), addr, end_addr, va_next)); + pmap_store(l1, srcptepaddr & ~ATTR_SW_WIRED); + pmap_resident_count_inc(dst_pmap, L1_SIZE / PAGE_SIZE); + continue; } + va_next = (addr + L2_SIZE) & ~L2_OFFSET; if (va_next < addr) va_next = end_addr; @@ -4326,7 +4494,7 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_ if ((addr & L2_OFFSET) != 0 || addr + L2_SIZE > end_addr) continue; - l2 = pmap_alloc_l2(dst_pmap, addr, &dst_l2pg, NULL); + l2 = pmap_alloc_l2(dst_pmap, addr, &dst_m, NULL); if (l2 == NULL) break; if (pmap_load(l2) == 0 && @@ -4342,7 +4510,7 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_ PAGE_SIZE); atomic_add_long(&pmap_l2_mappings, 1); } else - pmap_abort_ptp(dst_pmap, addr, dst_l2pg); + pmap_abort_ptp(dst_pmap, addr, dst_m); continue; } KASSERT((srcptepaddr & ATTR_DESCR_MASK) == L2_TABLE, @@ -5247,13 +5415,21 @@ pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t va_next = eva; continue; } + + va_next = (sva + L1_SIZE) & ~L1_OFFSET; + if (va_next < sva) + va_next = eva; l1 = pmap_l0_to_l1(l0, sva); - if (pmap_load(l1) == 0) { - va_next = (sva + L1_SIZE) & ~L1_OFFSET; - if (va_next < sva) - va_next = eva; + if (pmap_load(l1) == 0) continue; + if ((pmap_load(l1) & ATTR_DESCR_MASK) == L1_BLOCK) { + KASSERT(va_next <= eva, + ("partial update of non-transparent 1G page " + "l1 %#lx sva %#lx eva %#lx va_next %#lx", + pmap_load(l1), sva, eva, va_next)); + continue; } + va_next = (sva + L2_SIZE) & ~L2_OFFSET; if (va_next < sva) va_next = eva; Modified: head/sys/arm64/include/param.h ============================================================================== --- head/sys/arm64/include/param.h Wed Sep 23 19:18:53 2020 (r366088) +++ head/sys/arm64/include/param.h Wed Sep 23 19:33:47 2020 (r366089) @@ -95,7 +95,7 @@ #define PAGE_SIZE_64K (1 << PAGE_SHIFT_64K) #define PAGE_MASK_64K (PAGE_SIZE_64K - 1) -#define MAXPAGESIZES 2 /* maximum number of supported page sizes */ +#define MAXPAGESIZES 3 /* maximum number of supported page sizes */ #ifndef KSTACK_PAGES #define KSTACK_PAGES 4 /* pages of kernel stack (with pcb) */ _______________________________________________ svn-src-head@freebsd.org mailing list https://lists.freebsd.org/mailman/listinfo/svn-src-head To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org"
