Author: markj Date: Mon Jul 15 17:13:32 2019 New Revision: 350004 URL: https://svnweb.freebsd.org/changeset/base/350004
Log: Implement software access and dirty bit management for arm64. Previously the arm64 pmap did no reference or modification tracking; all mappings were treated as referenced and all read-write mappings were treated as dirty. This change implements software management of these attributes. Dirty bit management is implemented to emulate ARMv8.1's optional hardware dirty bit modifier management, following a suggestion from alc. In particular, a mapping with ATTR_SW_DBM set is logically writeable and is dirty if the ATTR_AP_RW_BIT bit is clear. Mappings with ATTR_AP_RW_BIT set are write-protected, and a write access will trigger a permission fault. pmap_fault() handles permission faults for such mappings and marks the page dirty by clearing ATTR_AP_RW_BIT, thus mapping the page read-write. Reviewed by: alc MFC after: 1 month Sponsored by: The FreeBSD Foundation Differential Revision: https://reviews.freebsd.org/D20907 Modified: head/sys/arm64/arm64/pmap.c head/sys/arm64/arm64/trap.c head/sys/arm64/include/pte.h Modified: head/sys/arm64/arm64/pmap.c ============================================================================== --- head/sys/arm64/arm64/pmap.c Mon Jul 15 15:45:33 2019 (r350003) +++ head/sys/arm64/arm64/pmap.c Mon Jul 15 17:13:32 2019 (r350004) @@ -217,6 +217,13 @@ __FBSDID("$FreeBSD$"); #define VM_PAGE_TO_PV_LIST_LOCK(m) \ PHYS_TO_PV_LIST_LOCK(VM_PAGE_TO_PHYS(m)) +/* + * The presence of this flag indicates that the mapping is writeable. + * If the ATTR_AP_RO bit is also set, then the mapping is clean, otherwise it is + * dirty. This flag may only be set on managed mappings. + */ +static pt_entry_t ATTR_SW_DBM; + struct pmap kernel_pmap_store; /* Used for mapping ACPI memory before VM is initialized */ @@ -315,11 +322,12 @@ static __inline vm_page_t pmap_remove_pt_page(pmap_t p * They need to be atomic as the System MMU may write to the table at * the same time as the CPU. */ -#define pmap_clear(table) atomic_store_64(table, 0) -#define pmap_load_store(table, entry) atomic_swap_64(table, entry) -#define pmap_set(table, mask) atomic_set_64(table, mask) -#define pmap_load_clear(table) atomic_swap_64(table, 0) -#define pmap_load(table) (*table) +#define pmap_clear(table) atomic_store_64(table, 0) +#define pmap_clear_bits(table, bits) atomic_clear_64(table, bits) +#define pmap_load(table) (*table) +#define pmap_load_clear(table) atomic_swap_64(table, 0) +#define pmap_load_store(table, entry) atomic_swap_64(table, entry) +#define pmap_set_bits(table, bits) atomic_set_64(table, bits) /********************/ /* Inline functions */ @@ -532,15 +540,18 @@ pmap_l3_valid(pt_entry_t l3) CTASSERT(L1_BLOCK == L2_BLOCK); /* - * Checks if the page is dirty. We currently lack proper tracking of this on - * arm64 so for now assume is a page mapped as rw was accessed it is. + * Checks if the PTE is dirty. */ static inline int pmap_pte_dirty(pt_entry_t pte) { - return ((pte & (ATTR_AF | ATTR_AP_RW_BIT)) == - (ATTR_AF | ATTR_AP(ATTR_AP_RW))); + KASSERT((pte & ATTR_SW_MANAGED) != 0, ("pte %#lx is unmanaged", pte)); + KASSERT((pte & (ATTR_AP_RW_BIT | ATTR_SW_DBM)) != 0, + ("pte %#lx is writeable and missing ATTR_SW_DBM", pte)); + + return ((pte & (ATTR_AP_RW_BIT | ATTR_SW_DBM)) == + (ATTR_AP(ATTR_AP_RW) | ATTR_SW_DBM)); } static __inline void @@ -765,13 +776,18 @@ pmap_bootstrap(vm_offset_t l0pt, vm_offset_t l1pt, vm_ vm_size_t kernlen) { u_int l1_slot, l2_slot; - uint64_t kern_delta; pt_entry_t *l2; vm_offset_t va, freemempos; vm_offset_t dpcpu, msgbufpv; vm_paddr_t start_pa, pa, min_pa; + uint64_t kern_delta, reg; int i; + /* Determine whether the hardware implements DBM management. */ + reg = READ_SPECIALREG(ID_AA64MMFR1_EL1); + ATTR_SW_DBM = ID_AA64MMFR1_HAFDBS(reg) == ID_AA64MMFR1_HAFDBS_AF_DBS ? + ATTR_DBM : _ATTR_SW_DBM; + kern_delta = KERNBASE - kernstart; printf("pmap_bootstrap %lx %lx %lx\n", l1pt, kernstart, kernlen); @@ -2478,7 +2494,7 @@ pmap_remove_l2(pmap_t pmap, pt_entry_t *l2, vm_offset_ /* * pmap_remove_l3: do the things to unmap a page in a process */ -static int +static int __unused pmap_remove_l3(pmap_t pmap, pt_entry_t *l3, vm_offset_t va, pd_entry_t l2e, struct spglist *free, struct rwlock **lockp) { @@ -2788,7 +2804,8 @@ retry: * pmap_protect_l2: do the things to protect a 2MB page in a pmap */ static void -pmap_protect_l2(pmap_t pmap, pt_entry_t *l2, vm_offset_t sva, pt_entry_t nbits) +pmap_protect_l2(pmap_t pmap, pt_entry_t *l2, vm_offset_t sva, pt_entry_t mask, + pt_entry_t nbits) { pd_entry_t old_l2; vm_page_t m, mt; @@ -2804,7 +2821,8 @@ pmap_protect_l2(pmap_t pmap, pt_entry_t *l2, vm_offset * Return if the L2 entry already has the desired access restrictions * in place. */ - if ((old_l2 | nbits) == old_l2) +retry: + if ((old_l2 & mask) == nbits) return; /* @@ -2819,7 +2837,8 @@ pmap_protect_l2(pmap_t pmap, pt_entry_t *l2, vm_offset vm_page_dirty(mt); } - pmap_set(l2, nbits); + if (!atomic_fcmpset_64(l2, &old_l2, (old_l2 & ~mask) | nbits)) + goto retry; /* * Since a promotion must break the 4KB page mappings before making @@ -2837,7 +2856,7 @@ pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t { vm_offset_t va, va_next; pd_entry_t *l0, *l1, *l2; - pt_entry_t *l3p, l3, nbits; + pt_entry_t *l3p, l3, mask, nbits; KASSERT((prot & ~VM_PROT_ALL) == 0, ("invalid prot %x", prot)); if (prot == VM_PROT_NONE) { @@ -2845,12 +2864,16 @@ pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t return; } - nbits = 0; - if ((prot & VM_PROT_WRITE) == 0) + mask = nbits = 0; + if ((prot & VM_PROT_WRITE) == 0) { + mask |= ATTR_AP_RW_BIT | ATTR_SW_DBM; nbits |= ATTR_AP(ATTR_AP_RO); - if ((prot & VM_PROT_EXECUTE) == 0) + } + if ((prot & VM_PROT_EXECUTE) == 0) { + mask |= ATTR_XN; nbits |= ATTR_XN; - if (nbits == 0) + } + if (mask == 0) return; PMAP_LOCK(pmap); @@ -2882,7 +2905,7 @@ pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t if ((pmap_load(l2) & ATTR_DESCR_MASK) == L2_BLOCK) { if (sva + L2_SIZE == va_next && eva >= va_next) { - pmap_protect_l2(pmap, l2, sva, nbits); + pmap_protect_l2(pmap, l2, sva, mask, nbits); continue; } else if (pmap_demote_l2(pmap, l2, sva) == NULL) continue; @@ -2896,6 +2919,8 @@ pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t va = va_next; for (l3p = pmap_l2_to_l3(l2, sva); sva != va_next; l3p++, sva += L3_SIZE) { + l3 = pmap_load(l3p); +retry: /* * Go to the next L3 entry if the current one is * invalid or already has the desired access @@ -2904,16 +2929,13 @@ pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t * workload, almost 1 out of 4 L3 entries already * have the desired restrictions.) */ - l3 = pmap_load(l3p); - if (!pmap_l3_valid(l3) || (l3 | nbits) == l3) { + if (!pmap_l3_valid(l3) || (l3 & mask) == nbits) { if (va != va_next) { pmap_invalidate_range(pmap, va, sva); va = va_next; } continue; } - if (va == va_next) - va = sva; /* * When a dirty read/write mapping is write protected, @@ -2924,7 +2946,10 @@ pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t pmap_pte_dirty(l3)) vm_page_dirty(PHYS_TO_VM_PAGE(l3 & ~ATTR_MASK)); - pmap_set(l3p, nbits); + if (!atomic_fcmpset_64(l3p, &l3, (l3 & ~mask) | nbits)) + goto retry; + if (va == va_next) + va = sva; } if (va != va_next) pmap_invalidate_range(pmap, va, sva); @@ -3059,17 +3084,32 @@ pmap_promote_l2(pmap_t pmap, pd_entry_t *l2, vm_offset firstl3 = pmap_l2_to_l3(l2, sva); newl2 = pmap_load(firstl3); - /* Check the alingment is valid */ - if (((newl2 & ~ATTR_MASK) & L2_OFFSET) != 0) { +setl2: + if (((newl2 & (~ATTR_MASK | ATTR_AF)) & L2_OFFSET) != ATTR_AF) { atomic_add_long(&pmap_l2_p_failures, 1); CTR2(KTR_PMAP, "pmap_promote_l2: failure for va %#lx" " in pmap %p", va, pmap); return; } + if ((newl2 & (ATTR_AP_RW_BIT | ATTR_SW_DBM)) == + (ATTR_AP(ATTR_AP_RO) | ATTR_SW_DBM)) { + if (!atomic_fcmpset_64(l2, &newl2, newl2 & ~ATTR_SW_DBM)) + goto setl2; + newl2 &= ~ATTR_SW_DBM; + } + pa = newl2 + L2_SIZE - PAGE_SIZE; for (l3 = firstl3 + NL3PG - 1; l3 > firstl3; l3--) { oldl3 = pmap_load(l3); +setl3: + if ((oldl3 & (ATTR_AP_RW_BIT | ATTR_SW_DBM)) == + (ATTR_AP(ATTR_AP_RO) | ATTR_SW_DBM)) { + if (!atomic_fcmpset_64(l3, &oldl3, oldl3 & + ~ATTR_SW_DBM)) + goto setl3; + oldl3 &= ~ATTR_SW_DBM; + } if (oldl3 != pa) { atomic_add_long(&pmap_l2_p_failures, 1); CTR2(KTR_PMAP, "pmap_promote_l2: failure for va %#lx" @@ -3152,8 +3192,14 @@ pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, v new_l3 |= ATTR_SW_WIRED; if (va < VM_MAXUSER_ADDRESS) new_l3 |= ATTR_AP(ATTR_AP_USER) | ATTR_PXN; - if ((m->oflags & VPO_UNMANAGED) == 0) + if ((m->oflags & VPO_UNMANAGED) == 0) { new_l3 |= ATTR_SW_MANAGED; + if ((prot & VM_PROT_WRITE) != 0) { + new_l3 |= ATTR_SW_DBM; + if ((flags & VM_PROT_WRITE) == 0) + new_l3 |= ATTR_AP(ATTR_AP_RO); + } + } CTR2(KTR_PMAP, "pmap_enter: %.16lx -> %.16lx", va, pa); @@ -3341,6 +3387,7 @@ validate: KASSERT(opa == pa, ("pmap_enter: invalid update")); if ((orig_l3 & ~ATTR_AF) != (new_l3 & ~ATTR_AF)) { /* same PA, different attributes */ + /* XXXMJ need to reload orig_l3 for hardware DBM. */ pmap_load_store(l3, new_l3); pmap_invalidate_page(pmap, va); if ((orig_l3 & ATTR_SW_MANAGED) != 0 && @@ -3403,8 +3450,10 @@ pmap_enter_2mpage(pmap_t pmap, vm_offset_t va, vm_page new_l2 = (pd_entry_t)(VM_PAGE_TO_PHYS(m) | ATTR_DEFAULT | ATTR_IDX(m->md.pv_memattr) | ATTR_AP(ATTR_AP_RO) | L2_BLOCK); - if ((m->oflags & VPO_UNMANAGED) == 0) + if ((m->oflags & VPO_UNMANAGED) == 0) { new_l2 |= ATTR_SW_MANAGED; + new_l2 &= ~ATTR_AF; + } if ((prot & VM_PROT_EXECUTE) == 0 || m->md.pv_memattr == DEVICE_MEMORY) new_l2 |= ATTR_XN; if (va < VM_MAXUSER_ADDRESS) @@ -3704,8 +3753,10 @@ pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, v /* * Now validate mapping with RO protection */ - if ((m->oflags & VPO_UNMANAGED) == 0) + if ((m->oflags & VPO_UNMANAGED) == 0) { l3_val |= ATTR_SW_MANAGED; + l3_val &= ~ATTR_AF; + } /* Sync icache before the mapping is stored to PTE */ if ((prot & VM_PROT_EXECUTE) && pmap != kernel_pmap && @@ -3834,7 +3885,7 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_ struct rwlock *lock; struct spglist free; pd_entry_t *l0, *l1, *l2, srcptepaddr; - pt_entry_t *dst_pte, ptetemp, *src_pte; + pt_entry_t *dst_pte, mask, nbits, ptetemp, *src_pte; vm_offset_t addr, end_addr, va_next; vm_page_t dst_l2pg, dstmpte, srcmpte; @@ -3885,8 +3936,12 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_ ((srcptepaddr & ATTR_SW_MANAGED) == 0 || pmap_pv_insert_l2(dst_pmap, addr, srcptepaddr, PMAP_ENTER_NORECLAIM, &lock))) { - (void)pmap_load_store(l2, srcptepaddr & - ~ATTR_SW_WIRED); + mask = ATTR_AF | ATTR_SW_WIRED; + nbits = 0; + if ((srcptepaddr & ATTR_SW_DBM) != 0) + nbits |= ATTR_AP_RW_BIT; + (void)pmap_load_store(l2, + (srcptepaddr & ~mask) | nbits); pmap_resident_count_inc(dst_pmap, L2_SIZE / PAGE_SIZE); atomic_add_long(&pmap_l2_mappings, 1); @@ -3930,11 +3985,13 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_ /* * Clear the wired, modified, and accessed * (referenced) bits during the copy. - * - * XXX not yet */ - (void)pmap_load_store(dst_pte, ptetemp & - ~ATTR_SW_WIRED); + mask = ATTR_AF | ATTR_SW_WIRED; + nbits = 0; + if ((ptetemp & ATTR_SW_DBM) != 0) + nbits |= ATTR_AP_RW_BIT; + (void)pmap_load_store(dst_pte, + (ptetemp & ~mask) | nbits); pmap_resident_count_inc(dst_pmap, 1); } else { SLIST_INIT(&free); @@ -4560,7 +4617,7 @@ retry_pv_loop: } va = pv->pv_va; pte = pmap_pte(pmap, pv->pv_va, &lvl); - if ((pmap_load(pte) & ATTR_AP_RW_BIT) == ATTR_AP(ATTR_AP_RW)) + if ((pmap_load(pte) & ATTR_SW_DBM) != 0) (void)pmap_demote_l2_locked(pmap, pte, va, &lock); KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m), ("inconsistent pv lock %p %p for page %p", @@ -4583,13 +4640,14 @@ retry_pv_loop: } } pte = pmap_pte(pmap, pv->pv_va, &lvl); -retry: oldpte = pmap_load(pte); - if ((oldpte & ATTR_AP_RW_BIT) == ATTR_AP(ATTR_AP_RW)) { - if (!atomic_cmpset_long(pte, oldpte, - oldpte | ATTR_AP(ATTR_AP_RO))) +retry: + if ((oldpte & ATTR_SW_DBM) != 0) { + if (!atomic_fcmpset_long(pte, &oldpte, + (oldpte | ATTR_AP_RW_BIT) & ~ATTR_SW_DBM)) goto retry; - if ((oldpte & ATTR_AF) != 0) + if ((oldpte & ATTR_AP_RW_BIT) == + ATTR_AP(ATTR_AP_RW)) vm_page_dirty(m); pmap_invalidate_page(pmap, pv->pv_va); } @@ -4599,13 +4657,6 @@ retry: vm_page_aflag_clear(m, PGA_WRITEABLE); } -static __inline boolean_t -safe_to_clear_referenced(pmap_t pmap, pt_entry_t pte) -{ - - return (FALSE); -} - /* * pmap_ts_referenced: * @@ -4631,12 +4682,10 @@ pmap_ts_referenced(vm_page_t m) struct rwlock *lock; pd_entry_t *pde, tpde; pt_entry_t *pte, tpte; - pt_entry_t *l3; vm_offset_t va; vm_paddr_t pa; - int cleared, md_gen, not_cleared, lvl, pvh_gen; + int cleared, lvl, md_gen, not_cleared, pvh_gen; struct spglist free; - bool demoted; KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_ts_referenced: page %p is not managed", m)); @@ -4683,17 +4732,18 @@ retry: */ vm_page_dirty(m); } + if ((tpte & ATTR_AF) != 0) { /* - * Since this reference bit is shared by 512 4KB - * pages, it should not be cleared every time it is - * tested. Apply a simple "hash" function on the - * physical page number, the virtual superpage number, - * and the pmap address to select one 4KB page out of - * the 512 on which testing the reference bit will - * result in clearing that reference bit. This - * function is designed to avoid the selection of the - * same 4KB page for every 2MB page mapping. + * Since this reference bit is shared by 512 4KB pages, + * it should not be cleared every time it is tested. + * Apply a simple "hash" function on the physical page + * number, the virtual superpage number, and the pmap + * address to select one 4KB page out of the 512 on + * which testing the reference bit will result in + * clearing that reference bit. This function is + * designed to avoid the selection of the same 4KB page + * for every 2MB page mapping. * * On demotion, a mapping that hasn't been referenced * is simply destroyed. To avoid the possibility of a @@ -4705,39 +4755,9 @@ retry: if ((((pa >> PAGE_SHIFT) ^ (pv->pv_va >> L2_SHIFT) ^ (uintptr_t)pmap) & (Ln_ENTRIES - 1)) == 0 && (tpte & ATTR_SW_WIRED) == 0) { - if (safe_to_clear_referenced(pmap, tpte)) { - /* - * TODO: We don't handle the access - * flag at all. We need to be able - * to set it in the exception handler. - */ - panic("ARM64TODO: " - "safe_to_clear_referenced\n"); - } else if (pmap_demote_l2_locked(pmap, pte, - pv->pv_va, &lock) != NULL) { - demoted = true; - va += VM_PAGE_TO_PHYS(m) - - (tpte & ~ATTR_MASK); - l3 = pmap_l2_to_l3(pte, va); - pmap_remove_l3(pmap, l3, va, - pmap_load(pte), NULL, &lock); - } else - demoted = true; - - if (demoted) { - /* - * The superpage mapping was removed - * entirely and therefore 'pv' is no - * longer valid. - */ - if (pvf == pv) - pvf = NULL; - pv = NULL; - } + pmap_clear_bits(pte, ATTR_AF); + pmap_invalidate_page(pmap, pv->pv_va); cleared++; - KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m), - ("inconsistent pv lock %p %p for page %p", - lock, VM_PAGE_TO_PV_LIST_LOCK(m), m)); } else not_cleared++; } @@ -4782,29 +4802,10 @@ small_mappings: if (pmap_pte_dirty(tpte)) vm_page_dirty(m); if ((tpte & ATTR_AF) != 0) { - if (safe_to_clear_referenced(pmap, tpte)) { - /* - * TODO: We don't handle the access flag - * at all. We need to be able to set it in - * the exception handler. - */ - panic("ARM64TODO: safe_to_clear_referenced\n"); - } else if ((tpte & ATTR_SW_WIRED) == 0) { - /* - * Wired pages cannot be paged out so - * doing accessed bit emulation for - * them is wasted effort. We do the - * hard work for unwired pages only. - */ - pmap_remove_l3(pmap, pte, pv->pv_va, tpde, - &free, &lock); + if ((tpte & ATTR_SW_WIRED) == 0) { + pmap_clear_bits(pte, ATTR_AF); + pmap_invalidate_page(pmap, pv->pv_va); cleared++; - if (pvf == pv) - pvf = NULL; - pv = NULL; - KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m), - ("inconsistent pv lock %p %p for page %p", - lock, VM_PAGE_TO_PV_LIST_LOCK(m), m)); } else not_cleared++; } @@ -4839,6 +4840,14 @@ pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t void pmap_clear_modify(vm_page_t m) { + struct md_page *pvh; + struct rwlock *lock; + pmap_t pmap; + pv_entry_t next_pv, pv; + pd_entry_t *l2, oldl2; + pt_entry_t *l3, oldl3; + vm_offset_t va; + int md_gen, pvh_gen; KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_clear_modify: page %p is not managed", m)); @@ -4847,14 +4856,81 @@ pmap_clear_modify(vm_page_t m) ("pmap_clear_modify: page %p is exclusive busied", m)); /* - * If the page is not PGA_WRITEABLE, then no PTEs can have PG_M set. - * If the object containing the page is locked and the page is not + * If the page is not PGA_WRITEABLE, then no PTEs can have ATTR_SW_DBM + * set. If the object containing the page is locked and the page is not * exclusive busied, then PGA_WRITEABLE cannot be concurrently set. */ if ((m->aflags & PGA_WRITEABLE) == 0) return; - - /* ARM64TODO: We lack support for tracking if a page is modified */ + pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : + pa_to_pvh(VM_PAGE_TO_PHYS(m)); + lock = VM_PAGE_TO_PV_LIST_LOCK(m); + rw_wlock(lock); +restart: + TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_next, next_pv) { + pmap = PV_PMAP(pv); + if (!PMAP_TRYLOCK(pmap)) { + pvh_gen = pvh->pv_gen; + rw_wunlock(lock); + PMAP_LOCK(pmap); + rw_wlock(lock); + if (pvh_gen != pvh->pv_gen) { + PMAP_UNLOCK(pmap); + goto restart; + } + } + va = pv->pv_va; + l2 = pmap_l2(pmap, va); + oldl2 = pmap_load(l2); + if ((oldl2 & ATTR_SW_DBM) != 0) { + if (pmap_demote_l2_locked(pmap, l2, va, &lock)) { + if ((oldl2 & ATTR_SW_WIRED) == 0) { + /* + * Write protect the mapping to a + * single page so that a subsequent + * write access may repromote. + */ + va += VM_PAGE_TO_PHYS(m) - + (oldl2 & ~ATTR_MASK); + l3 = pmap_l2_to_l3(l2, va); + oldl3 = pmap_load(l3); + if (pmap_l3_valid(oldl3)) { + while (!atomic_fcmpset_long(l3, + &oldl3, (oldl3 & ~ATTR_SW_DBM) | + ATTR_AP(ATTR_AP_RO))) + cpu_spinwait(); + vm_page_dirty(m); + pmap_invalidate_page(pmap, va); + } + } + } + } + PMAP_UNLOCK(pmap); + } + TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { + pmap = PV_PMAP(pv); + if (!PMAP_TRYLOCK(pmap)) { + md_gen = m->md.pv_gen; + pvh_gen = pvh->pv_gen; + rw_wunlock(lock); + PMAP_LOCK(pmap); + rw_wlock(lock); + if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) { + PMAP_UNLOCK(pmap); + goto restart; + } + } + l2 = pmap_l2(pmap, pv->pv_va); + l3 = pmap_l2_to_l3(l2, pv->pv_va); + oldl3 = pmap_load(l3); + if (pmap_l3_valid(oldl3) && + (oldl3 & (ATTR_AP_RW_BIT | ATTR_SW_DBM)) == ATTR_SW_DBM) { + pmap_set_bits(l3, ATTR_AP(ATTR_AP_RO)); + pmap_invalidate_page(pmap, pv->pv_va); + } + PMAP_UNLOCK(pmap); + } + rw_wunlock(lock); } void * @@ -5328,14 +5404,15 @@ pmap_demote_l2_locked(pmap_t pmap, pt_entry_t *l2, vm_ pmap_resident_count_inc(pmap, 1); } } - l3phys = VM_PAGE_TO_PHYS(ml3); l3 = (pt_entry_t *)PHYS_TO_DMAP(l3phys); - /* Address the range points at */ phys = oldl2 & ~ATTR_MASK; /* The attributed from the old l2 table to be copied */ newl3 = (oldl2 & (ATTR_MASK & ~ATTR_DESCR_MASK)) | L3_PAGE; + KASSERT((oldl2 & (ATTR_AP_RW_BIT | ATTR_SW_DBM)) != + (ATTR_AP(ATTR_AP_RO) | ATTR_SW_DBM), + ("pmap_demote_l2: L2 entry is writeable but not dirty")); /* * If the page table page is not leftover from an earlier promotion, @@ -5442,15 +5519,16 @@ retry: panic("pmap_mincore: invalid level %d", lvl); } + managed = (tpte & ATTR_SW_MANAGED) != 0; val = MINCORE_INCORE; if (lvl != 3) val |= MINCORE_SUPER; - if (pmap_pte_dirty(tpte)) + if ((managed && pmap_pte_dirty(tpte)) || (!managed && + (tpte & ATTR_AP_RW_BIT) == ATTR_AP(ATTR_AP_RW))) val |= MINCORE_MODIFIED | MINCORE_MODIFIED_OTHER; if ((tpte & ATTR_AF) == ATTR_AF) val |= MINCORE_REFERENCED | MINCORE_REFERENCED_OTHER; - managed = (tpte & ATTR_SW_MANAGED) == ATTR_SW_MANAGED; pa = (tpte & ~ATTR_MASK) | (addr & mask); } else managed = false; @@ -5560,22 +5638,58 @@ pmap_sync_icache(pmap_t pmap, vm_offset_t va, vm_size_ int pmap_fault(pmap_t pmap, uint64_t esr, uint64_t far) { -#ifdef SMP + pt_entry_t *pte; register_t intr; - uint64_t par; + uint64_t ec, par; + int lvl, rv; - switch (ESR_ELx_EXCEPTION(esr)) { + rv = KERN_FAILURE; + + ec = ESR_ELx_EXCEPTION(esr); + switch (ec) { case EXCP_INSN_ABORT_L: case EXCP_INSN_ABORT: case EXCP_DATA_ABORT_L: case EXCP_DATA_ABORT: break; default: - return (KERN_FAILURE); + return (rv); } - /* Data and insn aborts use same encoding for FCS field. */ + /* Data and insn aborts use same encoding for FSC field. */ switch (esr & ISS_DATA_DFSC_MASK) { + case ISS_DATA_DFSC_AFF_L1: + case ISS_DATA_DFSC_AFF_L2: + case ISS_DATA_DFSC_AFF_L3: + PMAP_LOCK(pmap); + pte = pmap_pte(pmap, far, &lvl); + if (pte != NULL) { + pmap_set_bits(pte, ATTR_AF); + rv = KERN_SUCCESS; + /* + * XXXMJ as an optimization we could mark the entry + * dirty if this is a write fault. + */ + } + PMAP_UNLOCK(pmap); + break; + case ISS_DATA_DFSC_PF_L1: + case ISS_DATA_DFSC_PF_L2: + case ISS_DATA_DFSC_PF_L3: + if ((ec != EXCP_DATA_ABORT_L && ec != EXCP_DATA_ABORT) || + (esr & ISS_DATA_WnR) == 0) + return (rv); + PMAP_LOCK(pmap); + pte = pmap_pte(pmap, far, &lvl); + if (pte != NULL && + (pmap_load(pte) & (ATTR_AP_RW_BIT | ATTR_SW_DBM)) == + (ATTR_AP(ATTR_AP_RO) | ATTR_SW_DBM)) { + pmap_clear_bits(pte, ATTR_AP_RW_BIT); + pmap_invalidate_page(pmap, trunc_page(far)); + rv = KERN_SUCCESS; + } + PMAP_UNLOCK(pmap); + break; case ISS_DATA_DFSC_TF_L0: case ISS_DATA_DFSC_TF_L1: case ISS_DATA_DFSC_TF_L2: @@ -5596,14 +5710,11 @@ pmap_fault(pmap_t pmap, uint64_t esr, uint64_t far) * return success to the trap handler. */ if (PAR_SUCCESS(par)) - return (KERN_SUCCESS); + rv = KERN_SUCCESS; break; - default: - break; } -#endif - return (KERN_FAILURE); + return (rv); } /* Modified: head/sys/arm64/arm64/trap.c ============================================================================== --- head/sys/arm64/arm64/trap.c Mon Jul 15 15:45:33 2019 (r350003) +++ head/sys/arm64/arm64/trap.c Mon Jul 15 17:13:32 2019 (r350004) @@ -192,32 +192,16 @@ data_abort(struct thread *td, struct trapframe *frame, } /* - * The call to pmap_fault can be dangerous when coming from the - * kernel as it may be not be able to lock the pmap to check if - * the address is now valid. Because of this we filter the cases - * when we are not going to see superpage activity. + * Try to handle translation, access flag, and permission faults. + * Translation faults may occur as a result of the required + * break-before-make sequence used when promoting or demoting + * superpages. Such faults must not occur while holding the pmap lock, + * or pmap_fault() will recurse on that lock. */ - if (!lower) { - /* - * We may fault in a DMAP region due to a superpage being - * unmapped when the access took place. - */ - if (map == kernel_map && !VIRT_IN_DMAP(far)) - goto no_pmap_fault; - /* - * We can also fault in the userspace handling functions, - * e.g. copyin. In these cases we will have set a fault - * handler so we can check if this is set before calling - * pmap_fault. - */ - if (map != kernel_map && pcb->pcb_onfault == 0) - goto no_pmap_fault; - } - - if (pmap_fault(map->pmap, esr, far) == KERN_SUCCESS) + if ((lower || map == kernel_map || pcb->pcb_onfault != 0) && + pmap_fault(map->pmap, esr, far) == KERN_SUCCESS) return; -no_pmap_fault: KASSERT(td->td_md.md_spinlock_count == 0, ("data abort with spinlock held")); if (td->td_critnest != 0 || WITNESS_CHECK(WARN_SLEEPOK | @@ -229,9 +213,11 @@ no_pmap_fault: } va = trunc_page(far); - ftype = ((esr >> 6) & 1) ? VM_PROT_READ | VM_PROT_WRITE : VM_PROT_READ; if (exec) - ftype |= VM_PROT_EXECUTE; + ftype = VM_PROT_EXECUTE; + else + ftype = (esr & ISS_DATA_WnR) == 0 ? VM_PROT_READ : + VM_PROT_READ | VM_PROT_WRITE; /* Fault in the page. */ error = vm_fault(map, va, ftype, VM_FAULT_NORMAL); Modified: head/sys/arm64/include/pte.h ============================================================================== --- head/sys/arm64/include/pte.h Mon Jul 15 15:45:33 2019 (r350003) +++ head/sys/arm64/include/pte.h Mon Jul 15 17:13:32 2019 (r350004) @@ -39,11 +39,12 @@ typedef uint64_t pt_entry_t; /* page table entry */ #endif /* Block and Page attributes */ -/* TODO: Add the upper attributes */ #define ATTR_MASK_H UINT64_C(0xfff0000000000000) #define ATTR_MASK_L UINT64_C(0x0000000000000fff) #define ATTR_MASK (ATTR_MASK_H | ATTR_MASK_L) /* Bits 58:55 are reserved for software */ +#define ATTR_SW_UNUSED (1UL << 58) +#define _ATTR_SW_DBM (1UL << 57) #define ATTR_SW_MANAGED (1UL << 56) #define ATTR_SW_WIRED (1UL << 55) #define ATTR_UXN (1UL << 54) _______________________________________________ svn-src-head@freebsd.org mailing list https://lists.freebsd.org/mailman/listinfo/svn-src-head To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org"
