Author: kib Date: Fri Jan 4 17:33:07 2019 New Revision: 342769 URL: https://svnweb.freebsd.org/changeset/base/342769
Log: i386: Use atomic 64bit load to read PDE value from PAE pagetables in pmap_kextract(). pmap_kextract() can race with promotion/demotion on the kernel page table, in which case current non-atomic 64bit read would see torn value, breaking pmap_kextract(). pmap_kextract() would correctly handle either promoted or demoted PDE, but not a mix where one word is from a different state. It requires PAE and > 4G memory to reproduce. We observed this in real loads, both for intensive use of malloc(9)/free(9) where vtoslab() returned invalid pointer to the slab, and with the use of busdma_bounce, where incorrect page was bounced. In collaboration with: pho Reviewed by: markj Sponsored by: The FreeBSD Foundation MFC after: 1 week Differential revision: https://reviews.freebsd.org/D18714 Modified: head/sys/i386/include/pmap.h Modified: head/sys/i386/include/pmap.h ============================================================================== --- head/sys/i386/include/pmap.h Fri Jan 4 17:31:50 2019 (r342768) +++ head/sys/i386/include/pmap.h Fri Jan 4 17:33:07 2019 (r342769) @@ -234,6 +234,32 @@ extern pd_entry_t *IdlePTD; /* physical address of "Id */ extern pt_entry_t *KPTmap; +#if (defined(PAE) || defined(PAE_TABLES)) + +#define pde_cmpset(pdep, old, new) atomic_cmpset_64_i586(pdep, old, new) +#define pte_load_store(ptep, pte) atomic_swap_64_i586(ptep, pte) +#define pte_load_clear(ptep) atomic_swap_64_i586(ptep, 0) +#define pte_store(ptep, pte) atomic_store_rel_64_i586(ptep, pte) +#define pte_load(ptep) atomic_load_acq_64_i586(ptep) + +extern pt_entry_t pg_nx; + +#else /* !(PAE || PAE_TABLES) */ + +#define pde_cmpset(pdep, old, new) atomic_cmpset_int(pdep, old, new) +#define pte_load_store(ptep, pte) atomic_swap_int(ptep, pte) +#define pte_load_clear(ptep) atomic_swap_int(ptep, 0) +#define pte_store(ptep, pte) do { \ + *(u_int *)(ptep) = (u_int)(pte); \ +} while (0) +#define pte_load(ptep) atomic_load_acq_int(ptep) + +#endif /* !(PAE || PAE_TABLES) */ + +#define pte_clear(ptep) pte_store(ptep, 0) + +#define pde_store(pdep, pde) pte_store(pdep, pde) + /* * Extract from the kernel page table the physical address that is mapped by * the given virtual address "va". @@ -245,7 +271,7 @@ pmap_kextract(vm_offset_t va) { vm_paddr_t pa; - if ((pa = PTD[va >> PDRSHIFT]) & PG_PS) { + if ((pa = pte_load(&PTD[va >> PDRSHIFT])) & PG_PS) { pa = (pa & PG_PS_FRAME) | (va & PDRMASK); } else { /* @@ -260,30 +286,6 @@ pmap_kextract(vm_offset_t va) } return (pa); } - -#if (defined(PAE) || defined(PAE_TABLES)) - -#define pde_cmpset(pdep, old, new) atomic_cmpset_64_i586(pdep, old, new) -#define pte_load_store(ptep, pte) atomic_swap_64_i586(ptep, pte) -#define pte_load_clear(ptep) atomic_swap_64_i586(ptep, 0) -#define pte_store(ptep, pte) atomic_store_rel_64_i586(ptep, pte) - -extern pt_entry_t pg_nx; - -#else /* !(PAE || PAE_TABLES) */ - -#define pde_cmpset(pdep, old, new) atomic_cmpset_int(pdep, old, new) -#define pte_load_store(ptep, pte) atomic_swap_int(ptep, pte) -#define pte_load_clear(ptep) atomic_swap_int(ptep, 0) -#define pte_store(ptep, pte) do { \ - *(u_int *)(ptep) = (u_int)(pte); \ -} while (0) - -#endif /* !(PAE || PAE_TABLES) */ - -#define pte_clear(ptep) pte_store(ptep, 0) - -#define pde_store(pdep, pde) pte_store(pdep, pde) #endif /* _KERNEL */ _______________________________________________ svn-src-head@freebsd.org mailing list https://lists.freebsd.org/mailman/listinfo/svn-src-head To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org"
