When mapping /dev/zero with MAP_PRIVATE, one ends up with strange VMAs originating from Linux's distant past.
These have vma->vm_file set but NULL vma->vm_ops, meaning they satisfy vma_is_anonymous() but otherwise resemble a file-backed VMA. The introduction of virtual page offsets and their subsequent use as indexes for MAP_PRIVATE-file-backed mappings mean the rmap does the right thing with these but we are left with inconsistencies. The vma_start_pgoff(vma) == vma_start_virt_pgoff(vma) invariant is true for all other anonymous VMAs, but not these. These VMAs are also observable as files in /proc/<pid>/[s]maps but otherwise behave like anonymous mappings. Therefore let's make these VMAs actually anonymous at mapping time which will activate the anonymous code path for mappings. This means we no longer have to account for this discrepancy anywhere and no longer have to think about these at all. A previous commit gave us map_is_dev_zero() to positively identify these mappings, so we expressly only do so for these alone. The impact of this change should be low as likely very few are relying upon this in any case, and in using them are asking for anonymous memory, so no longer seeing these as file mappings in smaps should have no meaningful impact. Update assert_sane_pgoff(), the comment for vma_start_pgoff() and linear_virt_page_index() to reflect the change. We make this change in call_mmap_prepare() alone as /dev/zero has been converted to an mmap_prepare hook and we do not permit nested MAP_PRIVATE mapping of /dev/zero. We also remove the now defunct vma_desc_set_anonymous() and eliminate the temporary bisection hazard fix from the previous commit. Also update the VMA userland tests to reflect the change. Signed-off-by: Lorenzo Stoakes (ARM) <[email protected]> --- include/linux/mm.h | 10 ++-------- include/linux/pagemap.h | 3 +-- mm/vma.c | 26 +++++++++++++++++--------- mm/vma.h | 3 --- tools/testing/vma/include/dup.h | 3 +-- 5 files changed, 21 insertions(+), 24 deletions(-) diff --git a/include/linux/mm.h b/include/linux/mm.h index 7614afe99c96..7fabe6c66b4b 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1554,11 +1554,6 @@ static inline void vma_set_anonymous(struct vm_area_struct *vma) vma->vm_ops = NULL; } -static inline void vma_desc_set_anonymous(struct vm_area_desc *desc) -{ - desc->vm_ops = NULL; -} - static inline bool vma_is_anonymous(const struct vm_area_struct *vma) { return !vma->vm_ops; @@ -4352,9 +4347,8 @@ static inline unsigned long vma_pages(const struct vm_area_struct *vma) * If @vma is a MAP_PRIVATE file-backed mapping, then this returns the * page offset within the file. * - * Edge cases: nommu does not abide by these, MAP_PRIVATE-/dev/zero satisfies - * vma_is_anonymous() but has file-backed page offset, and MAP_PRIVATE-pfnmap - * regions have their page offset set to the first PFN in the range. + * Edge cases: nommu does not abide by these and CoW MAP_PRIVATE-pfnmap regions + * have their page offset set to the first PFN in the range. * * Returns: The page offset of the start of @vma. */ diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index 3307c0551a0a..4e8b2b29f6d3 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -1136,8 +1136,7 @@ static inline pgoff_t linear_virt_page_index(const struct vm_area_struct *vma, const pgoff_t pgoff = __linear_virt_page_index(vma, address); VM_WARN_ON_ONCE(vma_test(vma, VMA_SHARED_BIT)); - /* Account for MAP_PRIVATE-/dev/zero which is only semi-anonymous. */ - if (vma_is_anonymous(vma) && !vma->vm_file) + if (vma_is_anonymous(vma)) VM_WARN_ON_ONCE(pgoff != linear_page_index(vma, address)); return pgoff; diff --git a/mm/vma.c b/mm/vma.c index 103a6a134dec..d3e0457a11a1 100644 --- a/mm/vma.c +++ b/mm/vma.c @@ -2622,6 +2622,13 @@ static bool map_is_dev_zero(const struct mmap_state *map) return imajor(inode) == MEM_MAJOR && iminor(inode) == DEVZERO_MINOR; } +static void map_set_anon(struct mmap_state *map) +{ + map->file = NULL; + map->vm_ops = NULL; + map->pgoff = map->addr >> PAGE_SHIFT; +} + static bool map_is_private(const struct mmap_state *map) { return !vma_flags_test(&map->vma_flags, VMA_SHARED_BIT); @@ -2629,10 +2636,7 @@ static bool map_is_private(const struct mmap_state *map) static bool map_is_anon(const struct mmap_state *map) { - if (!map_is_private(map)) - return false; - - return !map->file || map_is_dev_zero(map); + return map_is_private(map) && !map->file; } /* @@ -2664,7 +2668,7 @@ static int __mmap_new_vma(struct mmap_state *map, struct vm_area_struct **vmap, vma_iter_config(vmi, map->addr, map->end); - if (is_anon && !map->file) + if (is_anon) vma_set_anonymous(vma); vma_set_range(vma, map->addr, map->end, map->pgoff, map->virt_pgoff); @@ -2682,10 +2686,6 @@ static int __mmap_new_vma(struct mmap_state *map, struct vm_area_struct **vmap, else if (!is_anon) error = shmem_zero_setup(vma); - /* Temporary MAP_PRIVATE-/dev/zero workaround. */ - if (is_anon && map->file) - vma_set_anonymous(vma); - if (error) goto free_iter_vma; @@ -2814,6 +2814,14 @@ static int call_mmap_prepare(struct mmap_state *map, map->vm_ops = desc->vm_ops; map->vm_private_data = desc->private_data; + /* + * MAP_PRIVATE-/dev/zero mappings are an ancient way of getting + * anonymous mappings. Rather than allowing these mappings to be odd + * outliers, simply make them truly anonymous. + */ + if (map_is_private(map) && map_is_dev_zero(map)) + map_set_anon(map); + return 0; } diff --git a/mm/vma.h b/mm/vma.h index 94277837ce6e..44fbb50dc3fe 100644 --- a/mm/vma.h +++ b/mm/vma.h @@ -267,9 +267,6 @@ static inline void assert_sane_pgoff(struct vm_area_struct *vma, pgoff_t pgoff) */ if (!vma_is_anonymous(vma)) return; - /* MAP_PRIVATE-/dev/zero is anon, non-NULL vm_file, but has file pgoff. */ - if (vma->vm_file) - return; /* If faulted in, could have been remapped. */ if (vma->anon_vma) return; diff --git a/tools/testing/vma/include/dup.h b/tools/testing/vma/include/dup.h index c617ee2532bd..c5191b9718fd 100644 --- a/tools/testing/vma/include/dup.h +++ b/tools/testing/vma/include/dup.h @@ -1648,8 +1648,7 @@ static inline pgoff_t linear_virt_page_index(const struct vm_area_struct *vma, const pgoff_t pgoff = __linear_virt_page_index(vma, address); VM_WARN_ON_ONCE(vma_test(vma, VMA_SHARED_BIT)); - /* Account for MAP_PRIVATE-/dev/zero which is only semi-anonymous. */ - if (vma_is_anonymous(vma) && !vma->vm_file) + if (vma_is_anonymous(vma)) VM_WARN_ON_ONCE(pgoff != linear_page_index(vma, address)); return pgoff; -- 2.55.0
