In order to lay the foundation for work that permits us to track the virtual page offset of MAP_PRIVATE file-backed mappings, we abstract the assignment of vma->vm_pgoff to vma_set_pgoff().
We additionally add a lock check here using the newly introduced vma_assert_can_modify(). This asserts the VMA write lock if the VMA is attached. We also assert that, if this is an anonymous VMA and unfaulted, that its (virtual) page offset is equal to the page offset of the VMA's address. In order to maintain correctness given this assert, we also update __install_special_mapping() to invoke vma_set_range() after it's set vma->vm_ops (which determine whether the VMA is anonymous or not). We do not use vma_set_pgoff() in vm_area_init_from(), as at the point of forking, we don't necessarily have correct locking state. Updating vma_set_range() covers most cases, but in addition to this we also update insert_vm_struct(), compat_set_vma_from_desc() and nommu callers. We also update vma_add_pgoff() and vma_sub_pgoff() to use vma_set_pgoff(). While we're here, we drop a BUG_ON() and update insert_vm_struct()'s comment to reflect the fact anonymous mappings can be added here. No functional change intended. Signed-off-by: Lorenzo Stoakes <[email protected]> --- mm/nommu.c | 2 +- mm/vma.c | 14 +++++++------- mm/vma.h | 15 ++++++++++++--- tools/testing/vma/include/dup.h | 2 +- 4 files changed, 21 insertions(+), 12 deletions(-) diff --git a/mm/nommu.c b/mm/nommu.c index c7fafcd87c14..ba1c923c0942 100644 --- a/mm/nommu.c +++ b/mm/nommu.c @@ -1059,7 +1059,7 @@ unsigned long do_mmap(struct file *file, region->vm_pgoff = pgoff; vm_flags_init(vma, vm_flags); - vma->vm_pgoff = pgoff; + vma_set_pgoff(vma, pgoff); if (file) { region->vm_file = get_file(file); diff --git a/mm/vma.c b/mm/vma.c index 0579fc8c9bd5..d727150e377a 100644 --- a/mm/vma.c +++ b/mm/vma.c @@ -81,7 +81,7 @@ static void vma_set_range(struct vm_area_struct *vma, unsigned long start, unsigned long end, pgoff_t pgoff) { __vma_set_range(vma, start, end); - vma->vm_pgoff = pgoff; + vma_set_pgoff(vma, pgoff); } /* Was this VMA ever forked from a parent, i.e. maybe contains CoW mappings? */ @@ -3345,9 +3345,9 @@ int __vm_munmap(unsigned long start, size_t len, bool unlock) return ret; } -/* Insert vm structure into process list sorted by address - * and into the inode's i_mmap tree. If vm_file is non-NULL - * then i_mmap_rwsem is taken here. +/* + * Insert vm structure into process list sorted by address + * and into the inode's i_mmap tree if file-backed. */ int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma) { @@ -3373,8 +3373,8 @@ int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma) * Similarly in do_mmap and in do_brk_flags. */ if (vma_is_anonymous(vma)) { - BUG_ON(vma->anon_vma); - vma->vm_pgoff = vma->vm_start >> PAGE_SHIFT; + WARN_ON_ONCE(vma->anon_vma); + vma_set_pgoff(vma, vma->vm_start >> PAGE_SHIFT); } if (vma_link(mm, vma)) { @@ -3420,7 +3420,6 @@ struct vm_area_struct *__install_special_mapping( if (unlikely(vma == NULL)) return ERR_PTR(-ENOMEM); - vma_set_range(vma, addr, addr + len, 0); vm_flags |= mm->def_flags | VM_DONTEXPAND; if (pgtable_supports_soft_dirty()) vm_flags |= VM_SOFTDIRTY; @@ -3429,6 +3428,7 @@ struct vm_area_struct *__install_special_mapping( vma->vm_ops = ops; vma->vm_private_data = priv; + vma_set_range(vma, addr, addr + len, 0); ret = insert_vm_struct(mm, vma); if (ret) diff --git a/mm/vma.h b/mm/vma.h index 9658e0c678ad..155eadda47aa 100644 --- a/mm/vma.h +++ b/mm/vma.h @@ -247,16 +247,25 @@ static inline pgoff_t vmg_end_pgoff(const struct vma_merge_struct *vmg) return vmg_start_pgoff(vmg) + vmg_pages(vmg); } +static inline void vma_set_pgoff(struct vm_area_struct *vma, pgoff_t pgoff) +{ + vma_assert_can_modify(vma); + + VM_WARN_ON_ONCE(vma_is_anonymous(vma) && !vma->anon_vma && + pgoff != vma->vm_start >> PAGE_SHIFT); + vma->vm_pgoff = pgoff; +} + static inline void vma_add_pgoff(struct vm_area_struct *vma, pgoff_t delta) { vma_assert_can_modify(vma); - vma->vm_pgoff += delta; + vma_set_pgoff(vma, vma_start_pgoff(vma) + delta); } static inline void vma_sub_pgoff(struct vm_area_struct *vma, pgoff_t delta) { vma_assert_can_modify(vma); - vma->vm_pgoff -= delta; + vma_set_pgoff(vma, vma_start_pgoff(vma) - delta); } #define VMG_STATE(name, mm_, vmi_, start_, end_, vma_flags_, pgoff_) \ @@ -332,7 +341,7 @@ static inline void compat_set_vma_from_desc(struct vm_area_struct *vma, */ /* Mutable fields. Populated with initial state. */ - vma->vm_pgoff = desc->pgoff; + vma_set_pgoff(vma, desc->pgoff); if (desc->vm_file != vma->vm_file) vma_set_file(vma, desc->vm_file); vma->flags = desc->vma_flags; diff --git a/tools/testing/vma/include/dup.h b/tools/testing/vma/include/dup.h index 41fea90a344d..5d7d0afd7765 100644 --- a/tools/testing/vma/include/dup.h +++ b/tools/testing/vma/include/dup.h @@ -1186,7 +1186,7 @@ static inline void vma_assert_can_modify(struct vm_area_struct *vma) static inline void vma_assert_detached(struct vm_area_struct *vma) { - WARN_ON_ONCE(refcount_read(&vma->vm_refcnt)); + WARN_ON_ONCE(vma_is_attached(vma)); } static inline void vma_assert_write_locked(struct vm_area_struct *); -- 2.54.0
