Propagate crash_memaction marks set through madvise() to the crash_memaction registry. Register the folios of a VMA carrying VM_CRASH_MARK with the crash memaction registry as they are mapped, and update the registry when a folio moves.
Marks are processed in rmap since the fault paths, CoW, swapin, migration and khugepaged collapse all converge here. The marks don't affect pinning, reclaim, compaction or hot-unplug. De-regsitering free'd marked pages is done in post_alloc_hook() and the hugetlb pool to account for stale data in free'd pages. The marking code is gated behind a static key that is only enabled when the crash_memaction registry is explicitly enabled by a cmdline argument, so there should be no performance impact unless the feature is actually used. Signed-off-by: Jan Sebastian Götte <[email protected]> Assisted-by: Claude Opus 5 <[email protected]> --- include/linux/crash_memaction.h | 15 +++++++++++++++ include/linux/rmap.h | 5 ++++- mm/huge_memory.c | 2 ++ mm/hugetlb.c | 6 +++--- mm/migrate.c | 2 +- mm/rmap.c | 8 ++++++++ mm/userfaultfd.c | 1 + 7 files changed, 34 insertions(+), 5 deletions(-) diff --git a/include/linux/crash_memaction.h b/include/linux/crash_memaction.h index 5f3e114c60ad..2de60bc12dd9 100644 --- a/include/linux/crash_memaction.h +++ b/include/linux/crash_memaction.h @@ -4,6 +4,7 @@ #include <linux/init.h> #include <linux/jump_label.h> +#include <linux/mm.h> #include <linux/types.h> struct kimage; @@ -50,6 +51,18 @@ static inline void crash_memaction_unmark_pfns(unsigned long pfn, __crash_memaction_unmark_pfns(pfn, nr_pages); } +static inline void crash_mark_pages(struct page *page, int nr_pages, + struct vm_area_struct *vma) +{ + if (!static_branch_unlikely(&crash_memaction_active)) + return; + + if (likely(!(vma->vm_flags & VM_CRASH_MARK))) + return; + + __crash_memaction_mark_pfns(page_to_pfn(page), nr_pages); +} + void crash_memaction_mark(void *addr, size_t size, int types); void crash_memaction_unmark(void *addr, size_t size); @@ -62,6 +75,8 @@ void crash_memaction_unload(struct kimage *image); static inline void crash_memaction_init(void) { } static inline void crash_memaction_unmark_pfns(unsigned long pfn, unsigned long nr_pages) { } +static inline void crash_mark_pages(struct page *page, int nr_pages, + struct vm_area_struct *vma) { } static inline void crash_memaction_mark(void *addr, size_t size, int types) { } static inline void crash_memaction_unmark(void *addr, size_t size) { } static inline int crash_memaction_types(void) { return 0; } diff --git a/include/linux/rmap.h b/include/linux/rmap.h index 74cca0e3c726..1eeb7762b4a3 100644 --- a/include/linux/rmap.h +++ b/include/linux/rmap.h @@ -9,6 +9,7 @@ #include <linux/slab.h> #include <linux/mm.h> #include <linux/rwsem.h> +#include <linux/crash_memaction.h> #include <linux/memcontrol.h> #include <linux/highmem.h> #include <linux/pagemap.h> @@ -472,13 +473,15 @@ static inline int hugetlb_try_share_anon_rmap(struct folio *folio) return 0; } -static inline void hugetlb_add_file_rmap(struct folio *folio) +static inline void hugetlb_add_file_rmap(struct folio *folio, + struct vm_area_struct *vma) { VM_WARN_ON_FOLIO(!folio_test_hugetlb(folio), folio); VM_WARN_ON_FOLIO(folio_test_anon(folio), folio); atomic_inc(&folio->_entire_mapcount); atomic_inc(&folio->_large_mapcount); + crash_mark_pages(&folio->page, folio_nr_pages(folio), vma); } static inline void hugetlb_remove_rmap(struct folio *folio) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 3fb9504dff7a..adbda968d58e 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -2966,6 +2966,8 @@ int move_pages_huge_pmd(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, pm folio_move_anon_rmap(src_folio, dst_vma); src_folio->index = linear_anon_page_index(dst_vma, dst_addr); + /* No rmap add, and the two VMAs need not agree on the flag. */ + crash_mark_pages(&src_folio->page, HPAGE_PMD_NR, dst_vma); _dst_pmd = folio_mk_pmd(src_folio, dst_vma->vm_page_prot); /* Follow mremap() behavior and treat the entry dirty after the move */ diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 36f0c0d8f5f1..309be0b301ca 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -5063,7 +5063,7 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src, * sleep during the process. */ if (!folio_test_anon(pte_folio)) { - hugetlb_add_file_rmap(pte_folio); + hugetlb_add_file_rmap(pte_folio, dst_vma); } else if (hugetlb_try_dup_anon_rmap(pte_folio, src_vma)) { pte_t src_pte_old = entry; struct folio *new_folio; @@ -5982,7 +5982,7 @@ static vm_fault_t hugetlb_no_page(struct address_space *mapping, if (new_anon_folio) hugetlb_add_new_anon_rmap(folio, vma, vmf->address); else - hugetlb_add_file_rmap(folio); + hugetlb_add_file_rmap(folio, vma); new_pte = make_huge_pte(vma, folio, vma->vm_flags & VM_SHARED); /* * If this pte was previously wr-protected, keep it wr-protected even @@ -6502,7 +6502,7 @@ int hugetlb_mfill_atomic_pte(pte_t *dst_pte, goto out_release_unlock; if (folio_in_pagecache) - hugetlb_add_file_rmap(folio); + hugetlb_add_file_rmap(folio, dst_vma); else hugetlb_add_new_anon_rmap(folio, dst_vma, dst_addr); diff --git a/mm/migrate.c b/mm/migrate.c index 7bdcdb57652f..467b695e6dd6 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -434,7 +434,7 @@ static bool remove_migration_pte(struct folio *folio, hugetlb_add_anon_rmap(folio, vma, pvmw.address, rmap_flags); else - hugetlb_add_file_rmap(folio); + hugetlb_add_file_rmap(folio, vma); set_huge_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte, psize); } else diff --git a/mm/rmap.c b/mm/rmap.c index fbd66a2823b7..21183478f606 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -1582,6 +1582,8 @@ static __always_inline void __folio_add_anon_rmap(struct folio *folio, */ if (folio_nr_pages(folio) == nr_pages) mlock_vma_folio(folio, vma); + + crash_mark_pages(page, nr_pages, vma); } /** @@ -1703,6 +1705,8 @@ void folio_add_new_anon_rmap(struct folio *folio, struct vm_area_struct *vma, __folio_mod_stat(folio, nr, nr_pmdmapped); mod_mthp_stat(folio_order(folio), MTHP_STAT_NR_ANON, 1); + + crash_mark_pages(&folio->page, nr, vma); } static __always_inline void __folio_add_file_rmap(struct folio *folio, @@ -1721,6 +1725,8 @@ static __always_inline void __folio_add_file_rmap(struct folio *folio, */ if (folio_nr_pages(folio) == nr_pages) mlock_vma_folio(folio, vma); + + crash_mark_pages(page, nr_pages, vma); } /** @@ -3190,6 +3196,7 @@ void hugetlb_add_anon_rmap(struct folio *folio, struct vm_area_struct *vma, SetPageAnonExclusive(&folio->page); VM_WARN_ON_FOLIO(folio_entire_mapcount(folio) > 1 && PageAnonExclusive(&folio->page), folio); + crash_mark_pages(&folio->page, folio_nr_pages(folio), vma); } void hugetlb_add_new_anon_rmap(struct folio *folio, @@ -3204,5 +3211,6 @@ void hugetlb_add_new_anon_rmap(struct folio *folio, folio_clear_hugetlb_restore_reserve(folio); __folio_set_anon(folio, vma, address, true); SetPageAnonExclusive(&folio->page); + crash_mark_pages(&folio->page, folio_nr_pages(folio), vma); } #endif /* CONFIG_HUGETLB_PAGE */ diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index b242fa8b22c8..bb0a35d59daa 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -1326,6 +1326,7 @@ static long move_present_ptes(struct mm_struct *mm, folio_move_anon_rmap(src_folio, dst_vma); src_folio->index = linear_anon_page_index(dst_vma, dst_addr); + crash_mark_pages(&src_folio->page, 1, dst_vma); orig_dst_pte = folio_mk_pte(src_folio, dst_vma->vm_page_prot); /* Set soft dirty bit so userspace can notice the pte was moved */ -- 2.55.0
