The PTE loop mixes its PTL-protected scan with the surrounding lock drops for rescheduling and large-folio splitting.
Move the scan into a helper that runs entirely under the PTE lock. It returns a locked and referenced split candidate, or stops at the existing scheduling boundary so the caller can drop the lock before yielding. No functional change intended. Assisted-by: LLM Signed-off-by: Gregory Price (Meta) <[email protected]> --- mm/madvise.c | 66 +++++++++++++++++++++++++++++++--------------------- 1 file changed, 39 insertions(+), 27 deletions(-) diff --git a/mm/madvise.c b/mm/madvise.c index 350b854ccc197..e3c3acfcd9b66 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -531,6 +531,28 @@ static bool madvise_lru_huge_pmd(pmd_t *pmd, unsigned long addr, } #endif +static struct folio * +madvise_lru_pte_range_locked(pte_t *pte, unsigned long *addr, + unsigned long end, struct mm_walk *walk, + struct list_head *folio_list, bool pageout_anon_only, int *nr, + unsigned int *batch_count) +{ + struct folio *folio; + + for (; *addr < end; pte += *nr, *addr += *nr * PAGE_SIZE) { + if (++(*batch_count) == SWAP_CLUSTER_MAX) { + *batch_count = 0; + if (need_resched()) + return NULL; + } + folio = madvise_lru_pte_batch_locked(pte, *addr, end, walk, + folio_list, pageout_anon_only, nr); + if (folio) + return folio; + } + return NULL; +} + static int madvise_lru_pmd_entry(pmd_t *pmd, unsigned long addr, unsigned long end, struct mm_walk *walk) { @@ -562,36 +584,26 @@ static int madvise_lru_pmd_entry(pmd_t *pmd, unsigned long addr, goto out; flush_tlb_batched_pending(mm); lazy_mmu_mode_enable(); - for (; addr < end; pte += nr, addr += nr * PAGE_SIZE) { - if (++batch_count == SWAP_CLUSTER_MAX) { - batch_count = 0; - if (need_resched()) { - lazy_mmu_mode_disable(); - pte_unmap_unlock(start_pte, ptl); - cond_resched(); - goto restart; - } - } - - folio = madvise_lru_pte_batch_locked(pte, addr, end, walk, - &folio_list, pageout_anon_only, &nr); - if (!folio) - continue; - + folio = madvise_lru_pte_range_locked(pte, &addr, end, walk, + &folio_list, pageout_anon_only, &nr, &batch_count); + if (!folio && addr < end) { lazy_mmu_mode_disable(); pte_unmap_unlock(start_pte, ptl); - start_pte = NULL; - if (!split_folio(folio)) - nr = 0; - folio_unlock(folio); - folio_put(folio); - start_pte = pte_offset_map_lock(mm, pmd, addr, &ptl); - if (!start_pte) - break; - pte = start_pte; - flush_tlb_batched_pending(mm); - lazy_mmu_mode_enable(); + cond_resched(); + goto restart; } + if (!folio) + goto out; + + lazy_mmu_mode_disable(); + pte_unmap_unlock(start_pte, ptl); + start_pte = NULL; + if (!split_folio(folio)) + nr = 0; + folio_unlock(folio); + folio_put(folio); + addr += nr * PAGE_SIZE; + goto restart; out: if (start_pte) { -- 2.53.0-Meta
