The commit is pushed to "branch-rh9-5.14.vz9.1.x-ovz" and will appear at
https://src.openvz.org/scm/ovz/vzkernel.git
after rh9-5.14.0-4.vz9.10.12
------>
commit ab337b41675eec3fe7e97f1efd155677d77d7890
Author: Andrey Ryabinin <ryabinin....@gmail.com>
Date: Wed Oct 20 11:40:43 2021 +0300
mm: Add and use batched version of __tlb_remove_table()
tlb_remove_table_rcu() removes tables on by one using the
__tlb_remove_table() -> free_page_and_swap_cache(table). Use batched
free_pages_and_swap_cache_nodrain() instead to remove all tables in
one go. This helps to remove contention on the memcgroups counters
since we decrease them only once instead of decrementing one by one
for each page individually.
https://jira.sw.ru/browse/PSBM-101300
Signed-off-by: Andrey Ryabinin <aryabi...@virtuozzo.com>
(cherry-picked from vz7 commit 9a3ca2497cdb ("mm: Add and use batched
version of
__tlb_remove_table()"))
https://jira.sw.ru/browse/PSBM-127854
Signed-off-by: Valeriy Vdovin <valeriy.vdo...@virtuozzo.com>
(cherry picked from vz8 commit 006557b2c3f6a6197fc480fea3155fe2f6856187)
Signed-off-by: Andrey Zhadchenko <andrey.zhadche...@virtuozzo.com>
---
arch/x86/include/asm/tlb.h | 5 +++++
include/linux/swap.h | 1 +
mm/mmu_gather.c | 6 +-----
mm/swap_state.c | 16 ++++++++++++++--
4 files changed, 21 insertions(+), 7 deletions(-)
diff --git a/arch/x86/include/asm/tlb.h b/arch/x86/include/asm/tlb.h
index 1bfe979bb9bc..f2a21d87e0e8 100644
--- a/arch/x86/include/asm/tlb.h
+++ b/arch/x86/include/asm/tlb.h
@@ -37,4 +37,9 @@ static inline void __tlb_remove_table(void *table)
free_page_and_swap_cache(table);
}
+static inline void __tlb_remove_tables(void **tables, int nr)
+{
+ free_pages_and_swap_cache_nodrain((struct page **)tables, nr);
+}
+
#endif /* _ASM_X86_TLB_H */
diff --git a/include/linux/swap.h b/include/linux/swap.h
index 68c186361397..f712ad1f0510 100644
--- a/include/linux/swap.h
+++ b/include/linux/swap.h
@@ -454,6 +454,7 @@ extern void clear_shadow_from_swap_cache(int type, unsigned
long begin,
extern void free_swap_cache(struct page *);
extern void free_page_and_swap_cache(struct page *);
extern void free_pages_and_swap_cache(struct page **, int);
+extern void free_pages_and_swap_cache_nodrain(struct page **, int);
extern struct page *lookup_swap_cache(swp_entry_t entry,
struct vm_area_struct *vma,
unsigned long addr);
diff --git a/mm/mmu_gather.c b/mm/mmu_gather.c
index 1b9837419bf9..2faa0d59aeca 100644
--- a/mm/mmu_gather.c
+++ b/mm/mmu_gather.c
@@ -95,11 +95,7 @@ bool __tlb_remove_page_size(struct mmu_gather *tlb, struct
page *page, int page_
static void __tlb_remove_table_free(struct mmu_table_batch *batch)
{
- int i;
-
- for (i = 0; i < batch->nr; i++)
- __tlb_remove_table(batch->tables[i]);
-
+ __tlb_remove_tables(batch->tables, batch->nr);
free_page((unsigned long)batch);
}
diff --git a/mm/swap_state.c b/mm/swap_state.c
index 7e3104d27d4f..52df32e65624 100644
--- a/mm/swap_state.c
+++ b/mm/swap_state.c
@@ -305,17 +305,29 @@ void free_page_and_swap_cache(struct page *page)
* Passed an array of pages, drop them all from swapcache and then release
* them. They are removed from the LRU and freed if this is their last use.
*/
-void free_pages_and_swap_cache(struct page **pages, int nr)
+void __free_pages_and_swap_cache(struct page **pages, int nr, bool drain)
{
struct page **pagep = pages;
int i;
- lru_add_drain();
+ if (drain)
+ lru_add_drain();
+
for (i = 0; i < nr; i++)
free_swap_cache(pagep[i]);
release_pages(pagep, nr);
}
+void free_pages_and_swap_cache(struct page **pages, int nr)
+{
+ __free_pages_and_swap_cache(pages, nr, true);
+}
+
+void free_pages_and_swap_cache_nodrain(struct page **pages, int nr)
+{
+ __free_pages_and_swap_cache(pages, nr, false);
+}
+
static inline bool swap_use_vma_readahead(void)
{
return READ_ONCE(enable_vma_readahead) && !atomic_read(&nr_rotate_swap);
_______________________________________________
Devel mailing list
Devel@openvz.org
https://lists.openvz.org/mailman/listinfo/devel
