Let's stop using the page lock in balloon code and instead use only the balloon_device_lock.
As soon as we set the PG_movable_ops flag, we might now get isolation callbacks for that page as we are no longer holding the page lock. In there, we'll simply synchronize using the balloon_device_lock. So in balloon_page_isolate() lookup the balloon_dev_info through page->private under balloon_device_lock. It's crucial that we update page->private under the balloon_device_lock, so the isolation callback can properly deal with concurrent deflation. Consequently, make sure that balloon_page_finalize() is called under balloon_device_lock as we remove a page from the list and clear page->private. balloon_page_insert() is already called with the balloon_device_lock held. Note that the core will still lock the pages, for example in isolate_movable_ops_page(). The lock is there still relevant for handling the PageMovableOpsIsolated flag, but that can be later changed to use an atomic test-and-set instead, or moved into the movable_ops backends. Signed-off-by: David Hildenbrand <[email protected]> --- include/linux/balloon_compaction.h | 27 +++++++++++---------- mm/balloon_compaction.c | 38 ++++++++++-------------------- 2 files changed, 26 insertions(+), 39 deletions(-) diff --git a/include/linux/balloon_compaction.h b/include/linux/balloon_compaction.h index e2d9eb40e1fbb..ad594af6ed100 100644 --- a/include/linux/balloon_compaction.h +++ b/include/linux/balloon_compaction.h @@ -12,24 +12,26 @@ * is derived from the page type (PageOffline()) combined with the * PG_movable_ops flag (PageMovableOps()). * + * Once the page type and the PG_movable_ops are set, migration code + * can initiate page isolation by invoking the + * movable_operations()->isolate_page() callback + * + * As long as page->private is set, the page is either on the balloon list + * or isolated for migration. If page->private is not set, the page is + * either still getting inflated, or was deflated to be freed by the balloon + * driver soon. Isolation is impossible in both cases. + * * As the page isolation scanning step a compaction thread does is a lockless * procedure (from a page standpoint), it might bring some racy situations while * performing balloon page compaction. In order to sort out these racy scenarios * and safely perform balloon's page compaction and migration we must, always, * ensure following these simple rules: * - * i. Setting the PG_movable_ops flag and page->private with the following - * lock order - * +-page_lock(page); - * +--spin_lock_irq(&balloon_pages_lock); + * i. Inflation/deflation must set/clear page->private under the + * balloon_pages_lock * * ii. isolation or dequeueing procedure must remove the page from balloon - * device page list under &balloon_pages_lock - * - * The functions provided by this interface are placed to help on coping with - * the aforementioned balloon page corner case, as well as to ensure the simple - * set of exposed rules are satisfied while we are dealing with balloon pages - * compaction / migration. + * device page list under balloon_pages_lock * * Copyright (C) 2012, Red Hat, Inc. Rafael Aquini <[email protected]> */ @@ -93,8 +95,7 @@ static inline struct balloon_dev_info *balloon_page_device(struct page *page) * @balloon : pointer to balloon device * @page : page to be assigned as a 'balloon page' * - * Caller must ensure the page is locked and the spin_lock protecting balloon - * pages list is held before inserting a page into the balloon device. + * Caller must ensure the balloon_pages_lock is held. */ static inline void balloon_page_insert(struct balloon_dev_info *balloon, struct page *page) @@ -119,7 +120,7 @@ static inline gfp_t balloon_mapping_gfp_mask(void) * balloon list for release to the page allocator * @page: page to be released to the page allocator * - * Caller must ensure that the page is locked. + * Caller must ensure the balloon_pages_lock is held. */ static inline void balloon_page_finalize(struct page *page) { diff --git a/mm/balloon_compaction.c b/mm/balloon_compaction.c index 97e838795354d..28ef0cb6b3bbc 100644 --- a/mm/balloon_compaction.c +++ b/mm/balloon_compaction.c @@ -20,15 +20,7 @@ static DEFINE_SPINLOCK(balloon_pages_lock); static void balloon_page_enqueue_one(struct balloon_dev_info *b_dev_info, struct page *page) { - /* - * Block others from accessing the 'page' when we get around to - * establishing additional references. We should be the only one - * holding a reference to the 'page' at this point. If we are not, then - * memory corruption is possible and we should stop execution. - */ - BUG_ON(!trylock_page(page)); balloon_page_insert(b_dev_info, page); - unlock_page(page); if (b_dev_info->adjust_managed_page_count) adjust_managed_page_count(page, -1); __count_vm_event(BALLOON_INFLATE); @@ -93,22 +85,12 @@ size_t balloon_page_list_dequeue(struct balloon_dev_info *b_dev_info, list_for_each_entry_safe(page, tmp, &b_dev_info->pages, lru) { if (n_pages == n_req_pages) break; - - /* - * Block others from accessing the 'page' while we get around to - * establishing additional references and preparing the 'page' - * to be released by the balloon driver. - */ - if (!trylock_page(page)) - continue; - list_del(&page->lru); if (b_dev_info->adjust_managed_page_count) adjust_managed_page_count(page, 1); balloon_page_finalize(page); __count_vm_event(BALLOON_DEFLATE); list_add(&page->lru, pages); - unlock_page(page); dec_node_page_state(page, NR_BALLOON_PAGES); n_pages++; } @@ -213,13 +195,19 @@ EXPORT_SYMBOL_GPL(balloon_page_dequeue); static bool balloon_page_isolate(struct page *page, isolate_mode_t mode) { - struct balloon_dev_info *b_dev_info = balloon_page_device(page); + struct balloon_dev_info *b_dev_info; unsigned long flags; - if (!b_dev_info) - return false; - spin_lock_irqsave(&balloon_pages_lock, flags); + b_dev_info = balloon_page_device(page); + if (!b_dev_info) { + /* + * The page already got deflated and removed from the + * balloon list. + */ + spin_unlock_irqrestore(&balloon_pages_lock, flags); + return false; + } list_del(&page->lru); b_dev_info->isolated_pages++; spin_unlock_irqrestore(&balloon_pages_lock, flags); @@ -249,9 +237,6 @@ static int balloon_page_migrate(struct page *newpage, struct page *page, unsigned long flags; int rc; - VM_BUG_ON_PAGE(!PageLocked(page), page); - VM_BUG_ON_PAGE(!PageLocked(newpage), newpage); - /* Isolated balloon pages cannot get deflated. */ if (WARN_ON_ONCE(!b_dev_info)) return -EAGAIN; @@ -291,10 +276,11 @@ static int balloon_page_migrate(struct page *newpage, struct page *page, } b_dev_info->isolated_pages--; - spin_unlock_irqrestore(&balloon_pages_lock, flags); /* Free the now-deflated page we isolated in balloon_page_isolate(). */ balloon_page_finalize(page); + spin_unlock_irqrestore(&balloon_pages_lock, flags); + put_page(page); return 0; -- 2.51.0
