[RFC PATCH 04/25] mm: migrate: Add copy_page_multithread into migrate_pages.

[Zi Yan]

From: Zi Yan <z...@nvidia.com>

An option is added to move_pages() syscall to use multi-threaded
page migration.

Signed-off-by: Zi Yan <z...@nvidia.com>
---
 include/linux/migrate_mode.h   |  1 +
 include/uapi/linux/mempolicy.h |  2 ++
 mm/migrate.c                   | 29 +++++++++++++++++++----------
 3 files changed, 22 insertions(+), 10 deletions(-)

diff --git a/include/linux/migrate_mode.h b/include/linux/migrate_mode.h
index da44940..5bc8a77 100644
--- a/include/linux/migrate_mode.h
+++ b/include/linux/migrate_mode.h
@@ -22,6 +22,7 @@ enum migrate_mode {
 
        MIGRATE_MODE_MASK = 3,
        MIGRATE_SINGLETHREAD    = 0,
+       MIGRATE_MT                              = 1<<4,
 };
 
 #endif         /* MIGRATE_MODE_H_INCLUDED */
diff --git a/include/uapi/linux/mempolicy.h b/include/uapi/linux/mempolicy.h
index 3354774..890269b 100644
--- a/include/uapi/linux/mempolicy.h
+++ b/include/uapi/linux/mempolicy.h
@@ -48,6 +48,8 @@ enum {
 #define MPOL_MF_LAZY    (1<<3) /* Modifies '_MOVE:  lazy migrate on fault */
 #define MPOL_MF_INTERNAL (1<<4)        /* Internal flags start here */
 
+#define MPOL_MF_MOVE_MT  (1<<6)        /* Use multi-threaded page copy routine 
*/
+
 #define MPOL_MF_VALID  (MPOL_MF_STRICT   |     \
                         MPOL_MF_MOVE     |     \
                         MPOL_MF_MOVE_ALL)
diff --git a/mm/migrate.c b/mm/migrate.c
index 2b2653e..dd6ccbe 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -572,6 +572,7 @@ static void copy_huge_page(struct page *dst, struct page 
*src,
 {
        int i;
        int nr_pages;
+       int rc = -EFAULT;
 
        if (PageHuge(src)) {
                /* hugetlbfs page */
@@ -588,10 +589,14 @@ static void copy_huge_page(struct page *dst, struct page 
*src,
                nr_pages = hpage_nr_pages(src);
        }
 
-       for (i = 0; i < nr_pages; i++) {
-               cond_resched();
-               copy_highpage(dst + i, src + i);
-       }
+       if (mode & MIGRATE_MT)
+               rc = copy_page_multithread(dst, src, nr_pages);
+
+       if (rc)
+               for (i = 0; i < nr_pages; i++) {
+                       cond_resched();
+                       copy_highpage(dst + i, src + i);
+               }
 }
 
 /*
@@ -1500,7 +1505,7 @@ static int store_status(int __user *status, int start, 
int value, int nr)
 }
 
 static int do_move_pages_to_node(struct mm_struct *mm,
-               struct list_head *pagelist, int node)
+               struct list_head *pagelist, int node, bool migrate_mt)
 {
        int err;
 
@@ -1508,7 +1513,8 @@ static int do_move_pages_to_node(struct mm_struct *mm,
                return 0;
 
        err = migrate_pages(pagelist, alloc_new_node_page, NULL, node,
-                       MIGRATE_SYNC, MR_SYSCALL);
+                       MIGRATE_SYNC | (migrate_mt ? MIGRATE_MT : 
MIGRATE_SINGLETHREAD),
+                       MR_SYSCALL);
        if (err)
                putback_movable_pages(pagelist);
        return err;
@@ -1629,7 +1635,8 @@ static int do_pages_move(struct mm_struct *mm, nodemask_t 
task_nodes,
                        current_node = node;
                        start = i;
                } else if (node != current_node) {
-                       err = do_move_pages_to_node(mm, &pagelist, 
current_node);
+                       err = do_move_pages_to_node(mm, &pagelist, current_node,
+                               flags & MPOL_MF_MOVE_MT);
                        if (err)
                                goto out;
                        err = store_status(status, start, current_node, i - 
start);
@@ -1652,7 +1659,8 @@ static int do_pages_move(struct mm_struct *mm, nodemask_t 
task_nodes,
                if (err)
                        goto out_flush;
 
-               err = do_move_pages_to_node(mm, &pagelist, current_node);
+               err = do_move_pages_to_node(mm, &pagelist, current_node,
+                               flags & MPOL_MF_MOVE_MT);
                if (err)
                        goto out;
                if (i > start) {
@@ -1667,7 +1675,8 @@ static int do_pages_move(struct mm_struct *mm, nodemask_t 
task_nodes,
                return err;
 
        /* Make sure we do not overwrite the existing error */
-       err1 = do_move_pages_to_node(mm, &pagelist, current_node);
+       err1 = do_move_pages_to_node(mm, &pagelist, current_node,
+                               flags & MPOL_MF_MOVE_MT);
        if (!err1)
                err1 = store_status(status, start, current_node, i - start);
        if (!err)
@@ -1763,7 +1772,7 @@ static int kernel_move_pages(pid_t pid, unsigned long 
nr_pages,
        nodemask_t task_nodes;
 
        /* Check flags */
-       if (flags & ~(MPOL_MF_MOVE|MPOL_MF_MOVE_ALL))
+       if (flags & ~(MPOL_MF_MOVE|MPOL_MF_MOVE_ALL|MPOL_MF_MOVE_MT))
                return -EINVAL;
 
        if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE))
-- 
2.7.4