Add and export interfaces to enable modules to build a mempolicy with (MPOL_BIND | MPOL_F_PRIVATE) pinned to a private NUMA node, so the drivers can stamp it onto VMAs they control (via vma->vm_policy).
mpol_bind_node() - acts the same as a userland mbind() mpol_private_bind() - bypasses the CAP_USER_NUMA check. Export it to modules (private:kmem, bind:kvm). Widen __mpol_put()'s module export to kmem so the driver can release the policy. Adjust mpol_set_nodemask to check for a pre-set MPOL_F_PRIVATE flag to allow the mpol_private_bind() check to bypass the N_MEMORY filter. Signed-off-by: Gregory Price <[email protected]> --- include/linux/mempolicy.h | 14 ++++++ mm/mempolicy.c | 102 +++++++++++++++++++++++++++++++++++++- 2 files changed, 114 insertions(+), 2 deletions(-) diff --git a/include/linux/mempolicy.h b/include/linux/mempolicy.h index 65c732d440d2f..715951a5b03c1 100644 --- a/include/linux/mempolicy.h +++ b/include/linux/mempolicy.h @@ -7,6 +7,7 @@ #define _LINUX_MEMPOLICY_H 1 #include <linux/sched.h> +#include <linux/err.h> #include <linux/mmzone.h> #include <linux/slab.h> #include <linux/rbtree.h> @@ -128,6 +129,9 @@ void mpol_free_shared_policy(struct shared_policy *sp); struct mempolicy *mpol_shared_policy_lookup(struct shared_policy *sp, pgoff_t idx); +struct mempolicy *mpol_private_bind(int nid); +struct mempolicy *mpol_bind_node(int nid); + struct mempolicy *get_task_policy(struct task_struct *p); struct mempolicy *__get_vma_policy(struct vm_area_struct *vma, unsigned long addr, pgoff_t *ilx); @@ -226,6 +230,16 @@ mpol_shared_policy_lookup(struct shared_policy *sp, pgoff_t idx) return NULL; } +static inline struct mempolicy *mpol_private_bind(int nid) +{ + return ERR_PTR(-EOPNOTSUPP); +} + +static inline struct mempolicy *mpol_bind_node(int nid) +{ + return ERR_PTR(-EOPNOTSUPP); +} + static inline struct mempolicy *get_vma_policy(struct vm_area_struct *vma, unsigned long addr, int order, pgoff_t *ilx) { diff --git a/mm/mempolicy.c b/mm/mempolicy.c index e83c2c7a94c1d..a3ffb09897489 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -406,7 +406,7 @@ static int mpol_new_preferred(struct mempolicy *pol, const nodemask_t *nodes) static int mpol_set_nodemask(struct mempolicy *pol, const nodemask_t *nodes, struct nodemask_scratch *nsc) { - int ret; + int ret, nid; /* * Default (pol==NULL) resp. local memory policies are not a @@ -432,6 +432,18 @@ static int mpol_set_nodemask(struct mempolicy *pol, else pol->w.cpuset_mems_allowed = cpuset_current_mems_allowed; + /* + * Private nodes are not in cpuset.mems, so they're always stripped. + * Driver-allocated policies will already have MPOL_F_PRIVATE set, + * if that's the case, add back in the requested set of private nodes. + */ + for_each_node_mask(nid, *nodes) { + if (!node_is_private(nid)) + continue; + if (pol->flags & MPOL_F_PRIVATE) + node_set(nid, nsc->mask2); + } + /* If any private nodes left in the nodemask - add the private flag */ if (nodes_intersects(nsc->mask2, node_states[N_MEMORY_PRIVATE])) pol->flags |= MPOL_F_PRIVATE; @@ -501,7 +513,7 @@ void __mpol_put(struct mempolicy *pol) */ kfree_rcu(pol, rcu); } -EXPORT_SYMBOL_FOR_MODULES(__mpol_put, "kvm"); +EXPORT_SYMBOL_FOR_MODULES(__mpol_put, "kvm,kmem"); static void mpol_rebind_default(struct mempolicy *pol, const nodemask_t *nodes) { @@ -1126,6 +1138,92 @@ static long do_set_mempolicy(unsigned short mode, unsigned short flags, return ret; } +/* + * Build a refcounted MPOL_BIND policy targeting the single node @nid, + * contextualised to the caller's cpuset like a userspace mbind(). + * + * @flags is MPOL_F_PRIVATE for the an explicit in-kernel user, letting + * a private node be bound without checking CAP_USER_NUMA. Otherwise, + * CAP_USER_NUMA is enforced. + * + * The caller owns the reference and frees it with mpol_put(). + */ +static struct mempolicy *__mpol_bind_node(int nid, unsigned short flags) +{ + struct mempolicy *pol; + nodemask_t nodes; + int err; + + NODEMASK_SCRATCH(scratch); + + if (!scratch) + return ERR_PTR(-ENOMEM); + + nodes_clear(nodes); + node_set(nid, nodes); + + pol = mpol_new(MPOL_BIND, flags, &nodes); + if (IS_ERR(pol)) { + NODEMASK_SCRATCH_FREE(scratch); + return pol; + } + + err = mpol_set_nodemask(pol, &nodes, scratch); + NODEMASK_SCRATCH_FREE(scratch); + if (err) { + mpol_put(pol); + return ERR_PTR(err); + } + return pol; +} + +/** + * mpol_private_bind - build an MPOL_BIND policy pinned to a private node + * @nid: an N_MEMORY_PRIVATE node + * + * Returns a refcounted mempolicy that binds allocations to @nid with the + * private-placement intent (MPOL_F_PRIVATE). This binds to @nid regardless + * of the node's CAP_USER_NUMA, providing a privileged way for node-owners + * to bind driver/service owned VMAs to the node. + * + * Like any MPOL_BIND it is relaxable: an unsatisfiable request falls back + * rather than failing. + * + * Must be called while @nid is N_MEMORY_PRIVATE. + * + * The caller owns the reference and frees it with mpol_put(). + * + * Return: the policy, or an ERR_PTR on failure. + */ +struct mempolicy *mpol_private_bind(int nid) +{ + if (!node_is_private(nid)) + return ERR_PTR(-EINVAL); + return __mpol_bind_node(nid, MPOL_F_PRIVATE); +} +EXPORT_SYMBOL_FOR_MODULES(mpol_private_bind, "kmem"); + +/** + * mpol_bind_node - build an MPOL_BIND policy targeting @nid for in-kernel use + * @nid: the node to bind to + * + * Returns a refcounted MPOL_BIND policy that places allocations on @nid, + * contextualised to the caller's cpuset exactly like a userspace mbind(). + * + * This interface should be used by services implenting mempolicy support with + * user-provided node bindings. N_MEMORY_PRIVATE node bindings are honored if + * the node has CAP_USER_NUMA, otherwise return -EINVAL. + * + * The caller owns the reference and frees it with mpol_put(). + * + * Return: the policy, or an ERR_PTR on failure. + */ +struct mempolicy *mpol_bind_node(int nid) +{ + return __mpol_bind_node(nid, 0); +} +EXPORT_SYMBOL_FOR_MODULES(mpol_bind_node, "kvm"); + /* * Return nodemask for policy for get_mempolicy() query * -- 2.53.0-Meta
