This patch introduces the freeze on gmem_inode which prevents the fallocate call and any new page fault allocation. This will avoid gmem file modification when it is being preserved
Used srcu lock to synchronise the freeze call, where write blocks until all the reads are free. And reads are re-entrant. Incase fault fails, It return -EPERM and VM_EXIT to userspace. userspace must handle this properly as every new fault will fail. Signed-off-by: Tarun Sahu <[email protected]> --- virt/kvm/guest_memfd.c | 117 +++++++++++++++++++++++++++++++++++++---- virt/kvm/guest_memfd.h | 5 ++ 2 files changed, 111 insertions(+), 11 deletions(-) diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 6740ae2bf948..b94639cdf312 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -7,11 +7,13 @@ #include <linux/mempolicy.h> #include <linux/pseudo_fs.h> #include <linux/pagemap.h> +#include <linux/srcu.h> #include "guest_memfd.h" #include "kvm_mm.h" static struct vfsmount *kvm_gmem_mnt; +static struct srcu_struct kvm_gmem_freeze_srcu; #define kvm_gmem_for_each_file(f, inode) \ @@ -96,6 +98,7 @@ static struct folio *kvm_gmem_get_folio(struct inode *inode, pgoff_t index) /* TODO: Support huge pages. */ struct mempolicy *policy; struct folio *folio; + int idx; /* * Fast-path: See if folio is already present in mapping to avoid @@ -105,12 +108,20 @@ static struct folio *kvm_gmem_get_folio(struct inode *inode, pgoff_t index) if (!IS_ERR(folio)) return folio; + idx = srcu_read_lock(&kvm_gmem_freeze_srcu); + if (kvm_gmem_is_frozen(inode)) { + srcu_read_unlock(&kvm_gmem_freeze_srcu, idx); + return ERR_PTR(-EPERM); + } + policy = mpol_shared_policy_lookup(&GMEM_I(inode)->policy, index); folio = __filemap_get_folio_mpol(inode->i_mapping, index, FGP_LOCK | FGP_CREAT, mapping_gfp_mask(inode->i_mapping), policy); mpol_cond_put(policy); + srcu_read_unlock(&kvm_gmem_freeze_srcu, idx); + /* * External interfaces like kvm_gmem_get_pfn() support dealing * with hugepages to a degree, but internally, guest_memfd currently @@ -273,16 +284,30 @@ static long kvm_gmem_allocate(struct inode *inode, loff_t offset, loff_t len) static long kvm_gmem_fallocate(struct file *file, int mode, loff_t offset, loff_t len) { + struct inode *inode = file_inode(file); int ret; + int idx; - if (!(mode & FALLOC_FL_KEEP_SIZE)) - return -EOPNOTSUPP; + idx = srcu_read_lock(&kvm_gmem_freeze_srcu); + if (kvm_gmem_is_frozen(inode)) { + srcu_read_unlock(&kvm_gmem_freeze_srcu, idx); + return -EPERM; + } - if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE)) - return -EOPNOTSUPP; + if (!(mode & FALLOC_FL_KEEP_SIZE)) { + ret = -EOPNOTSUPP; + goto out; + } - if (!PAGE_ALIGNED(offset) || !PAGE_ALIGNED(len)) - return -EINVAL; + if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE)) { + ret = -EOPNOTSUPP; + goto out; + } + + if (!PAGE_ALIGNED(offset) || !PAGE_ALIGNED(len)) { + ret = -EINVAL; + goto out; + } if (mode & FALLOC_FL_PUNCH_HOLE) ret = kvm_gmem_punch_hole(file_inode(file), offset, len); @@ -291,6 +316,9 @@ static long kvm_gmem_fallocate(struct file *file, int mode, loff_t offset, if (!ret) file_modified(file); + +out: + srcu_read_unlock(&kvm_gmem_freeze_srcu, idx); return ret; } @@ -944,7 +972,9 @@ static void kvm_gmem_destroy_inode(struct inode *inode) static void kvm_gmem_free_inode(struct inode *inode) { - kmem_cache_free(kvm_gmem_inode_cachep, GMEM_I(inode)); + struct gmem_inode *gi = GMEM_I(inode); + + kmem_cache_free(kvm_gmem_inode_cachep, gi); } static const struct super_operations kvm_gmem_super_operations = { @@ -1001,12 +1031,21 @@ int kvm_gmem_init(struct module *module) if (!kvm_gmem_inode_cachep) return -ENOMEM; + ret = init_srcu_struct(&kvm_gmem_freeze_srcu); + if (ret) + goto err_cache; + ret = kvm_gmem_init_mount(); - if (ret) { - kmem_cache_destroy(kvm_gmem_inode_cachep); - return ret; - } + if (ret) + goto err_srcu; + return 0; + +err_srcu: + cleanup_srcu_struct(&kvm_gmem_freeze_srcu); +err_cache: + kmem_cache_destroy(kvm_gmem_inode_cachep); + return ret; } void kvm_gmem_exit(void) @@ -1014,5 +1053,61 @@ void kvm_gmem_exit(void) kern_unmount(kvm_gmem_mnt); kvm_gmem_mnt = NULL; rcu_barrier(); + cleanup_srcu_struct(&kvm_gmem_freeze_srcu); kmem_cache_destroy(kvm_gmem_inode_cachep); } + +/** + * kvm_gmem_freeze - Freeze or unfreeze a guest_memfd inode mapping. + * @inode: The guest_memfd inode. + * @freeze: True to freeze, false to unfreeze. + * + * This API is used strictly during the live update / preservation transition + * window to prevent host userspace and guest-side faults from making any + * mapping modifications (such as fallocate or page fault allocation) + * to the guest_memfd page cache. + * + * Synchronization Strategy (Sleepable RCU): + * To avoid high-contention VFS locks (like inode_lock or + * filemap_invalidate_lock) on the vCPU page fault hot paths, this subsystem + * implements a lightweight, system-wide Sleepable RCU (SRCU) mechanism + * (`kvm_gmem_freeze_srcu`): + * + * Global vs. Per-Inode SRCU + * ====================== + * A single system-wide global static `srcu_struct` is used instead of a + * per-inode SRCU structure to completely prevent unprivileged users from + * exhausting the host's per-CPU memory allocator. Because + * `init_srcu_struct()` allocates per-CPU memory via `alloc_percpu()`, which + * is not accounted by memory cgroups (memcg), + * a per-inode SRCU structure would allow a tenant to bypass cgroup limits and + * trigger a system-wide Out-of-Memory (OOM) crash simply by spawning a large + * number of guest_memfd file descriptors (bounded only by RLIMIT_NOFILE). + * + * Flag Modification Note: + * Since `GUEST_MEMFD_F_MAPPING_FROZEN` is the ONLY flag in + * `GMEM_I(inode)->flags` that is mutated dynamically at runtime (all other + * flags are creation-time flags which remain strictly read-only), there is + * no possibility of concurrent bit-modification races. Therefore, a standard + * `WRITE_ONCE` is fully safe and does not require complex `cmpxchg` + * synchronization loops. + */ +void kvm_gmem_freeze(struct inode *inode, bool freeze) +{ + u64 flags = READ_ONCE(GMEM_I(inode)->flags); + + if (freeze) + flags |= GUEST_MEMFD_F_MAPPING_FROZEN; + else + flags &= ~GUEST_MEMFD_F_MAPPING_FROZEN; + + WRITE_ONCE(GMEM_I(inode)->flags, flags); + + if (freeze) + synchronize_srcu(&kvm_gmem_freeze_srcu); +} + +bool kvm_gmem_is_frozen(struct inode *inode) +{ + return READ_ONCE(GMEM_I(inode)->flags) & GUEST_MEMFD_F_MAPPING_FROZEN; +} diff --git a/virt/kvm/guest_memfd.h b/virt/kvm/guest_memfd.h index c528b046dd69..028c348a1023 100644 --- a/virt/kvm/guest_memfd.h +++ b/virt/kvm/guest_memfd.h @@ -29,11 +29,16 @@ struct gmem_inode { u64 flags; }; +/* Internal kernel-only flags (must not overlap with UAPI flags) */ +#define GUEST_MEMFD_F_MAPPING_FROZEN (1ULL << 63) + static inline struct gmem_inode *GMEM_I(struct inode *inode) { return container_of(inode, struct gmem_inode, vfs_inode); } struct file *__kvm_gmem_create_file(struct kvm *kvm, loff_t size, u64 flags); +void kvm_gmem_freeze(struct inode *inode, bool freeze); +bool kvm_gmem_is_frozen(struct inode *inode); #endif /* __KVM_GUEST_MEMFD_H__ */ -- 2.54.0.1032.g2f8565e1d1-goog
