Signed-off-by: Pavel Begunkov <[email protected]> --- drivers/nvme/host/Makefile | 1 + drivers/nvme/host/pci-dmabuf.c | 164 ++++++++++++++++++++++ drivers/nvme/host/pci-dmabuf.h | 73 ++++++++++ drivers/nvme/host/pci.c | 246 +++------------------------------ 4 files changed, 255 insertions(+), 229 deletions(-) create mode 100644 drivers/nvme/host/pci-dmabuf.c create mode 100644 drivers/nvme/host/pci-dmabuf.h
diff --git a/drivers/nvme/host/Makefile b/drivers/nvme/host/Makefile index 67563a69f7dc..8288b986d713 100644 --- a/drivers/nvme/host/Makefile +++ b/drivers/nvme/host/Makefile @@ -21,6 +21,7 @@ nvme-core-$(CONFIG_NVME_HWMON) += hwmon.o nvme-core-$(CONFIG_NVME_HOST_AUTH) += auth.o nvme-y += pci.o +nvme-$(CONFIG_DMA_SHARED_BUFFER) += pci-dmabuf.o nvme-fabrics-y += fabrics.o diff --git a/drivers/nvme/host/pci-dmabuf.c b/drivers/nvme/host/pci-dmabuf.c new file mode 100644 index 000000000000..d58a38a75d67 --- /dev/null +++ b/drivers/nvme/host/pci-dmabuf.c @@ -0,0 +1,164 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2026 Pavel Begunkov <[email protected]> + */ + +#include <linux/dma-resv.h> + +#include "pci-dmabuf.h" + +static void nvme_dmabuf_invalidate_mappings(struct dma_buf_attachment *attach) +{ + struct dma_buf_io_ctx *ctx = attach->importer_priv; + + dma_buf_io_invalidate_mappings(ctx); +} + +static const struct dma_buf_attach_ops nvme_dmabuf_importer_ops = { + .invalidate_mappings = nvme_dmabuf_invalidate_mappings, + .allow_peer2peer = true, +}; + +static struct dma_buf_io_map *nvme_dma_buf_io_map(struct dma_buf_io_ctx *ctx) +{ + unsigned nr_entries = ctx->dmabuf->size / NVME_CTRL_PAGE_SIZE; + struct nvme_dma_buf_io_ctx *nvme_ctx = ctx->dev_priv; + struct dma_buf_attachment *attach = nvme_ctx->attach; + unsigned long tmp, i = 0; + struct nvme_dmabuf_map *map; + struct scatterlist *sg; + struct sg_table *sgt; + int ret; + + dma_resv_assert_held(ctx->dmabuf->resv); + + if (ctx->dmabuf->size % NVME_CTRL_PAGE_SIZE) + return ERR_PTR(-EINVAL); + + map = kvmalloc_flex(*map, dma_list, nr_entries); + if (!map) + return ERR_PTR(-ENOMEM); + map->nr_entries = nr_entries; + + sgt = dma_buf_map_attachment(attach, ctx->dir); + if (IS_ERR(sgt)) { + ret = PTR_ERR(sgt); + sgt = NULL; + goto err; + } + + for_each_sgtable_dma_sg(sgt, sg, tmp) { + dma_addr_t dma_addr = sg_dma_address(sg); + unsigned long sg_len = sg_dma_len(sg); + + if ((sg_len % NVME_CTRL_PAGE_SIZE) || + (dma_addr % NVME_CTRL_PAGE_SIZE)) { + ret = -EINVAL; + goto err; + } + while (sg_len) { + map->dma_list[i++] = dma_addr; + dma_addr += NVME_CTRL_PAGE_SIZE; + sg_len -= NVME_CTRL_PAGE_SIZE; + } + } + + ret = dma_buf_io_init_map(ctx, &map->base, sgt); + if (ret) + goto err; + map->sgt = sgt; + return &map->base; +err: + if (sgt) + dma_buf_unmap_attachment(attach, sgt, ctx->dir); + kvfree(map); + return ERR_PTR(ret); +} + +static void nvme_dma_buf_io_unmap(struct dma_buf_io_ctx *ctx, + struct dma_buf_io_map *map_base) +{ + struct nvme_dma_buf_io_ctx *nvme_ctx = ctx->dev_priv; + struct nvme_dmabuf_map *map = to_nvme_dmabuf_map(map_base); + + dma_resv_assert_held(ctx->dmabuf->resv); + + dma_buf_unmap_attachment(nvme_ctx->attach, map->sgt, ctx->dir); + kvfree(map); +} + +static void nvme_dma_buf_io_release(struct dma_buf_io_ctx *base_ctx) +{ + struct nvme_dma_buf_io_ctx *ctx = base_ctx->dev_priv; + struct nvme_dma_buf_list *buf_list = ctx->buf_list; + + mutex_lock(&buf_list->lock); + if (!list_empty(&ctx->list)) { + dma_buf_detach(ctx->base->dmabuf, ctx->attach); + list_del_init(&ctx->list); + } + mutex_unlock(&buf_list->lock); + + nvme_put_ctrl(ctx->ctrl); + kfree(ctx); +} + +void __nvme_pci_remove_dmabuf(struct nvme_dma_buf_list *list) +{ + struct nvme_dma_buf_io_ctx *ctx, *tmp; + + mutex_lock(&list->lock); + list_for_each_entry_safe(ctx, tmp, &list->ctxs, list) { + dma_buf_io_detach(ctx->base); + dma_buf_detach(ctx->base->dmabuf, ctx->attach); + list_del_init(&ctx->list); + } + mutex_unlock(&list->lock); +} + +const struct dma_buf_io_ops nvme_dma_buf_io_ops = { + .map = nvme_dma_buf_io_map, + .unmap = nvme_dma_buf_io_unmap, + .release = nvme_dma_buf_io_release, +}; + +static int nvme_init_io_ctx_locked(struct nvme_ctrl *ctrl, + struct nvme_dma_buf_list *buf_list, + struct dma_buf_io_ctx *ctx) +{ + struct nvme_dma_buf_io_ctx *nvme_ctx; + struct dma_buf_attachment *attach; + + nvme_ctx = kzalloc_obj(*nvme_ctx); + if (!nvme_ctx) + return -ENOMEM; + + attach = dma_buf_dynamic_attach(ctx->dmabuf, ctrl->dev, + &nvme_dmabuf_importer_ops, ctx); + if (IS_ERR(attach)) { + kfree(nvme_ctx); + return PTR_ERR(attach); + } + + nvme_get_ctrl(ctrl); + list_add(&nvme_ctx->list, &buf_list->ctxs); + nvme_ctx->attach = attach; + nvme_ctx->base = ctx; + nvme_ctx->ctrl = ctrl; + nvme_ctx->buf_list = buf_list; + ctx->dev_priv = nvme_ctx; + ctx->dev_ops = &nvme_dma_buf_io_ops; + return 0; +} + +int __nvme_init_dma_buf_io_ctx(struct nvme_ctrl *ctrl, + struct nvme_dma_buf_list *buf_list, + struct dma_buf_io_ctx *ctx) +{ + int ret; + + mutex_lock(&buf_list->lock); + ret = nvme_init_io_ctx_locked(ctrl, buf_list, ctx); + mutex_unlock(&buf_list->lock); + return ret; +} diff --git a/drivers/nvme/host/pci-dmabuf.h b/drivers/nvme/host/pci-dmabuf.h new file mode 100644 index 000000000000..8e91f7f4ece5 --- /dev/null +++ b/drivers/nvme/host/pci-dmabuf.h @@ -0,0 +1,73 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2026 Pavel Begunkov <[email protected]> + */ + +#ifndef _NVME_PCI_DMABUF_H +#define _NVME_PCI_DMABUF_H + +#include <linux/dma-buf-io.h> + +#include "nvme.h" + +struct nvme_dma_buf_list { + struct list_head ctxs; + struct mutex lock; +}; + +struct nvme_dma_buf_io_ctx { + struct dma_buf_attachment *attach; + struct nvme_dma_buf_list *buf_list; + struct dma_buf_io_ctx *base; + struct nvme_ctrl *ctrl; + struct list_head list; +}; + +struct nvme_dmabuf_map { + struct dma_buf_io_map base; + struct sg_table *sgt; + unsigned nr_entries; + dma_addr_t dma_list[] __counted_by(nr_entries); +}; + +static inline struct nvme_dmabuf_map *to_nvme_dmabuf_map( + struct dma_buf_io_map *map) +{ + return container_of(map, struct nvme_dmabuf_map, base); +} + +static inline struct nvme_dmabuf_map *nvme_req_to_map(struct request *req) +{ + struct dma_buf_io_map *map = req->bio->bi_dmabuf_map; + + return to_nvme_dmabuf_map(map); +} + +void __nvme_pci_remove_dmabuf(struct nvme_dma_buf_list *list); +int __nvme_init_dma_buf_io_ctx(struct nvme_ctrl *ctrl, + struct nvme_dma_buf_list *buf_list, + struct dma_buf_io_ctx *ctx); + +static inline void nvme_pci_remove_dmabuf(struct nvme_dma_buf_list *list) +{ + if (IS_ENABLED(CONFIG_DMA_SHARED_BUFFER)) + __nvme_pci_remove_dmabuf(list); +} + +static inline int nvme_init_dma_buf_io_ctx(struct nvme_ctrl *ctrl, + struct nvme_dma_buf_list *buf_list, + struct dma_buf_io_ctx *ctx) +{ + if (!IS_ENABLED(CONFIG_DMA_SHARED_BUFFER)) + return -EOPNOTSUPP; + return __nvme_init_dma_buf_io_ctx(ctrl, buf_list, ctx); +} + +static inline void nvme_init_buf_list(struct nvme_dma_buf_list *buf_list) +{ + INIT_LIST_HEAD(&buf_list->ctxs); + mutex_init(&buf_list->lock); +} + + +#endif /* _NVME_PCI_DMABUF_H */ \ No newline at end of file diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index 68e0d4dcade4..a0506c93fa16 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -27,11 +27,10 @@ #include <linux/io-64-nonatomic-lo-hi.h> #include <linux/io-64-nonatomic-hi-lo.h> #include <linux/sed-opal.h> -#include <linux/dma-buf-io.h> -#include <linux/dma-resv.h> #include "trace.h" #include "nvme.h" +#include "pci-dmabuf.h" #define SQ_SIZE(q) ((q)->q_depth << (q)->sqes) #define CQ_SIZE(q) ((q)->q_depth * sizeof(struct nvme_completion)) @@ -320,8 +319,7 @@ struct nvme_dev { bool hmb; struct sg_table *hmb_sgt; mempool_t *dmavec_mempool; - struct list_head dmabuf_ctxs; - struct mutex dmabuf_lock; + struct nvme_dma_buf_list dmabufs; /* shadow doorbell buffer support: */ __le32 *dbbuf_dbs; @@ -401,13 +399,6 @@ struct nvme_queue { struct completion delete_done; }; -struct nvme_dmabuf_map { - struct dma_buf_io_map base; - struct sg_table *sgt; - unsigned nr_entries; - dma_addr_t dma_list[] __counted_by(nr_entries); -}; - /* bits for iod->flags */ enum nvme_iod_flags { /* this command has been aborted by the timeout handler */ @@ -876,48 +867,12 @@ static void nvme_free_descriptors(struct request *req) } } -static inline struct nvme_dmabuf_map * -to_nvme_dmabuf_map(struct dma_buf_io_map *map) -{ - return container_of(map, struct nvme_dmabuf_map, base); -} - -static void nvme_dmabuf_map_sync_for_cpu(struct nvme_dev *nvme_dev, - struct request *req) -{ - struct bio *bio = req->bio; - struct nvme_dmabuf_map *map = to_nvme_dmabuf_map(bio->bi_dmabuf_map); - - dma_sync_sgtable_for_cpu(nvme_dev->dev, map->sgt, rq_dma_dir(req)); -} - -static void nvme_dmabuf_map_sync_for_device(struct nvme_dev *nvme_dev, - struct request *req) -{ - struct bio *bio = req->bio; - struct nvme_dmabuf_map *map = to_nvme_dmabuf_map(bio->bi_dmabuf_map); - - dma_sync_sgtable_for_device(nvme_dev->dev, map->sgt, rq_dma_dir(req)); -} - -static void nvme_rq_clean_dmabuf_map(struct nvme_dev *dev, - struct request *req) -{ - struct nvme_iod *iod = blk_mq_rq_to_pdu(req); - - nvme_dmabuf_map_sync_for_cpu(dev, req); - - if (iod->nr_descriptors) - nvme_free_descriptors(req); -} - static blk_status_t nvme_rq_setup_dmabuf_map(struct request *req, struct nvme_queue *nvmeq) { struct nvme_iod *iod = blk_mq_rq_to_pdu(req); - struct bio *bio = req->bio; - struct nvme_dmabuf_map *map = to_nvme_dmabuf_map(bio->bi_dmabuf_map); - unsigned bvec_done = bio->bi_iter.bi_offset; + struct nvme_dmabuf_map *map = nvme_req_to_map(req); + unsigned bvec_done = req->bio->bi_iter.bi_offset; unsigned map_idx = bvec_done / NVME_CTRL_PAGE_SIZE; unsigned offset = bvec_done & (NVME_CTRL_PAGE_SIZE - 1); int length = blk_rq_payload_bytes(req) - (NVME_CTRL_PAGE_SIZE - offset); @@ -928,8 +883,6 @@ static blk_status_t nvme_rq_setup_dmabuf_map(struct request *req, __le64 *prp_list; unsigned i; - nvme_dmabuf_map_sync_for_device(nvmeq->dev, req); - if (length <= 0) { prp2_dma = 0; goto done; @@ -1062,10 +1015,8 @@ static void nvme_unmap_data(struct request *req) struct device *dma_dev = nvmeq->dev->dev; unsigned int attrs = 0; - if (blk_mq_rq_is_dmabuf(req)) { - nvme_rq_clean_dmabuf_map(nvmeq->dev, req); - return; - } + if (blk_mq_rq_is_dmabuf(req)) + goto out_free_desc; if (iod->flags & IOD_SINGLE_SEGMENT) { static_assert(offsetof(union nvme_data_ptr, prp1) == @@ -1091,6 +1042,7 @@ static void nvme_unmap_data(struct request *req) nvme_free_prps(req, attrs); } +out_free_desc: if (iod->nr_descriptors) nvme_free_descriptors(req); } @@ -2411,175 +2363,6 @@ static int nvme_create_queue(struct nvme_queue *nvmeq, int qid, bool polled) return result; } -#ifdef CONFIG_DMA_SHARED_BUFFER - -struct nvme_dma_buf_io_ctx { - struct dma_buf_attachment *attach; - struct dma_buf_io_ctx *ctx; - struct nvme_dev *dev; - struct list_head list; -}; - -static void nvme_dmabuf_invalidate_mappings(struct dma_buf_attachment *attach) -{ - struct dma_buf_io_ctx *ctx = attach->importer_priv; - - dma_buf_io_invalidate_mappings(ctx); -} - -static const struct dma_buf_attach_ops nvme_dmabuf_importer_ops = { - .invalidate_mappings = nvme_dmabuf_invalidate_mappings, - .allow_peer2peer = true, -}; - -static struct dma_buf_io_map *nvme_dma_buf_io_map(struct dma_buf_io_ctx *ctx) -{ - unsigned nr_entries = ctx->dmabuf->size / NVME_CTRL_PAGE_SIZE; - struct nvme_dma_buf_io_ctx *nvme_ctx = ctx->dev_priv; - struct dma_buf_attachment *attach = nvme_ctx->attach; - unsigned long tmp, i = 0; - struct nvme_dmabuf_map *map; - struct scatterlist *sg; - struct sg_table *sgt; - int ret; - - dma_resv_assert_held(ctx->dmabuf->resv); - - if (ctx->dmabuf->size % NVME_CTRL_PAGE_SIZE) - return ERR_PTR(-EINVAL); - - map = kvmalloc_flex(*map, dma_list, nr_entries); - if (!map) - return ERR_PTR(-ENOMEM); - map->nr_entries = nr_entries; - - sgt = dma_buf_map_attachment(attach, ctx->dir); - if (IS_ERR(sgt)) { - ret = PTR_ERR(sgt); - sgt = NULL; - goto err; - } - - for_each_sgtable_dma_sg(sgt, sg, tmp) { - dma_addr_t dma_addr = sg_dma_address(sg); - unsigned long sg_len = sg_dma_len(sg); - - if ((sg_len % NVME_CTRL_PAGE_SIZE) || - (dma_addr % NVME_CTRL_PAGE_SIZE)) { - ret = -EINVAL; - goto err; - } - while (sg_len) { - map->dma_list[i++] = dma_addr; - dma_addr += NVME_CTRL_PAGE_SIZE; - sg_len -= NVME_CTRL_PAGE_SIZE; - } - } - - ret = dma_buf_io_init_map(ctx, &map->base, sgt); - if (ret) - goto err; - map->sgt = sgt; - return &map->base; -err: - if (sgt) - dma_buf_unmap_attachment(attach, sgt, ctx->dir); - kvfree(map); - return ERR_PTR(ret); -} - -static void nvme_dma_buf_io_unmap(struct dma_buf_io_ctx *ctx, - struct dma_buf_io_map *map_base) -{ - struct nvme_dma_buf_io_ctx *nvme_ctx = ctx->dev_priv; - struct nvme_dmabuf_map *map = to_nvme_dmabuf_map(map_base); - - dma_resv_assert_held(ctx->dmabuf->resv); - - dma_buf_unmap_attachment(nvme_ctx->attach, map->sgt, ctx->dir); - kvfree(map); -} - -static void nvme_dma_buf_io_release(struct dma_buf_io_ctx *ctx) -{ - struct nvme_dma_buf_io_ctx *nvme_ctx = ctx->dev_priv; - struct nvme_dev *dev = nvme_ctx->dev; - - mutex_lock(&dev->dmabuf_lock); - if (!list_empty(&nvme_ctx->list)) { - dma_buf_detach(ctx->dmabuf, nvme_ctx->attach); - list_del_init(&nvme_ctx->list); - } - mutex_unlock(&dev->dmabuf_lock); - - nvme_put_ctrl(&dev->ctrl); - kfree(nvme_ctx); -} - -const struct dma_buf_io_ops nvme_dma_buf_io_ops = { - .map = nvme_dma_buf_io_map, - .unmap = nvme_dma_buf_io_unmap, - .release = nvme_dma_buf_io_release, -}; - -static int __nvme_pci_init_dma_buf_io_ctx(struct nvme_ctrl *ctrl, - struct dma_buf_io_ctx *ctx) -{ - struct nvme_dev *dev = to_nvme_dev(ctrl); - struct nvme_dma_buf_io_ctx *nvme_ctx; - struct dma_buf_attachment *attach; - - nvme_ctx = kzalloc_obj(*nvme_ctx); - if (!nvme_ctx) - return -ENOMEM; - - attach = dma_buf_dynamic_attach(ctx->dmabuf, dev->dev, - &nvme_dmabuf_importer_ops, ctx); - if (IS_ERR(attach)) { - kfree(nvme_ctx); - return PTR_ERR(attach); - } - - nvme_get_ctrl(ctrl); - list_add(&nvme_ctx->list, &dev->dmabuf_ctxs); - nvme_ctx->attach = attach; - nvme_ctx->ctx = ctx; - nvme_ctx->dev = dev; - ctx->dev_priv = nvme_ctx; - ctx->dev_ops = &nvme_dma_buf_io_ops; - return 0; -} - -static int nvme_pci_init_dma_buf_io_ctx(struct nvme_ctrl *ctrl, - struct dma_buf_io_ctx *ctx) -{ - struct nvme_dev *dev = to_nvme_dev(ctrl); - int ret; - - mutex_lock(&dev->dmabuf_lock); - ret = __nvme_pci_init_dma_buf_io_ctx(ctrl, ctx); - mutex_unlock(&dev->dmabuf_lock); - return ret; -} - -static void nvme_pci_remove_dmabuf(struct nvme_dev *dev) -{ - struct nvme_dma_buf_io_ctx *ctx, *tmp; - - mutex_lock(&dev->dmabuf_lock); - list_for_each_entry_safe(ctx, tmp, &dev->dmabuf_ctxs, list) { - dma_buf_io_detach(ctx->ctx); - dma_buf_detach(ctx->ctx->dmabuf, ctx->attach); - list_del_init(&ctx->list); - } - mutex_unlock(&dev->dmabuf_lock); -} -#else -static void nvme_pci_remove_dmabuf(struct nvme_dev *dev) -{ -} -#endif - static const struct blk_mq_ops nvme_mq_admin_ops = { .queue_rq = nvme_queue_rq, .complete = nvme_pci_complete_rq, @@ -3655,7 +3438,7 @@ static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown) mutex_unlock(&dev->shutdown_lock); if (shutdown) - nvme_pci_remove_dmabuf(dev); + nvme_pci_remove_dmabuf(&dev->dmabufs); } static int nvme_disable_prepare_reset(struct nvme_dev *dev, bool shutdown) @@ -3863,6 +3646,14 @@ static unsigned long nvme_pci_get_virt_boundary(struct nvme_ctrl *ctrl, return 0; } +static int nvme_pci_init_dma_buf_io_ctx(struct nvme_ctrl *ctrl, + struct dma_buf_io_ctx *ctx) +{ + struct nvme_dev *dev = to_nvme_dev(ctrl); + + return nvme_init_dma_buf_io_ctx(ctrl, &dev->dmabufs, ctx); +} + static const struct nvme_ctrl_ops nvme_pci_ctrl_ops = { .name = "pcie", .module = THIS_MODULE, @@ -3878,9 +3669,7 @@ static const struct nvme_ctrl_ops nvme_pci_ctrl_ops = { .print_device_info = nvme_pci_print_device_info, .supports_pci_p2pdma = nvme_pci_supports_pci_p2pdma, .get_virt_boundary = nvme_pci_get_virt_boundary, -#ifdef CONFIG_DMA_SHARED_BUFFER .init_dma_buf_io_ctx = nvme_pci_init_dma_buf_io_ctx, -#endif }; static int nvme_dev_map(struct nvme_dev *dev) @@ -4003,8 +3792,7 @@ static struct nvme_dev *nvme_pci_alloc_dev(struct pci_dev *pdev, return ERR_PTR(-ENOMEM); INIT_WORK(&dev->ctrl.reset_work, nvme_reset_work); mutex_init(&dev->shutdown_lock); - INIT_LIST_HEAD(&dev->dmabuf_ctxs); - mutex_init(&dev->dmabuf_lock); + nvme_init_buf_list(&dev->dmabufs); dev->nr_write_queues = write_queues; dev->nr_poll_queues = poll_queues; -- 2.54.0
