In preparation to support VIRTIO_F_DMB, create a mechanism to allocate and map memory from the Device Memory Buffer (DMB). The DMB is a shared memory region a device exposes and owns. A device that negotiates the feature expects its virtqueues and all the buffers we hand it to live in that region, and every address we publish to it is a byte offset into the region.
Add virtio_dmb_init(), which locates the region by the shared memory id the device reports and builds a page-granular allocator over it, and virtio_dmb_destroy() to tear that down. Add virtio_dmb_map_ops, a struct virtio_map_ops implementation that hands out allocations from that allocator as region offsets: alloc() places a virtqueue area in the region, map_page() copies a buffer that lives elsewhere into it and copies it back on unmap. The map operations reach that allocator through a new dmb member of union virtio_map. The shared memory id is transport specific, so add a get_dmb_shm_id() callback to struct virtio_config_ops for a transport to report it. A transport that does not implement it must not accept VIRTIO_F_DMB. Add CONFIG_VIRTIO_DMB to enable this support. It defaults to y, and a kernel that will never meet such a device can turn it off to leave the allocator and its bookkeeping out. Link: https://lore.kernel.org/virtio-comment/[email protected]/ Assisted-by: Kiro:claude-opus-5 checkpatch sparse Signed-off-by: Alexander Graf <[email protected]> --- drivers/virtio/Kconfig | 15 + drivers/virtio/Makefile | 3 +- drivers/virtio/virtio_dmb.c | 1317 +++++++++++++++++++++++++++++++++ drivers/virtio/virtio_dmb.h | 28 + include/linux/virtio.h | 3 + include/linux/virtio_config.h | 8 + 6 files changed, 1373 insertions(+), 1 deletion(-) create mode 100644 drivers/virtio/virtio_dmb.c create mode 100644 drivers/virtio/virtio_dmb.h diff --git a/drivers/virtio/Kconfig b/drivers/virtio/Kconfig index ce5bc0d9ea28..b6b5a36c3c21 100644 --- a/drivers/virtio/Kconfig +++ b/drivers/virtio/Kconfig @@ -188,6 +188,21 @@ config VIRTIO_DEBUG If unsure, say N. +config VIRTIO_DMB + bool "Device Memory Buffer support" + depends on VIRTIO + default y + help + Support devices that place their virtqueues and buffers in a shared + memory region they own, rather than in memory the driver allocates. + + Enabling this adds a page allocator and per-page bookkeeping, both + of which are set up only for a device that negotiates the feature. + When disabled the feature is never accepted and such a device is + driven as an ordinary one. + + If unsure, say Y. + config VIRTIO_RTC tristate "Virtio RTC driver" depends on VIRTIO diff --git a/drivers/virtio/Makefile b/drivers/virtio/Makefile index eefcfe90d6b8..ba785ff44a16 100644 --- a/drivers/virtio/Makefile +++ b/drivers/virtio/Makefile @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 -obj-$(CONFIG_VIRTIO) += virtio.o virtio_ring.o +virtio-dmb-$(CONFIG_VIRTIO_DMB) := virtio_dmb.o +obj-$(CONFIG_VIRTIO) += virtio.o virtio_ring.o $(virtio-dmb-y) obj-$(CONFIG_VIRTIO_ANCHOR) += virtio_anchor.o obj-$(CONFIG_VIRTIO_PCI_LIB) += virtio_pci_modern_dev.o obj-$(CONFIG_VIRTIO_PCI_LIB_LEGACY) += virtio_pci_legacy_dev.o diff --git a/drivers/virtio/virtio_dmb.c b/drivers/virtio/virtio_dmb.c new file mode 100644 index 000000000000..42126f928bc0 --- /dev/null +++ b/drivers/virtio/virtio_dmb.c @@ -0,0 +1,1317 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Device Memory Buffer support for virtio devices. + * + * A device that negotiates VIRTIO_F_DMB owns one shared memory region, the + * Device Memory Buffer, that holds its virtqueues and the buffers they + * reference. Every address the driver publishes to such a device is a byte + * offset from the start of that region. + * + * This file provides an allocator over the region and the virtio_map_ops + * implementation that turns allocations into those offsets. A mapping handle + * belonging to a DMB device is a region offset and nothing else: no code + * outside these operations may treat it as a DMA address. + * + * The region is shared with the device, which may read or write any of it at + * any time. Nothing this file reads back from the region is used to compute + * a kernel address, a length or an index. Handles and their sizes arrive + * from the ring's own bookkeeping in kernel memory, and are range-checked + * anyway so that a bug there cannot reach outside the arrays below. + */ + +#include <linux/align.h> +#include <linux/bitmap.h> +#include <linux/bits.h> +#include <linux/cache.h> +#include <linux/cpumask.h> +#include <linux/dma-mapping.h> +#include <linux/export.h> +#include <linux/highmem.h> +#include <linux/io.h> +#include <linux/ioport.h> +#include <linux/limits.h> +#include <linux/log2.h> +#include <linux/math.h> +#include <linux/minmax.h> +#include <linux/module.h> +#include <linux/overflow.h> +#include <linux/slab.h> +#include <linux/smp.h> +#include <linux/spinlock.h> +#include <linux/virtio.h> +#include <linux/virtio_config.h> + +#include "virtio_dmb.h" + +/* No source recorded for this slot: it holds no bounced mapping. */ +#define DMB_SRC_NONE ((phys_addr_t)-1) + +/** + * struct virtio_dmb_slot - what one PAGE_SIZE slot of the pool records + * @src: physical address this slot bounces, DMB_SRC_NONE for a virtqueue area + * @end: one past the last slot of the allocation this slot belongs to, or + * zero when the slot is free + * @tail: how many bytes of slot @end - 1 the allocation covers, PAGE_SIZE when + * its length is a whole number of pages, zero when the slot is free + * + * Every slot of an allocation records the same @end and the same @tail, so a + * handle that points into the middle of one still yields both the allocation's + * extent in slots and its end in bytes from a single read. + * + * @tail is what makes the length a mapping is bounced against a byte count + * rather than a page count. It is bounded by PAGE_SIZE, so u32 holds it on + * every configuration, which a whole length would not: virtio_dmb_op_alloc() + * bounds a request by the entire pool and a pool may exceed 4 GiB. + */ +struct virtio_dmb_slot { + phys_addr_t src; + u32 end; + u32 tail; +}; + +/* + * Bounds on how many slots one area covers. + * + * The floor is one cacheline of bitmap, so that no two areas contend on the + * line their separate locks exist to keep apart, and at least 512 slots. On a + * 64-byte line those coincide; a wider line raises the floor, which is + * correct. Either way the floor is a power of two of at least BITS_PER_LONG, + * which is what makes an area own whole bitmap words and what makes the area + * of a slot a shift; virtio_dmb_init() asserts both rather than leaving them + * to inspection. + * + * The ceiling has no counterpart in kernel/dma/swiotlb.c, which this geometry + * otherwise follows, and it is needed because the search differs: swiotlb + * finds a run through a per-slot free-run list, while this sweeps a bitmap, + * so the cost of one search here is linear in the size of an area. Without a + * ceiling that cost grows with the region, which is the thing being bounded. + */ +#define DMB_AREA_CACHELINE_SLOTS ((unsigned int)L1_CACHE_BYTES * BITS_PER_BYTE) +#define DMB_AREA_MIN_SLOTS (DMB_AREA_CACHELINE_SLOTS > 512u ? \ + DMB_AREA_CACHELINE_SLOTS : 512u) +#define DMB_AREA_MAX_SLOTS 4096u + +/** + * struct virtio_dmb_area - one independently locked range of the pool + * @used: slots of this area that are allocated; exact under @lock + * @index: slot this area's next search starts from, relative to the area base + * @lock: covers this area's bits of the pool bitmap, @used and @index + * + * Cacheline-aligned where that means anything, so that two areas' locks do not + * share a line; the alignment compiles away on !SMP, where nothing contends. + */ +struct virtio_dmb_area { + unsigned int used; + unsigned int index; + /* Serialises this area's bits of the pool bitmap, @used and @index. */ + spinlock_t lock; +} ____cacheline_aligned_in_smp; + +/** + * struct virtio_dmb - driver-side state for one Device Memory Buffer + * @vdev: the device that owns the region, for message context + * @map_va: what memremap() returned, for memunmap() + * @map_phys: physical base of the region, for release_mem_region() + * @map_len: length of the claimed and mapped part of the region + * @map_claimed: whether request_mem_region() succeeded for that range + * @prev_map: map operations the transport had installed, restored on teardown + * @prev_vmap: mapping token that went with @prev_map + * @base_va: kernel address the pool starts at, inside the mapping + * @base_off: pool start as an offset from the start of the region + * @nslots: pool size in PAGE_SIZE slots + * @bitmap: @nslots bits, set when the slot is allocated + * @slots: @nslots slot records + * @areas: the @nareas ranges the pool is divided into + * @nareas: how many areas the pool is divided into + * @area_slots: slots one area covers, a power of two; the last area covers + * fewer when @nslots is not a multiple of it + * @area_shift: ilog2(@area_slots), so slot >> @area_shift names its area + * @shm_id: shared memory id the device reported for the region + */ +struct virtio_dmb { + struct virtio_device *vdev; + void *map_va; + phys_addr_t map_phys; + size_t map_len; + bool map_claimed; + const struct virtio_map_ops *prev_map; + union virtio_map prev_vmap; + void *base_va; + u64 base_off; + unsigned int nslots; + unsigned long *bitmap; + struct virtio_dmb_slot *slots; + struct virtio_dmb_area *areas; + unsigned int nareas; + unsigned int area_slots; + unsigned int area_shift; + u16 shm_id; +}; + +/* First slot of area @i. */ +static unsigned int virtio_dmb_area_base(const struct virtio_dmb *dmb, + unsigned int i) +{ + return i << dmb->area_shift; +} + +/* Slots area @i covers. The last area is short unless nslots divides. */ +static unsigned int virtio_dmb_area_len(const struct virtio_dmb *dmb, + unsigned int i) +{ + return min(dmb->area_slots, + dmb->nslots - virtio_dmb_area_base(dmb, i)); +} + +static unsigned int virtio_dmb_slots(size_t size) +{ + return DIV_ROUND_UP(size, PAGE_SIZE); +} + +static size_t virtio_dmb_pool_size(const struct virtio_dmb *dmb) +{ + return (size_t)dmb->nslots << PAGE_SHIFT; +} + +/* + * The largest buffer mapping this pool will serve: the smaller of an eighth of + * the pool and half of one area, but never less than one page. That floor is + * what governs a region at the four-slot minimum, since an eighth of four + * pages rounds down to nothing. + * + * The eighth is a choice, not a derived value: it bounds the capacity one + * mapping can deny the rest of the device to seven eighths of the pool, so + * that a device with several virtqueues can still make forward progress while + * one large mapping is outstanding. + * + * The half-area is derived, and it is the reason this function has to be + * consulted rather than the eighth alone. An allocation has to lie inside one + * area, because that is what lets one lock cover it and what lets the release + * path find that lock from the slot index. Half rather than all of an area is + * headroom: a request the size of a whole area could only ever be satisfied by + * a completely empty one, and an area may be as small as + * DMB_AREA_MIN_SLOTS, so there is no expectation that one is empty. The + * figure is half of the nominal area size, so a request at the cap can exceed + * the short last area outright; that costs a claim in one area out of nareas + * and the walk tries the others. + * + * The cap applies to map_page() only. An alloc() is a virtqueue area, which + * is structural rather than in-flight: it lives for as long as the queue + * does and no back-pressure can defer it, so a cap on it could only shrink a + * queue on a region that is too small, or refuse one outright where the ring + * layout cannot be shrunk. Sizing the region for the areas as well as the + * buffers is the device's obligation. + */ +static size_t virtio_dmb_max_mapping(const struct virtio_dmb *dmb) +{ + size_t eighth = ALIGN_DOWN(virtio_dmb_pool_size(dmb) / 8, PAGE_SIZE); + size_t half_area = ((size_t)dmb->area_slots / 2) << PAGE_SHIFT; + + return max_t(size_t, min(eighth, half_area), PAGE_SIZE); +} + +/* + * Claim @nr contiguous slots from area @i, or -ENOMEM when that one area + * cannot satisfy the request. Takes and drops that area's lock and touches + * no other area's state, so no path ever holds two of these locks and there + * is no ordering between them to get right. + */ +static long virtio_dmb_area_claim(struct virtio_dmb *dmb, unsigned int i, + unsigned int nr) +{ + struct virtio_dmb_area *area = &dmb->areas[i]; + unsigned int base = virtio_dmb_area_base(dmb, i); + unsigned int end = base + virtio_dmb_area_len(dmb, i); + unsigned long flags, slot; + + spin_lock_irqsave(&area->lock, flags); + + /* + * Exact, and inside the lock. Written as a subtraction from the + * area's own length rather than as len - used < nr, which underflows. + */ + if (nr > (end - base) - area->used) + goto not_found; + + /* + * Both sweeps are bounded at the area end, so an allocation cannot + * span two areas and the release path can find one lock from the slot + * index. bitmap_find_next_zero_area() returns a value whose sum with + * @nr exceeds the size it was given when it finds nothing, so that sum + * is the test; the whole-pool "slot >= nslots" form does not transfer. + */ + slot = bitmap_find_next_zero_area(dmb->bitmap, end, + base + area->index, nr, 0); + if (slot + nr > end && area->index) + slot = bitmap_find_next_zero_area(dmb->bitmap, end, base, + nr, 0); + if (slot + nr > end) + goto not_found; + + bitmap_set(dmb->bitmap, slot, nr); + area->used += nr; + area->index = slot + nr < end ? slot + nr - base : 0; + + spin_unlock_irqrestore(&area->lock, flags); + + return slot; + +not_found: + spin_unlock_irqrestore(&area->lock, flags); + + return -ENOMEM; +} + +/* + * Claim @nr contiguous slots. Returns the first slot, or -ENOMEM when no + * area can satisfy the request. Exhaustion is a routine condition: the + * region's length bounds how much virtqueue data can be in flight. What a + * caller makes of it is the caller's, and it is not always back-pressure: a + * network receive fill has nothing to push back on when it cannot post a + * buffer, and repolls instead. Reporting it at any level a working device + * would print would therefore be a log flood, and it is the only signal an + * undersized region produces at all, so it is reported through dynamic debug + * where it costs nothing until somebody asks for it. + * + * Next fit within one area, from a hint that advances past each claim and + * rewinds to each release, beginning in the area belonging to the running CPU + * and then trying each other area in turn. So a multi-slot request can fail + * while the total free count would have satisfied it: a map_page() request is + * bounded by virtio_dmb_max_mapping(), which keeps it inside one area, but + * fragmentation within that area can still cost capacity and fail an + * individual mapping. Nothing is moved to recover: handles are live in + * descriptors the device is reading. + * + * The walk visits every area and tests capacity inside that area's lock, so a + * refusal is a true statement about the pool rather than about one area. It + * releases the lock and restores interrupts between areas, which is what + * bounds the interrupts-off window to a single area's sweep; the total work in + * the failing case is a whole-pool sweep either way. + */ +static long virtio_dmb_claim(struct virtio_dmb *dmb, unsigned int nr) +{ + unsigned int i, start; + long ret; + + /* + * raw_smp_processor_id() and not smp_processor_id(): the index is + * computed before any lock is taken, so preemption or migration + * between the read and the claim only changes which area is tried + * first. kernel/dma/swiotlb.c picks its home area on the same + * reasoning. + */ + start = raw_smp_processor_id() % dmb->nareas; + i = start; + do { + ret = virtio_dmb_area_claim(dmb, i, nr); + if (ret >= 0) + return ret; + + if (++i >= dmb->nareas) + i = 0; + } while (i != start); + + /* + * The geometry rather than a free count: there is no instant at which + * a total free count is true under per-area locking, so printing one + * would mean either a walk taking every lock or a torn read. + */ + dev_dbg_ratelimited(&dmb->vdev->dev, + "device memory buffer has no run of %u pages in any of %u areas of %u pages\n", + nr, dmb->nareas, dmb->area_slots); + + return -ENOMEM; +} + +static void virtio_dmb_release(struct virtio_dmb *dmb, unsigned int slot, + unsigned int nr) +{ + struct virtio_dmb_area *area; + unsigned long flags; + unsigned int i, a; + + if (dev_WARN_ONCE(&dmb->vdev->dev, + !nr || slot >= dmb->nslots || nr > dmb->nslots - slot, + "bad device memory buffer slot range %u+%u\n", + slot, nr)) + return; + + /* + * One area holds the whole allocation, so one lock covers it. The + * allocator establishes that by bounding both of its sweeps at an area + * end; enforce it here rather than inherit it, because this is the path + * that depends on it to pick a lock at all, and picking the wrong one + * would clear bits and adjust a count under a lock that does not cover + * either. + */ + a = slot >> dmb->area_shift; + if (dev_WARN_ONCE(&dmb->vdev->dev, + ((slot + nr - 1) >> dmb->area_shift) != a, + "device memory buffer allocation %u+%u spans two areas\n", + slot, nr)) + return; + area = &dmb->areas[a]; + + spin_lock_irqsave(&area->lock, flags); + + /* + * Releasing a range that is not wholly allocated would put slots that + * a different allocation now owns back on the free list, which is what + * a second release of one handle does. The bitmap is the only record + * that can answer whether that is happening, and the test costs less + * than the bitmap_clear() it guards. + */ + if (dev_WARN_ONCE(&dmb->vdev->dev, + find_next_zero_bit(dmb->bitmap, slot + nr, slot) < + slot + nr, + "device memory buffer double release %u+%u\n", + slot, nr)) + goto out; + + /* + * Clear the records before the bits, so that a slot reachable from the + * free list never carries an extent that virtio_dmb_resolve() would + * trust. Doing it here rather than in the callers covers every + * release path with one copy of the invariant. + * + * WRITE_ONCE() because virtio_dmb_resolve() reads these three fields + * without the lock, which is the pattern + * tools/memory-model/Documentation/access-marking.txt calls + * "Lock-Protected Writes With Lockless Reads" and asks to be marked on + * both sides. + */ + for (i = slot; i < slot + nr; i++) { + WRITE_ONCE(dmb->slots[i].src, DMB_SRC_NONE); + WRITE_ONCE(dmb->slots[i].end, 0); + WRITE_ONCE(dmb->slots[i].tail, 0); + } + + bitmap_clear(dmb->bitmap, slot, nr); + area->used -= nr; + area->index = slot - virtio_dmb_area_base(dmb, a); + +out: + spin_unlock_irqrestore(&area->lock, flags); +} + +/** + * struct virtio_dmb_ref - a handle resolved against the pool + * @slot: the slot the handle lands in + * @nr: slots the allocation still holds from @slot on + * @src: the physical address @slot bounces, or DMB_SRC_NONE for a queue area + * + * @nr and @src are derived from the same read of the record that validated the + * handle, so the extent a caller releases and the pages the copy touches are + * the ones that were checked and not a later re-read of a field another CPU may + * meanwhile have cleared. + */ +struct virtio_dmb_ref { + unsigned int slot; + unsigned int nr; + phys_addr_t src; +}; + +/* Extra conditions virtio_dmb_resolve() enforces for particular callers. */ +#define DMB_RESOLVE_BOUNCED BIT(0) /* must be a mapping, not a queue area */ +#define DMB_RESOLVE_WHOLE BIT(1) /* must be the allocation's first slot */ + +/* + * Turn a handle and a length into a slot index and an extent, rejecting + * anything that does not lie wholly inside the pool. The bound is exclusive, + * so a handle at the end of the pool and a zero length are both refused. The + * arithmetic is done in u64 so that it cannot wrap where dma_addr_t is + * narrower. + * + * A handle may point into the middle of a mapping, because + * virtqueue_map_sync_single_range_for_cpu() and its counterpart pass + * handle + offset, so the extent is taken from the slot the handle lands in + * rather than from the first slot of the allocation. That rejects a handle + * released twice, a length that runs past the end of its allocation, and a + * range that would continue into a neighbouring one. + * + * The length is checked in bytes, not in pages. A page-granular check would + * pass a length ending anywhere inside the allocation's last slot, which is up + * to PAGE_SIZE - 1 bytes past what was mapped, and virtio_dmb_copy() would + * then touch the page after the source run. swiotlb_bounce() keeps the same + * bound in alloc_size (kernel/dma/swiotlb.c:890) and clamps an over-long + * mapping to it, because it has callers it cannot refuse. Every caller here + * passes a length the map side recorded, so an over-long one is a caller bug + * and the copy is refused outright rather than truncated. + * + * DMB_RESOLVE_BOUNCED additionally requires the allocation to be one this file + * bounced rather than a virtqueue area. DMB_RESOLVE_WHOLE requires the handle + * to be the start of its allocation, which is what the callers that release it + * need: mid-extent tolerance exists for the sync ops alone, and a mid-extent + * release would free the tail of an allocation and leak its head for good. + * + * The resolved slot is read without the lock, which is legitimate for a caller + * that owns it: virtio_dmb_claim() published it exclusively to this caller, so + * nothing else writes it. The DMB_RESOLVE_WHOLE test also reads the preceding + * slot, which the caller does not own, and that read rests on a different + * argument, stated where it is made. + */ +static bool virtio_dmb_resolve(struct virtio_dmb *dmb, dma_addr_t handle, + size_t size, unsigned int rflags, + struct virtio_dmb_ref *ref) +{ + u64 off, end, limit; + phys_addr_t src; + u32 slot_end, tail; + + if (!size || (u64)handle < dmb->base_off) + goto bad_handle; + + off = (u64)handle - dmb->base_off; + if (check_add_overflow(off, (u64)size, &end)) + goto bad_handle; + if (end > (u64)virtio_dmb_pool_size(dmb)) + goto bad_handle; + + ref->slot = off >> PAGE_SHIFT; + + /* + * Snapshot the record here, once. Every test below, the extent the + * caller goes on to release and the pages virtio_dmb_copy() touches are + * taken from these three reads rather than from a second look at the + * entry, so what was validated is what gets used. Reading the slot + * without the lock is legitimate for a caller that owns it, as above, + * but the callers these tests exist to catch are exactly the ones that + * do not own it, and for those another CPU may be writing the entry. + */ + slot_end = READ_ONCE(dmb->slots[ref->slot].end); + tail = READ_ONCE(dmb->slots[ref->slot].tail); + src = READ_ONCE(dmb->slots[ref->slot].src); + + if (dev_WARN_ONCE(&dmb->vdev->dev, !slot_end, + "device memory buffer handle %pad holds no allocation\n", + &handle)) + return false; + + /* The allocation's exclusive end in bytes, reachable from any slot. */ + limit = ((u64)(slot_end - 1) << PAGE_SHIFT) + tail; + + if (dev_WARN_ONCE(&dmb->vdev->dev, end > limit, + "device memory buffer handle %pad length %zu leaves its allocation\n", + &handle, size)) + return false; + + /* + * Allocations are disjoint and every slot of one records the same end, + * so the preceding slot shares that end if and only if this slot is not + * the first of the allocation. + * + * That slot may belong to another caller, which is the read the + * ownership argument above does not cover. Disjointness covers it + * instead: a neighbouring allocation ends at or before this slot and a + * free slot records zero, so whichever of the two a concurrent + * virtio_dmb_record() or virtio_dmb_release() leaves visible, neither + * can equal slot_end. + */ + if ((rflags & DMB_RESOLVE_WHOLE) && + dev_WARN_ONCE(&dmb->vdev->dev, + !IS_ALIGNED(off, PAGE_SIZE) || + (ref->slot && + READ_ONCE(dmb->slots[ref->slot - 1].end) == slot_end), + "device memory buffer handle %pad is not the start of its allocation\n", + &handle)) + return false; + + if ((rflags & DMB_RESOLVE_BOUNCED) && + dev_WARN_ONCE(&dmb->vdev->dev, src == DMB_SRC_NONE, + "device memory buffer handle %pad holds no mapping\n", + &handle)) + return false; + + ref->nr = slot_end - ref->slot; + ref->src = src; + return true; + +bad_handle: + dev_WARN_ONCE(&dmb->vdev->dev, 1, + "device memory buffer handle %pad length %zu out of range\n", + &handle, size); + return false; +} + +/* + * Copy between the region and the pages a mapping bounces. @ref is what + * virtio_dmb_resolve() validated for this handle, or what the map side has just + * recorded; every source address is derived from its snapshot rather than from + * dmb->slots[], which this function does not read at all, so validation and use + * cannot disagree about where the source is. Walk the source one page at a + * time through kmap_local_page(): the source may be highmem, and the per-slot + * record is a physical address precisely so that no assumption about a linear + * kernel mapping across the mapping's pages is needed. The region side needs + * no such split, being one contiguous mapping. + */ +static void virtio_dmb_copy(struct virtio_dmb *dmb, + const struct virtio_dmb_ref *ref, + dma_addr_t handle, size_t size, bool to_region) +{ + size_t off = (size_t)((u64)handle - dmb->base_off); + phys_addr_t base = ref->src - ((phys_addr_t)ref->slot << PAGE_SHIFT); + size_t done = 0; + + while (done < size) { + size_t pos = off + done; + phys_addr_t src = base + pos; + unsigned int in_src = offset_in_page(src); + void *region = dmb->base_va + pos; + size_t n; + void *va; + + n = min(size - done, (size_t)PAGE_SIZE - in_src); + + va = kmap_local_page(pfn_to_page(PHYS_PFN(src))); + if (to_region) + memcpy(region, va + in_src, n); + else + memcpy(va + in_src, region, n); + kunmap_local(va); + + done += n; + } +} + +/* + * Record @nr slots from @slot as one allocation of @size bytes bouncing @src. + * + * Each slot carries the source address of its own page rather than only the + * first slot carrying the address of the mapping. That is what makes + * virtio_dmb_copy() independent of which slot of the allocation a handle + * resolved to: taking the slot index back off the recorded address yields the + * same base from any slot of the mapping. The sync ops do resolve to a slot + * in the middle of one, being handed handle + offset. + * + * The tail is recorded on every slot for the same reason, so that the byte end + * of the allocation is derivable from a mid-extent handle without also knowing + * which slot the allocation starts at. + */ +static void virtio_dmb_record(struct virtio_dmb *dmb, unsigned int slot, + unsigned int nr, size_t size, phys_addr_t src) +{ + unsigned int i; + u32 tail; + + /* + * @nr is virtio_dmb_slots(size) at both call sites, so the last slot + * carries between 1 and PAGE_SIZE bytes and the cast cannot truncate. + */ + tail = (u32)(size - ((size_t)(nr - 1) << PAGE_SHIFT)); + + for (i = 0; i < nr; i++) { + WRITE_ONCE(dmb->slots[slot + i].src, + src == DMB_SRC_NONE ? + DMB_SRC_NONE : src + ((phys_addr_t)i << PAGE_SHIFT)); + WRITE_ONCE(dmb->slots[slot + i].end, slot + nr); + WRITE_ONCE(dmb->slots[slot + i].tail, tail); + } +} + +static void *virtio_dmb_op_alloc(union virtio_map map, size_t size, + dma_addr_t *map_handle, gfp_t gfp) +{ + struct virtio_dmb *dmb = map.dmb; + unsigned int nr, slot; + void *va; + long ret; + + /* + * The allocation-behaviour bits of gfp are ignored, because claiming + * slots neither sleeps nor allocates; __GFP_NOWARN is honoured, for + * the reason the failure path below gives. The result is zeroed + * because this stands in for dma_alloc_coherent(), whose callers rely + * on that. + */ + if (!size) + return NULL; + + /* + * Bound the request before a slot count is derived from it, as + * map_page() does. The bound is the whole pool rather than the + * fraction of it virtio_dmb_max_mapping() reports, for the reason + * that function gives. It is not the effective limit: every search + * is bounded at one pool area, so an allocation larger than + * area_slots pages fails even on an empty pool. + */ + if (size > virtio_dmb_pool_size(dmb)) + goto no_room; + + nr = virtio_dmb_slots(size); + ret = virtio_dmb_claim(dmb, nr); + if (ret < 0) + goto no_room; + slot = ret; + + va = dmb->base_va + ((size_t)slot << PAGE_SHIFT); + memset(va, 0, (size_t)nr << PAGE_SHIFT); + virtio_dmb_record(dmb, slot, nr, size, DMB_SRC_NONE); + + *map_handle = dmb->base_off + ((u64)slot << PAGE_SHIFT); + return va; + +no_room: + /* + * A buffer that does not fit is back-pressure and stays quiet, but + * this is a virtqueue area: no back-pressure can defer it, and a + * region sized for the buffers but not for the areas otherwise fails + * queue setup with nothing to tell it apart from every other reason + * find_vqs() can fail, and it is the one of those a larger region + * fixes. + * + * Which is why __GFP_NOWARN has to be honoured rather than ignored. + * vring_alloc_queue_split() walks the queue size down from the size + * the device asked for and marks every attempt but the last with the + * flag, so warning regardless would print a line for every attempt + * but the last, for a probe that then succeeds. Dynamic debug still carries the + * message, which is what a packed ring has to rely on: none of its + * three areas can be made smaller and all three set the flag. + */ + if (gfp & __GFP_NOWARN) + dev_dbg(&dmb->vdev->dev, + "no room for a %zu-byte virtqueue area in %u pages\n", + size, dmb->nslots); + else + dev_warn(&dmb->vdev->dev, + "no room for a %zu-byte virtqueue area in %u pages\n", + size, dmb->nslots); + return NULL; +} + +/* + * DMB_RESOLVE_WHOLE alone: there is deliberately no converse of + * DMB_RESOLVE_BOUNCED insisting that the allocation is a virtqueue area. A + * caller reaching this with a mapping has called free() on something it got + * from map_page(), and would lose the copy-out that unmap_page() does; the + * bytes it loses are its own, and the only in-tree caller of the exported + * virtqueue_map_free_coherent() is vring_free_queue(), which frees an area. + * Refusing here would trade that for a leak of the slots, which is worse. + */ +static void virtio_dmb_op_free(union virtio_map map, size_t size, void *vaddr, + dma_addr_t map_handle, unsigned long attrs) +{ + struct virtio_dmb *dmb = map.dmb; + struct virtio_dmb_ref ref; + + if (!virtio_dmb_resolve(dmb, map_handle, size, DMB_RESOLVE_WHOLE, &ref)) + return; + + virtio_dmb_release(dmb, ref.slot, ref.nr); +} + +static dma_addr_t virtio_dmb_op_map_page(union virtio_map map, + struct page *page, + unsigned long offset, size_t size, + enum dma_data_direction dir, + unsigned long attrs) +{ + struct virtio_dmb *dmb = map.dmb; + phys_addr_t src = page_to_phys(page) + offset; + struct virtio_dmb_ref ref; + unsigned int nr, slot; + dma_addr_t handle; + long ret; + + if (!size || size > virtio_dmb_max_mapping(dmb)) + return DMA_MAPPING_ERROR; + + nr = virtio_dmb_slots(size); + ret = virtio_dmb_claim(dmb, nr); + if (ret < 0) + return DMA_MAPPING_ERROR; + slot = ret; + + virtio_dmb_record(dmb, slot, nr, size, src); + + handle = dmb->base_off + ((u64)slot << PAGE_SHIFT); + + ref.slot = slot; + ref.nr = nr; + ref.src = src; + + /* + * Copy the caller's buffer in whatever the direction is, and without + * honouring DMA_ATTR_SKIP_CPU_SYNC. swiotlb_tbl_map_single() bounces + * unconditionally for the same two reasons: a device that writes less + * than the whole buffer must leave the rest of the caller's bytes + * intact, and the mapped bytes must not reach the device as whatever + * the slot held before. + * + * Those bytes and no others. size need not be a multiple of + * PAGE_SIZE, and [size, nr << PAGE_SHIFT) keeps what the slots held + * before: a freed virtqueue area, an earlier mapping of this device, + * or what the device left there itself. Nothing outside a mapping's + * own length is ever copied in, so the device reads nothing there + * that the region did not already hold for it, and the descriptor + * carries a length. swiotlb leaves the remainder of its last slot + * the same way. + */ + virtio_dmb_copy(dmb, &ref, handle, size, true); + + return handle; +} + +static void virtio_dmb_op_unmap_page(union virtio_map map, + dma_addr_t map_handle, size_t size, + enum dma_data_direction dir, + unsigned long attrs) +{ + struct virtio_dmb *dmb = map.dmb; + struct virtio_dmb_ref ref; + + if (!virtio_dmb_resolve(dmb, map_handle, size, + DMB_RESOLVE_BOUNCED | DMB_RESOLVE_WHOLE, &ref)) + return; + + if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) && + (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)) + virtio_dmb_copy(dmb, &ref, map_handle, size, false); + + /* + * The slot count comes from what map_page() recorded rather than from + * the caller's size, so a mismatched size cannot release a different + * number of slots than were claimed. + */ + virtio_dmb_release(dmb, ref.slot, ref.nr); +} + +static void virtio_dmb_op_sync_single_for_cpu(union virtio_map map, + dma_addr_t map_handle, + size_t size, + enum dma_data_direction dir) +{ + struct virtio_dmb *dmb = map.dmb; + struct virtio_dmb_ref ref; + + /* A zero-length sync is a no-op and not a bad handle. */ + if (!size) + return; + + if (!virtio_dmb_resolve(dmb, map_handle, size, DMB_RESOLVE_BOUNCED, + &ref)) + return; + + if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL) + virtio_dmb_copy(dmb, &ref, map_handle, size, false); +} + +static void virtio_dmb_op_sync_single_for_device(union virtio_map map, + dma_addr_t map_handle, + size_t size, + enum dma_data_direction dir) +{ + struct virtio_dmb *dmb = map.dmb; + struct virtio_dmb_ref ref; + + /* A zero-length sync is a no-op and not a bad handle. */ + if (!size) + return; + + if (!virtio_dmb_resolve(dmb, map_handle, size, DMB_RESOLVE_BOUNCED, + &ref)) + return; + + if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL) + virtio_dmb_copy(dmb, &ref, map_handle, size, true); +} + +static bool virtio_dmb_op_need_sync(union virtio_map map, dma_addr_t map_handle) +{ + /* Every mapping is a bounce, so every sync is a real copy. */ + return true; +} + +static int virtio_dmb_op_mapping_error(union virtio_map map, + dma_addr_t map_handle) +{ + /* + * DMA_MAPPING_ERROR is the value virtio_ring reserves. Offset 0 is + * reserved too, by the proposal and by every device implementation + * that reads a queue address of zero as a queue that was never + * programmed, so it is not an address this driver may publish either. + * Nothing allocated here yields it, because the pool starts after the + * first byte of the region for the reason virtio_dmb_init() gives, but + * a premapped buffer carries an address its caller obtained and + * vring_map_one_sg() asks this operation to judge that one. Whether + * such an address came from this map cannot be answered here, and a + * containment test would answer a different question, but zero can be + * answered: it is the one value the proposal rules out outright. + */ + if (map_handle == DMA_MAPPING_ERROR || !map_handle) + return -ENOMEM; + + return 0; +} + +static size_t virtio_dmb_op_max_mapping_size(union virtio_map map) +{ + return virtio_dmb_max_mapping(map.dmb); +} + +static const struct virtio_map_ops virtio_dmb_map_ops = { + .map_page = virtio_dmb_op_map_page, + .unmap_page = virtio_dmb_op_unmap_page, + .sync_single_for_cpu = virtio_dmb_op_sync_single_for_cpu, + .sync_single_for_device = virtio_dmb_op_sync_single_for_device, + .alloc = virtio_dmb_op_alloc, + .free = virtio_dmb_op_free, + .need_sync = virtio_dmb_op_need_sync, + .mapping_error = virtio_dmb_op_mapping_error, + .max_mapping_size = virtio_dmb_op_max_mapping_size, +}; + +/* + * Whether the device still has virtqueues. vqs_list_lock is what protects + * that list against a concurrent adder. No caller here can race one, because + * every path that reaches this runs under the device lock and before or after + * the driver's find_vqs(), but the invariant is worth enforcing rather than + * inheriting from callers this file does not control. + */ +static bool virtio_dmb_vqs_live(struct virtio_device *vdev) +{ + bool live; + + spin_lock(&vdev->vqs_list_lock); + live = !list_empty(&vdev->vqs); + spin_unlock(&vdev->vqs_list_lock); + + return live; +} + +/** + * virtio_dmb_destroy - release the Device Memory Buffer state of a device + * @vdev: the device + * + * Does nothing unless @vdev is currently using a Device Memory Buffer, and + * refuses if the device still has virtqueues: they would be left pointing + * into a region that is no longer mapped. The mapping and the physical + * region claim then stay behind until something deletes those virtqueues and + * calls again, which unbinding the driver does: virtio_dev_remove() calls the + * driver's remove() before this. + */ +void virtio_dmb_destroy(struct virtio_device *vdev) +{ + struct virtio_dmb *dmb; + + /* + * vdev->map identifies which member of vdev->vmap is live, so it is + * also the test for whether the union holds a Device Memory Buffer. + */ + if (vdev->map != &virtio_dmb_map_ops) + return; + + /* + * A virtqueue keeps the mapping token it was created with, while + * vdev->map is consulted afresh on every dispatch. Clearing vdev->map + * therefore does not disarm a live virtqueue, it redirects that + * virtqueue's copy of the token into the DMA API, where the pointer + * this frees would be used as a struct device. Refuse instead and + * leak the mapping, which is unconditionally better than a + * use-after-free. + * + * Reported rather than warned about, because a driver that left its + * virtqueues in place is not the only way to get here. A device that + * fails to report its region on the way back from a suspend takes + * virtio_device_restore() to its error path, which calls this, and a + * driver with no freeze callback still has its virtqueues at that + * point, correctly. A condition a correct driver can satisfy must + * not taint the kernel. + */ + if (virtio_dmb_vqs_live(vdev)) { + dev_warn(&vdev->dev, + "device memory buffer not released, virtqueues are still live\n"); + return; + } + + dmb = vdev->vmap.dmb; + + /* Put back exactly what the transport had installed. */ + vdev->map = dmb->prev_map; + vdev->vmap = dmb->prev_vmap; + + memunmap(dmb->map_va); + if (dmb->map_claimed) + release_mem_region(dmb->map_phys, dmb->map_len); + kvfree(dmb->slots); + kfree(dmb->areas); + bitmap_free(dmb->bitmap); + kfree(dmb); +} +EXPORT_SYMBOL_GPL(virtio_dmb_destroy); + +/** + * virtio_dmb_init - make a device's Device Memory Buffer state current + * @vdev: the device, with feature negotiation complete + * + * Reads the shared memory id the device reports, locates the region, builds + * an allocator over it and routes every mapping of the device through it. + * When the feature is not negotiated, releases any state a previous + * negotiation left behind. + * + * The operation is "make the state match what the device reports now", and it + * is reached again from resume and from reset completion. A device that + * reports the region it reported last time keeps the state it already has, so + * handles held by a virtqueue that outlived the transition stay valid. A + * device that reports a different region has the state rebuilt when no + * virtqueue is live, and is refused otherwise: a virtqueue holds kernel + * addresses inside the mapping and cannot be redirected into a new one. + * + * A caller that gets an error must set the FAILED device status bit, and must + * not touch the device for anything else before it does. The device has + * already confirmed the feature by the time this runs, so it is entitled to + * assume the driver will address it through the region; the bit is what tells + * it the driver gave up instead. + * + * Return: 0 on success, or a negative errno. + */ +int virtio_dmb_init(struct virtio_device *vdev) +{ + struct virtio_shm_region region; + struct virtio_dmb *dmb; + unsigned int nslots, skew; + unsigned int area_slots, nareas, target, i; + u64 base_off, slots = 0; + size_t map_len; + u16 shm_id; + int err; + + if (!virtio_has_feature(vdev, VIRTIO_F_DMB)) { + /* + * The feature may have been withdrawn across re-negotiation. + * + * virtio_dmb_destroy() refuses under live virtqueues, and + * returning 0 after a refusal would leave this file's map + * operations installed for a device that has not negotiated the + * feature, so every later mapping would resolve a handle + * against a region the device no longer agrees it has. Report + * the refusal to the caller instead, which sets the FAILED + * device status bit. No path reaches this today: it needs map + * operations an earlier negotiation installed, which unbinding + * destroys, so only the restore path can find them, and that + * path hands finalize_features() the word already accepted + * rather than the offer, so a transport could drop the feature + * there only in reaction to a device that changed what it + * offers after the driver bound, and no reset does that. + */ + virtio_dmb_destroy(vdev); + if (vdev->map == &virtio_dmb_map_ops) + return -EBUSY; + return 0; + } + + /* + * Without both of these the region cannot be located at all, which is + * what keeps a transport that does not implement them from offering + * the feature in the first place. + */ + if (!vdev->config->get_dmb_shm_id || !vdev->config->get_shm_region) { + dev_warn(&vdev->dev, + "transport cannot locate a device memory buffer\n"); + return -EINVAL; + } + + /* The feature is only defined together with VIRTIO_F_ACCESS_PLATFORM. */ + if (!virtio_has_feature(vdev, VIRTIO_F_ACCESS_PLATFORM)) { + dev_warn(&vdev->dev, + "device memory buffer without VIRTIO_F_ACCESS_PLATFORM\n"); + return -EINVAL; + } + + err = vdev->config->get_dmb_shm_id(vdev, &shm_id); + if (err) + return err; + + /* A region is looked up by a u8 id. */ + if (shm_id > U8_MAX) { + dev_warn(&vdev->dev, + "device memory buffer id %u out of range\n", shm_id); + return -EINVAL; + } + + if (!virtio_get_shm_region(vdev, ®ion, shm_id)) { + dev_warn(&vdev->dev, + "cannot locate device memory buffer region %u\n", + shm_id); + return -ENODEV; + } + + /* + * The region base carries no alignment guarantee, but every virtqueue + * layout requires one of the areas placed in it. Start the pool at a + * PAGE_SIZE-aligned address and record the skew, so that page-granular + * allocation makes every absolute address aligned. + * + * PAGE_SIZE - skew is the distance from the start of the region to the + * first aligned address strictly after it, so the pool never begins at + * the region's first byte and no handle is ever 0. The proposal + * reserves offset 0: it is not the address of any structure the driver + * places in the region, and a device may treat it as an error. A device + * that predates the reservation reads a queue address of 0 as the queue + * never having been programmed and ignores it, so the value is unusable + * either way. Keeping it out of the pool costs one page of an + * already-aligned region and nothing at all of a misaligned one, whose + * leading partial page was unusable regardless. + */ + skew = offset_in_page(region.addr); + base_off = PAGE_SIZE - skew; + + if (region.len > base_off) + slots = (region.len - base_off) >> PAGE_SHIFT; + + /* + * The least a region could hold: one minimally-sized virtqueue plus + * one buffer in flight against it. A packed queue costs three + * allocations, a descriptor ring and two event structures, and a split + * queue up to two when the transport aligns its areas to PAGE_SIZE, so + * four slots is the floor for either layout. + * + * Four slots is four pages of pool, which is five pages of region for + * a region whose base is already aligned, since the page the pool + * starts after is not part of it. + * + * That derivation counts one virtqueue. A device that also offers an + * administration virtqueue has its areas allocated from the same region + * through the same path, and one administration command occupies + * several slots more, so four pages is a floor such a device is + * misconfigured to sit on rather than a size it can work at. + * + * This floor is enforced, but it is a floor and not a sufficiency + * check. How much a device actually needs depends on how many + * virtqueues its driver creates and how deep they are, neither of + * which is known here: this runs during feature negotiation, before + * find_vqs(). A region above this floor but still too small fails + * there instead, which for a split ring reduces the queue depth and + * for a packed ring fails the queue. + * + * The upper bounds are what a slot index, a mapping length and a + * published handle can each represent. The mapping length bound is + * exclusive because the length mapped is base_off larger than the + * pool, and base_off is a whole page where the region base is + * aligned: at the last representable slot count that sum would wrap + * to zero on a 32-bit size_t. + */ + if (slots < 4 || slots > UINT_MAX || + slots >= (u64)(SIZE_MAX >> PAGE_SHIFT) || + base_off + (slots << PAGE_SHIFT) - 1 > + DMA_BIT_MASK(BITS_PER_TYPE(dma_addr_t))) { + dev_warn(&vdev->dev, + "device memory buffer region holds %llu usable pages\n", + slots); + return -EINVAL; + } + nslots = slots; + + /* Nothing outside the pool and the bytes ahead of it is used. */ + map_len = base_off + ((size_t)nslots << PAGE_SHIFT); + + /* + * The transport reports the region in 64 bits while a resource is + * addressed in resource_size_t. Refuse a region that does not fit + * rather than claim and map a truncated one. + */ + if (region.addr > (u64)(resource_size_t)-1 - map_len) { + dev_warn(&vdev->dev, + "device memory buffer region at 0x%llx is not addressable\n", + region.addr); + return -EINVAL; + } + + /* + * Everything that identifies the region is known now and nothing has + * been touched yet, so an unchanged region can be adopted instead of + * being torn down and rebuilt identically. That is what lets a + * virtqueue which outlived a suspend or a reset keep handles that are + * still valid, and it is why no separate freeze-time teardown is + * needed. A region that moved can be neither adopted nor replaced + * under live virtqueues, so refuse without a warning: a device that + * moves its region while its driver still has virtqueues is + * misbehaving, which is not evidence of a kernel bug. + */ + if (vdev->map == &virtio_dmb_map_ops) { + dmb = vdev->vmap.dmb; + + if (dmb->shm_id == shm_id && dmb->map_phys == region.addr && + dmb->map_len == map_len) + return 0; + + if (virtio_dmb_vqs_live(vdev)) + return -EBUSY; + + virtio_dmb_destroy(vdev); + } + + dmb = kzalloc(sizeof(*dmb), GFP_KERNEL); + if (!dmb) + return -ENOMEM; + + dmb->vdev = vdev; + dmb->shm_id = shm_id; + dmb->base_off = base_off; + dmb->map_phys = region.addr; + dmb->map_len = map_len; + + /* + * A shared memory region need not lie in a BAR the transport already + * claimed, so record a claim on the range here. The claim is + * advisory: where the region does lie in such a BAR the transport's + * own claim already covers it, and that is not a conflict with + * anything, so it must not fail the device. A foreign driver cannot + * own another device's BAR range either, so a refusal here is not + * evidence that anything is wrong. + */ + dmb->map_claimed = request_mem_region(region.addr, map_len, + "virtio-dmb") != NULL; + if (!dmb->map_claimed) + dev_dbg(&vdev->dev, + "device memory buffer region at 0x%llx already reserved\n", + region.addr); + + /* + * MEMREMAP_DEC because the region is memory shared with the device, + * which is what the proposal requires of a driver wherever the platform + * distinguishes that from memory private to the driver. Without it + * x86's arch_memremap_wb() applies the guest's own encryption to the + * mapping, and the device would see ciphertext wherever a guest encrypts + * its memory. x86 is the only architecture that reads the flag; arm64 + * ignores it and reaches ioremap_prot() instead, where a realm guest's + * hook finds the region is not protected memory and shares the mapping. + * Anywhere else that draws the distinction, the proposal forbids the + * device from offering the feature at all. + */ + dmb->map_va = memremap(region.addr, map_len, + MEMREMAP_WB | MEMREMAP_DEC); + if (!dmb->map_va) { + /* + * memremap() is silent, and every sibling failure in this + * function names itself. Without this the device is left with + * the FAILED status bit set and nothing saying why. + */ + dev_warn(&vdev->dev, + "cannot map device memory buffer region at 0x%llx\n", + region.addr); + err = -ENOMEM; + goto err_unclaim; + } + + /* + * The skew was derived from the physical base, so the pool is aligned + * in the mapping only if the mapping kept that page offset. Check it + * rather than assume it. + */ + if (offset_in_page(dmb->map_va) != skew) { + dev_warn(&vdev->dev, + "device memory buffer mapping is not page-congruent\n"); + err = -EINVAL; + goto err_unmap; + } + + dmb->base_va = dmb->map_va + (size_t)base_off; + + dmb->bitmap = bitmap_zalloc(nslots, GFP_KERNEL); + if (!dmb->bitmap) { + err = -ENOMEM; + goto err_unmap; + } + + /* A zeroed record has end == 0, which is what marks a slot free. */ + dmb->slots = kvcalloc(nslots, sizeof(*dmb->slots), GFP_KERNEL); + if (!dmb->slots) { + err = -ENOMEM; + goto err_free_bitmap; + } + + /* + * Divide the pool into independently locked areas, so that mappings on + * different CPUs do not serialise on one lock and the interrupts-off + * window of one search does not grow with the region. This is the + * structure kernel/dma/swiotlb.c adopted in commit 20347fca71a3 + * ("swiotlb: split up the global swiotlb lock"), for the same reason. + * + * area_slots is the power of two and nareas is derived from it, which + * is the reverse of swiotlb. swiotlb indexes areas with a mask and + * rounds its pool size up to suit; a region's length is the device's + * and cannot be rounded up, and dividing a power-of-two area count into + * it would leave area_slots neither a power of two nor a multiple of + * BITS_PER_LONG. That matters for correctness rather than for tuning: + * bitmap_set() and bitmap_clear() are non-atomic read-modify-write on + * an unsigned long, so two areas sharing a bitmap word under separate + * locks would lose updates. A power-of-two area_slots at least + * BITS_PER_LONG makes every area own whole words, and makes the area of + * a slot a shift. The cost is one division per claim where swiotlb has + * a mask, which is a fraction of the two copies every mapping already + * performs. + * + * num_possible_cpus() and not num_online_cpus(), so that the division + * is sized for the CPUs that can run rather than for the ones running + * when the region is installed, which on a guest that onlines the rest + * later would divide the pool for one. Nothing is allocated per CPU + * and no area belongs to one, so a CPU going away strands no capacity + * and there is no hotplug callback. A pool too small to divide that + * far yields fewer areas than CPUs, which then share. + */ + BUILD_BUG_ON(DMB_AREA_MIN_SLOTS > DMB_AREA_MAX_SLOTS); + BUILD_BUG_ON(DMB_AREA_MIN_SLOTS < BITS_PER_LONG); + BUILD_BUG_ON(!is_power_of_2(DMB_AREA_MIN_SLOTS)); + + target = nslots / roundup_pow_of_two(num_possible_cpus()); + if (target < DMB_AREA_MIN_SLOTS) + /* rounddown_pow_of_two(0) is undefined. */ + area_slots = DMB_AREA_MIN_SLOTS; + else + area_slots = clamp_t(unsigned int, + rounddown_pow_of_two(target), + DMB_AREA_MIN_SLOTS, DMB_AREA_MAX_SLOTS); + + nareas = DIV_ROUND_UP(nslots, area_slots); + + dmb->areas = kcalloc(nareas, sizeof(*dmb->areas), GFP_KERNEL); + if (!dmb->areas) { + err = -ENOMEM; + goto err_free_slots; + } + + for (i = 0; i < nareas; i++) + spin_lock_init(&dmb->areas[i].lock); + + dmb->nslots = nslots; + dmb->nareas = nareas; + dmb->area_slots = area_slots; + dmb->area_shift = ilog2(area_slots); + + /* Published last: until now nothing routes a mapping here. */ + dmb->prev_map = vdev->map; + dmb->prev_vmap = vdev->vmap; + vdev->vmap.dmb = dmb; + vdev->map = &virtio_dmb_map_ops; + + /* + * The feature moves every virtqueue of this device into a region and + * changes what every address published to it means, and it activates + * from a value the device supplies that nothing else records. Report + * the three facts about it that are recoverable nowhere else, on the + * device that negotiated it, and the derived area geometry with them so + * that it is visible without debugfs. + */ + dev_info(&vdev->dev, + "device memory buffer %u at %pa, %u usable pages in %u areas of %u pages\n", + shm_id, &dmb->map_phys, nslots, dmb->nareas, + dmb->area_slots); + + return 0; + +err_free_slots: + kvfree(dmb->slots); +err_free_bitmap: + bitmap_free(dmb->bitmap); +err_unmap: + memunmap(dmb->map_va); +err_unclaim: + if (dmb->map_claimed) + release_mem_region(dmb->map_phys, dmb->map_len); + kfree(dmb); + return err; +} +EXPORT_SYMBOL_GPL(virtio_dmb_init); + +MODULE_DESCRIPTION("Virtio device memory buffer allocator"); +MODULE_LICENSE("GPL"); diff --git a/drivers/virtio/virtio_dmb.h b/drivers/virtio/virtio_dmb.h new file mode 100644 index 000000000000..69fbcbb9c2c0 --- /dev/null +++ b/drivers/virtio/virtio_dmb.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Device Memory Buffer support for virtio devices. + */ +#ifndef _DRIVERS_VIRTIO_VIRTIO_DMB_H +#define _DRIVERS_VIRTIO_VIRTIO_DMB_H + +struct virtio_device; + +#if IS_ENABLED(CONFIG_VIRTIO_DMB) + +int virtio_dmb_init(struct virtio_device *vdev); +void virtio_dmb_destroy(struct virtio_device *vdev); + +#else + +static inline int virtio_dmb_init(struct virtio_device *vdev) +{ + return 0; +} + +static inline void virtio_dmb_destroy(struct virtio_device *vdev) +{ +} + +#endif /* CONFIG_VIRTIO_DMB */ + +#endif /* _DRIVERS_VIRTIO_VIRTIO_DMB_H */ diff --git a/include/linux/virtio.h b/include/linux/virtio.h index 93e573c56563..cd3fe82ff328 100644 --- a/include/linux/virtio.h +++ b/include/linux/virtio.h @@ -44,12 +44,15 @@ struct virtqueue { }; struct vduse_vq_group; +struct virtio_dmb; union virtio_map { /* Device that performs DMA */ struct device *dma_dev; /* VDUSE specific virtqueue group for doing map */ struct vduse_vq_group *group; + /* Device Memory Buffer holding the virtqueues and their buffers */ + struct virtio_dmb *dmb; }; int virtqueue_add_outbuf(struct virtqueue *vq, diff --git a/include/linux/virtio_config.h b/include/linux/virtio_config.h index 69f84ea85d71..a6780aa85966 100644 --- a/include/linux/virtio_config.h +++ b/include/linux/virtio_config.h @@ -96,6 +96,13 @@ struct virtqueue_info { * @set_vq_affinity: set the affinity for a virtqueue (optional). * @get_vq_affinity: get the affinity for a virtqueue (optional). * @get_shm_region: get a shared memory region based on the index. + * @get_dmb_shm_id: get the shared memory id of the Device Memory Buffer + * (optional). + * vdev: the device + * id: where to store the shared memory id + * Returns 0 on success or error status + * Only valid once VIRTIO_F_DMB has been negotiated. A transport that + * does not implement this must not accept VIRTIO_F_DMB. * @disable_vq_and_reset: reset a queue individually (optional). * vq: the virtqueue * Returns 0 on success or error status @@ -135,6 +142,7 @@ struct virtio_config_ops { int index); bool (*get_shm_region)(struct virtio_device *vdev, struct virtio_shm_region *region, u8 id); + int (*get_dmb_shm_id)(struct virtio_device *vdev, u16 *id); int (*disable_vq_and_reset)(struct virtqueue *vq); int (*enable_vq_after_reset)(struct virtqueue *vq); };
