Hi, The intentional 4GB limit is for forwarding: what if you forward mbufs from a 64bit-capable interface to another one that doesn't support 64bit DMA? And even if you would only enable it if all interfaces are 64bit-capable, what if you plug in a 32bit USB/hotplug interface? We did not want to support bounce buffers in OpenBSD.
Reyk > On 11.07.2016, at 15:37, Stefan Fritsch <[email protected]> wrote: > > Hi, > > following the discussion about mbufs, I have some questions about 64bit > DMA in general. > > Openbsd on amd64 assumes that DMA is only possible to the lower 4GB. But > there are many devices (PCIe, virtio, ...) that can do DMA to the whole > memory. Is it feasible to have known good devices opt-in into 64bit DMA? > > I have done a patch that allows virtio to do 64bit DMA. This works insofar > as the queues used by the device will now be allocated above 4GB. But this > is only a small amount of memory. More interesting would be bufs and > mbufs. > > > For bufs, Bob added some code for copying bufs above/below 4GB. But this > code only has a single flag B_DMA to denote if DMA is possible into a buf > or not. Would it make sense to replace that by a mechanism that is device > specific, so that we can use devices efficiently that allow 64bit DMA? > Maybe a flag in the device vnode? > > > Does it make sense to commit something like the diff below (not tested > much), even if it saves at most a few MB below 4GB right now? > > Cheers, > Stefan > > > diff --git sys/arch/amd64/amd64/bus_dma.c sys/arch/amd64/amd64/bus_dma.c > index 8eaa2e7..1aba7c0 100644 > --- sys/arch/amd64/amd64/bus_dma.c > +++ sys/arch/amd64/amd64/bus_dma.c > @@ -293,6 +293,7 @@ _bus_dmamap_load_raw(bus_dma_tag_t t, bus_dmamap_t map, > bus_dma_segment_t *segs, > { > bus_addr_t paddr, baddr, bmask, lastaddr = 0; > bus_size_t plen, sgsize, mapsize; > + struct uvm_constraint_range *constraint = t->_cookie; > int first = 1; > int i, seg = 0; > > @@ -320,7 +321,7 @@ _bus_dmamap_load_raw(bus_dma_tag_t t, bus_dmamap_t map, > bus_dma_segment_t *segs, > if (plen < sgsize) > sgsize = plen; > > - if (paddr > dma_constraint.ucr_high) > + if (paddr > constraint->ucr_high) > panic("Non dma-reachable buffer at paddr > %#lx(raw)", > paddr); > > @@ -405,15 +406,11 @@ _bus_dmamem_alloc(bus_dma_tag_t t, bus_size_t size, > bus_size_t alignment, > bus_size_t boundary, bus_dma_segment_t *segs, int nsegs, int *rsegs, > int flags) > { > + struct uvm_constraint_range *constraint = t->_cookie; > > - /* > - * XXX in the presence of decent (working) iommus and bouncebuffers > - * we can then fallback this allocation to a range of { 0, -1 }. > - * However for now we err on the side of caution and allocate dma > - * memory under the 4gig boundary. > - */ > return (_bus_dmamem_alloc_range(t, size, alignment, boundary, > - segs, nsegs, rsegs, flags, (bus_addr_t)0, (bus_addr_t)0xffffffff)); > + segs, nsegs, rsegs, flags, (bus_addr_t)constraint->ucr_low, > + (bus_addr_t)constraint->ucr_high)); > } > > /* > @@ -567,6 +564,7 @@ _bus_dmamap_load_buffer(bus_dma_tag_t t, bus_dmamap_t > map, void *buf, > bus_size_t sgsize; > bus_addr_t curaddr, lastaddr, baddr, bmask; > vaddr_t vaddr = (vaddr_t)buf; > + struct uvm_constraint_range *constraint = t->_cookie; > int seg; > pmap_t pmap; > > @@ -584,7 +582,7 @@ _bus_dmamap_load_buffer(bus_dma_tag_t t, bus_dmamap_t > map, void *buf, > */ > pmap_extract(pmap, vaddr, (paddr_t *)&curaddr); > > - if (curaddr > dma_constraint.ucr_high) > + if (curaddr > constraint->ucr_high) > panic("Non dma-reachable buffer at curaddr %#lx(raw)", > curaddr); > > diff --git sys/arch/amd64/amd64/machdep.c sys/arch/amd64/amd64/machdep.c > index de9f481..7640532 100644 > --- sys/arch/amd64/amd64/machdep.c > +++ sys/arch/amd64/amd64/machdep.c > @@ -201,6 +201,12 @@ struct vm_map *phys_map = NULL; > > /* UVM constraint ranges. */ > struct uvm_constraint_range isa_constraint = { 0x0, 0x00ffffffUL }; > + /* > + * XXX in the presence of decent (working) iommus and bouncebuffers > + * we can then fallback this allocation to a range of { 0, -1 }. > + * However for now we err on the side of caution and allocate dma > + * memory under the 4gig boundary. > + */ > struct uvm_constraint_range dma_constraint = { 0x0, 0xffffffffUL }; > struct uvm_constraint_range *uvm_md_constraints[] = { > &isa_constraint, > diff --git sys/arch/amd64/include/pci_machdep.h > sys/arch/amd64/include/pci_machdep.h > index 27b833b..bf54f31 100644 > --- sys/arch/amd64/include/pci_machdep.h > +++ sys/arch/amd64/include/pci_machdep.h > @@ -41,6 +41,7 @@ > */ > > extern struct bus_dma_tag pci_bus_dma_tag; > +extern struct bus_dma_tag virtio_pci_bus_dma_tag; > > /* > * Types provided to machine-independent PCI code > diff --git sys/arch/amd64/isa/isa_machdep.c sys/arch/amd64/isa/isa_machdep.c > index 74dc907..ec35edead 100644 > --- sys/arch/amd64/isa/isa_machdep.c > +++ sys/arch/amd64/isa/isa_machdep.c > @@ -140,7 +140,7 @@ void _isa_dma_free_bouncebuf(bus_dma_tag_t, > bus_dmamap_t); > * buffers, if necessary. > */ > struct bus_dma_tag isa_bus_dma_tag = { > - NULL, /* _cookie */ > + &isa_constraint, > _isa_bus_dmamap_create, > _isa_bus_dmamap_destroy, > _isa_bus_dmamap_load, > diff --git sys/arch/amd64/pci/pci_machdep.c sys/arch/amd64/pci/pci_machdep.c > index 19983b0..5dca63a 100644 > --- sys/arch/amd64/pci/pci_machdep.c > +++ sys/arch/amd64/pci/pci_machdep.c > @@ -130,7 +130,7 @@ do { > \ > * of these functions. > */ > struct bus_dma_tag pci_bus_dma_tag = { > - NULL, /* _may_bounce */ > + &dma_constraint, > _bus_dmamap_create, > _bus_dmamap_destroy, > _bus_dmamap_load, > @@ -147,6 +147,25 @@ struct bus_dma_tag pci_bus_dma_tag = { > _bus_dmamem_mmap, > }; > > +struct bus_dma_tag virtio_pci_bus_dma_tag = { > + &no_constraint, > + _bus_dmamap_create, > + _bus_dmamap_destroy, > + _bus_dmamap_load, > + _bus_dmamap_load_mbuf, > + _bus_dmamap_load_uio, > + _bus_dmamap_load_raw, > + _bus_dmamap_unload, > + _bus_dmamap_sync, > + _bus_dmamem_alloc, > + _bus_dmamem_alloc_range, > + _bus_dmamem_free, > + _bus_dmamem_map, > + _bus_dmamem_unmap, > + _bus_dmamem_mmap, > +}; > + > + > void > pci_attach_hook(struct device *parent, struct device *self, > struct pcibus_attach_args *pba) > diff --git sys/arch/i386/pci/pci_machdep.h sys/arch/i386/pci/pci_machdep.h > index 2831470..6e6484b 100644 > --- sys/arch/i386/pci/pci_machdep.h > +++ sys/arch/i386/pci/pci_machdep.h > @@ -56,6 +56,7 @@ union i386_pci_tag_u { > }; > > extern struct bus_dma_tag pci_bus_dma_tag; > +#define virtio_pci_bus_dma_tag pci_bus_dma_tag > > /* > * Types provided to machine-independent PCI code > diff --git sys/dev/pci/virtio_pci.c sys/dev/pci/virtio_pci.c > index 489a449..6c08579 100644 > --- sys/dev/pci/virtio_pci.c > +++ sys/dev/pci/virtio_pci.c > @@ -183,7 +183,7 @@ virtio_pci_attach(struct device *parent, struct device > *self, void *aux) > > vsc->sc_ops = &virtio_pci_ops; > sc->sc_pc = pc; > - vsc->sc_dmat = pa->pa_dmat; > + vsc->sc_dmat = &virtio_pci_bus_dma_tag; > sc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI; > > if (pci_mapreg_map(pa, PCI_MAPREG_START, PCI_MAPREG_TYPE_IO, 0, >
