Author: br
Date: Wed May  8 15:22:27 2019
New Revision: 347337
URL: https://svnweb.freebsd.org/changeset/base/347337

Log:
  o Implement a bounce buffer based on device reserved memory.
    Grab device reserved physical memory regions from FDT using standard
    "memory-region" property and use vmem(9) to allocate buffers from it.
    The same vmem could be used by DMA engine drivers to allocate memory for
    DMA descriptors.
    This is required for platforms that provide uncached memory region
    reserved exclusively for DMA operations.
  o Change sleepable sx(9) lock type to non-sleepable mutex(9) since
    network drivers usually hold mutex during DMA operations. So we don't
    take sleepable lock after non-sleepable.
  
  Tested on U.S. Government Furnished Equipment (GFE) 64-bit RISC-V cores.
  
  Sponsored by: DARPA, AFRL

Modified:
  head/sys/dev/xdma/xdma.c
  head/sys/dev/xdma/xdma.h
  head/sys/dev/xdma/xdma_bank.c
  head/sys/dev/xdma/xdma_bio.c
  head/sys/dev/xdma/xdma_mbuf.c
  head/sys/dev/xdma/xdma_queue.c
  head/sys/dev/xdma/xdma_sg.c

Modified: head/sys/dev/xdma/xdma.c
==============================================================================
--- head/sys/dev/xdma/xdma.c    Wed May  8 15:20:16 2019        (r347336)
+++ head/sys/dev/xdma/xdma.c    Wed May  8 15:22:27 2019        (r347337)
@@ -1,7 +1,8 @@
 /*-
- * Copyright (c) 2016-2018 Ruslan Bukin <b...@bsdpad.com>
- * All rights reserved.
+ * SPDX-License-Identifier: BSD-2-Clause
  *
+ * Copyright (c) 2016-2019 Ruslan Bukin <b...@bsdpad.com>
+ *
  * This software was developed by SRI International and the University of
  * Cambridge Computer Laboratory under DARPA/AFRL contract FA8750-10-C-0237
  * ("CTSRD"), as part of the DARPA CRASH research programme.
@@ -43,7 +44,6 @@ __FBSDID("$FreeBSD$");
 #include <sys/lock.h>
 #include <sys/sysctl.h>
 #include <sys/systm.h>
-#include <sys/sx.h>
 
 #include <machine/bus.h>
 
@@ -61,12 +61,14 @@ __FBSDID("$FreeBSD$");
  * Multiple xDMA controllers may work with single DMA device,
  * so we have global lock for physical channel management.
  */
-static struct sx xdma_sx;
+static struct mtx xdma_mtx;
 
-#define        XDMA_LOCK()                     sx_xlock(&xdma_sx)
-#define        XDMA_UNLOCK()                   sx_xunlock(&xdma_sx)
-#define        XDMA_ASSERT_LOCKED()            sx_xassert(&xdma_sx, MA_OWNED)
+#define        XDMA_LOCK()                     mtx_lock(&xdma_mtx)
+#define        XDMA_UNLOCK()                   mtx_unlock(&xdma_mtx)
+#define        XDMA_ASSERT_LOCKED()            mtx_assert(&xdma_mtx, MA_OWNED)
 
+#define        FDT_REG_CELLS   4
+
 /*
  * Allocate virtual xDMA channel.
  */
@@ -95,11 +97,11 @@ xdma_channel_alloc(xdma_controller_t *xdma, uint32_t c
 
        TAILQ_INIT(&xchan->ie_handlers);
 
-       sx_init(&xchan->sx_lock, "xDMA chan");
-       sx_init(&xchan->sx_qin_lock, "xDMA qin");
-       sx_init(&xchan->sx_qout_lock, "xDMA qout");
-       sx_init(&xchan->sx_bank_lock, "xDMA bank");
-       sx_init(&xchan->sx_proc_lock, "xDMA proc");
+       mtx_init(&xchan->mtx_lock, "xDMA chan", NULL, MTX_DEF);
+       mtx_init(&xchan->mtx_qin_lock, "xDMA qin", NULL, MTX_DEF);
+       mtx_init(&xchan->mtx_qout_lock, "xDMA qout", NULL, MTX_DEF);
+       mtx_init(&xchan->mtx_bank_lock, "xDMA bank", NULL, MTX_DEF);
+       mtx_init(&xchan->mtx_proc_lock, "xDMA proc", NULL, MTX_DEF);
 
        TAILQ_INIT(&xchan->bank);
        TAILQ_INIT(&xchan->queue_in);
@@ -138,11 +140,11 @@ xdma_channel_free(xdma_channel_t *xchan)
 
        xdma_teardown_all_intr(xchan);
 
-       sx_destroy(&xchan->sx_lock);
-       sx_destroy(&xchan->sx_qin_lock);
-       sx_destroy(&xchan->sx_qout_lock);
-       sx_destroy(&xchan->sx_bank_lock);
-       sx_destroy(&xchan->sx_proc_lock);
+       mtx_destroy(&xchan->mtx_lock);
+       mtx_destroy(&xchan->mtx_qin_lock);
+       mtx_destroy(&xchan->mtx_qout_lock);
+       mtx_destroy(&xchan->mtx_bank_lock);
+       mtx_destroy(&xchan->mtx_proc_lock);
 
        TAILQ_REMOVE(&xdma->channels, xchan, xchan_next);
 
@@ -303,6 +305,95 @@ xdma_ofw_md_data(xdma_controller_t *xdma, pcell_t *cel
        return (ret);
 }
 
+static int
+xdma_handle_mem_node(vmem_t *vmem, phandle_t memory)
+{
+       pcell_t reg[FDT_REG_CELLS * FDT_MEM_REGIONS];
+       pcell_t *regp;
+       int addr_cells, size_cells;
+       int i, reg_len, ret, tuple_size, tuples;
+       vmem_addr_t mem_start;
+       vmem_size_t mem_size;
+
+       if ((ret = fdt_addrsize_cells(OF_parent(memory), &addr_cells,
+           &size_cells)) != 0)
+               return (ret);
+
+       if (addr_cells > 2)
+               return (ERANGE);
+
+       tuple_size = sizeof(pcell_t) * (addr_cells + size_cells);
+       reg_len = OF_getproplen(memory, "reg");
+       if (reg_len <= 0 || reg_len > sizeof(reg))
+               return (ERANGE);
+
+       if (OF_getprop(memory, "reg", reg, reg_len) <= 0)
+               return (ENXIO);
+
+       tuples = reg_len / tuple_size;
+       regp = (pcell_t *)&reg;
+       for (i = 0; i < tuples; i++) {
+               ret = fdt_data_to_res(regp, addr_cells, size_cells,
+                   &mem_start, &mem_size);
+               if (ret != 0)
+                       return (ret);
+
+               vmem_add(vmem, mem_start, mem_size, 0);
+               regp += addr_cells + size_cells;
+       }
+
+       return (0);
+}
+
+vmem_t *
+xdma_get_memory(device_t dev)
+{
+       phandle_t mem_node, node;
+       pcell_t mem_handle;
+       vmem_t *vmem;
+
+       node = ofw_bus_get_node(dev);
+       if (node <= 0) {
+               device_printf(dev,
+                   "%s called on not ofw based device.\n", __func__);
+               return (NULL);
+       }
+
+       if (!OF_hasprop(node, "memory-region"))
+               return (NULL);
+
+       if (OF_getencprop(node, "memory-region", (void *)&mem_handle,
+           sizeof(mem_handle)) <= 0)
+               return (NULL);
+
+       vmem = vmem_create("xDMA vmem", 0, 0, PAGE_SIZE,
+           PAGE_SIZE, M_BESTFIT | M_WAITOK);
+       if (vmem == NULL)
+               return (NULL);
+
+       mem_node = OF_node_from_xref(mem_handle);
+       if (xdma_handle_mem_node(vmem, mem_node) != 0) {
+               vmem_destroy(vmem);
+               return (NULL);
+       }
+
+       return (vmem);
+}
+
+void
+xdma_put_memory(vmem_t *vmem)
+{
+
+       vmem_destroy(vmem);
+}
+
+void
+xchan_set_memory(xdma_channel_t *xchan, vmem_t *vmem)
+{
+
+       xchan->vmem = vmem;
+}
+
 /*
  * Allocate xdma controller.
  */
@@ -400,7 +491,7 @@ static void
 xdma_init(void)
 {
 
-       sx_init(&xdma_sx, "xDMA");
+       mtx_init(&xdma_mtx, "xDMA", NULL, MTX_DEF);
 }
 
 SYSINIT(xdma, SI_SUB_DRIVERS, SI_ORDER_FIRST, xdma_init, NULL);

Modified: head/sys/dev/xdma/xdma.h
==============================================================================
--- head/sys/dev/xdma/xdma.h    Wed May  8 15:20:16 2019        (r347336)
+++ head/sys/dev/xdma/xdma.h    Wed May  8 15:22:27 2019        (r347337)
@@ -1,7 +1,8 @@
 /*-
- * Copyright (c) 2016-2018 Ruslan Bukin <b...@bsdpad.com>
- * All rights reserved.
+ * SPDX-License-Identifier: BSD-2-Clause
  *
+ * Copyright (c) 2016-2019 Ruslan Bukin <b...@bsdpad.com>
+ *
  * This software was developed by SRI International and the University of
  * Cambridge Computer Laboratory under DARPA/AFRL contract FA8750-10-C-0237
  * ("CTSRD"), as part of the DARPA CRASH research programme.
@@ -34,6 +35,7 @@
 #define _DEV_XDMA_XDMA_H_
 
 #include <sys/proc.h>
+#include <sys/vmem.h>
 
 enum xdma_direction {
        XDMA_MEM_TO_MEM,
@@ -73,6 +75,7 @@ struct xdma_controller {
        device_t dev;           /* DMA consumer device_t. */
        device_t dma_dev;       /* A real DMA device_t. */
        void *data;             /* OFW MD part. */
+       vmem_t *vmem;           /* Bounce memory. */
 
        /* List of virtual channels allocated. */
        TAILQ_HEAD(xdma_channel_list, xdma_channel)     channels;
@@ -84,6 +87,9 @@ struct xchan_buf {
        bus_dmamap_t                    map;
        uint32_t                        nsegs;
        uint32_t                        nsegs_left;
+       vm_offset_t                     vaddr;
+       vm_offset_t                     paddr;
+       vm_size_t                       size;
 };
 
 struct xdma_request {
@@ -117,6 +123,7 @@ struct xdma_sglist {
 
 struct xdma_channel {
        xdma_controller_t               *xdma;
+       vmem_t                          *vmem;
 
        uint32_t                        flags;
 #define        XCHAN_BUFS_ALLOCATED            (1 << 0)
@@ -139,11 +146,11 @@ struct xdma_channel {
        TAILQ_HEAD(, xdma_intr_handler) ie_handlers;
        TAILQ_ENTRY(xdma_channel)       xchan_next;
 
-       struct sx                       sx_lock;
-       struct sx                       sx_qin_lock;
-       struct sx                       sx_qout_lock;
-       struct sx                       sx_bank_lock;
-       struct sx                       sx_proc_lock;
+       struct mtx                      mtx_lock;
+       struct mtx                      mtx_qin_lock;
+       struct mtx                      mtx_qout_lock;
+       struct mtx                      mtx_bank_lock;
+       struct mtx                      mtx_proc_lock;
 
        /* Request queue. */
        bus_dma_tag_t                   dma_tag_bufs;
@@ -176,30 +183,30 @@ struct xdma_intr_handler {
 
 static MALLOC_DEFINE(M_XDMA, "xdma", "xDMA framework");
 
-#define        XCHAN_LOCK(xchan)               sx_xlock(&(xchan)->sx_lock)
-#define        XCHAN_UNLOCK(xchan)             sx_xunlock(&(xchan)->sx_lock)
+#define        XCHAN_LOCK(xchan)               mtx_lock(&(xchan)->mtx_lock)
+#define        XCHAN_UNLOCK(xchan)             mtx_unlock(&(xchan)->mtx_lock)
 #define        XCHAN_ASSERT_LOCKED(xchan)      \
-    sx_assert(&(xchan)->sx_lock, SX_XLOCKED)
+    mtx_assert(&(xchan)->mtx_lock, MA_OWNED)
 
-#define        QUEUE_IN_LOCK(xchan)            sx_xlock(&(xchan)->sx_qin_lock)
-#define        QUEUE_IN_UNLOCK(xchan)          
sx_xunlock(&(xchan)->sx_qin_lock)
+#define        QUEUE_IN_LOCK(xchan)            mtx_lock(&(xchan)->mtx_qin_lock)
+#define        QUEUE_IN_UNLOCK(xchan)          
mtx_unlock(&(xchan)->mtx_qin_lock)
 #define        QUEUE_IN_ASSERT_LOCKED(xchan)   \
-    sx_assert(&(xchan)->sx_qin_lock, SX_XLOCKED)
+    mtx_assert(&(xchan)->mtx_qin_lock, MA_OWNED)
 
-#define        QUEUE_OUT_LOCK(xchan)           sx_xlock(&(xchan)->sx_qout_lock)
-#define        QUEUE_OUT_UNLOCK(xchan)         
sx_xunlock(&(xchan)->sx_qout_lock)
+#define        QUEUE_OUT_LOCK(xchan)           
mtx_lock(&(xchan)->mtx_qout_lock)
+#define        QUEUE_OUT_UNLOCK(xchan)         
mtx_unlock(&(xchan)->mtx_qout_lock)
 #define        QUEUE_OUT_ASSERT_LOCKED(xchan)  \
-    sx_assert(&(xchan)->sx_qout_lock, SX_XLOCKED)
+    mtx_assert(&(xchan)->mtx_qout_lock, MA_OWNED)
 
-#define        QUEUE_BANK_LOCK(xchan)          sx_xlock(&(xchan)->sx_bank_lock)
-#define        QUEUE_BANK_UNLOCK(xchan)        
sx_xunlock(&(xchan)->sx_bank_lock)
+#define        QUEUE_BANK_LOCK(xchan)          
mtx_lock(&(xchan)->mtx_bank_lock)
+#define        QUEUE_BANK_UNLOCK(xchan)        
mtx_unlock(&(xchan)->mtx_bank_lock)
 #define        QUEUE_BANK_ASSERT_LOCKED(xchan) \
-    sx_assert(&(xchan)->sx_bank_lock, SX_XLOCKED)
+    mtx_assert(&(xchan)->mtx_bank_lock, MA_OWNED)
 
-#define        QUEUE_PROC_LOCK(xchan)          sx_xlock(&(xchan)->sx_proc_lock)
-#define        QUEUE_PROC_UNLOCK(xchan)        
sx_xunlock(&(xchan)->sx_proc_lock)
+#define        QUEUE_PROC_LOCK(xchan)          
mtx_lock(&(xchan)->mtx_proc_lock)
+#define        QUEUE_PROC_UNLOCK(xchan)        
mtx_unlock(&(xchan)->mtx_proc_lock)
 #define        QUEUE_PROC_ASSERT_LOCKED(xchan) \
-    sx_assert(&(xchan)->sx_proc_lock, SX_XLOCKED)
+    mtx_assert(&(xchan)->mtx_proc_lock, MA_OWNED)
 
 #define        XDMA_SGLIST_MAXLEN      2048
 #define        XDMA_MAX_SEG            128
@@ -207,11 +214,14 @@ static MALLOC_DEFINE(M_XDMA, "xdma", "xDMA framework")
 /* xDMA controller ops */
 xdma_controller_t *xdma_ofw_get(device_t dev, const char *prop);
 int xdma_put(xdma_controller_t *xdma);
+vmem_t * xdma_get_memory(device_t dev);
+void xdma_put_memory(vmem_t *vmem);
 
 /* xDMA channel ops */
 xdma_channel_t * xdma_channel_alloc(xdma_controller_t *, uint32_t caps);
 int xdma_channel_free(xdma_channel_t *);
 int xdma_request(xdma_channel_t *xchan, struct xdma_request *r);
+void xchan_set_memory(xdma_channel_t *xchan, vmem_t *vmem);
 
 /* SG interface */
 int xdma_prep_sg(xdma_channel_t *, uint32_t,

Modified: head/sys/dev/xdma/xdma_bank.c
==============================================================================
--- head/sys/dev/xdma/xdma_bank.c       Wed May  8 15:20:16 2019        
(r347336)
+++ head/sys/dev/xdma/xdma_bank.c       Wed May  8 15:22:27 2019        
(r347337)
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2018 Ruslan Bukin <b...@bsdpad.com>
+ * Copyright (c) 2018-2019 Ruslan Bukin <b...@bsdpad.com>
  * All rights reserved.
  *
  * This software was developed by SRI International and the University of
@@ -37,7 +37,6 @@ __FBSDID("$FreeBSD$");
 #include <sys/bus.h>
 #include <sys/kernel.h>
 #include <sys/malloc.h>
-#include <sys/sx.h>
 
 #include <machine/bus.h>
 

Modified: head/sys/dev/xdma/xdma_bio.c
==============================================================================
--- head/sys/dev/xdma/xdma_bio.c        Wed May  8 15:20:16 2019        
(r347336)
+++ head/sys/dev/xdma/xdma_bio.c        Wed May  8 15:22:27 2019        
(r347337)
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2017-2018 Ruslan Bukin <b...@bsdpad.com>
+ * Copyright (c) 2017-2019 Ruslan Bukin <b...@bsdpad.com>
  * All rights reserved.
  *
  * This software was developed by SRI International and the University of
@@ -37,7 +37,6 @@ __FBSDID("$FreeBSD$");
 #include <sys/bus.h>
 #include <sys/kernel.h>
 #include <sys/malloc.h>
-#include <sys/sx.h>
 
 #include <machine/bus.h>
 

Modified: head/sys/dev/xdma/xdma_mbuf.c
==============================================================================
--- head/sys/dev/xdma/xdma_mbuf.c       Wed May  8 15:20:16 2019        
(r347336)
+++ head/sys/dev/xdma/xdma_mbuf.c       Wed May  8 15:22:27 2019        
(r347337)
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2017-2018 Ruslan Bukin <b...@bsdpad.com>
+ * Copyright (c) 2017-2019 Ruslan Bukin <b...@bsdpad.com>
  * All rights reserved.
  *
  * This software was developed by SRI International and the University of
@@ -36,7 +36,6 @@ __FBSDID("$FreeBSD$");
 #include <sys/conf.h>
 #include <sys/bus.h>
 #include <sys/kernel.h>
-#include <sys/sx.h>
 #include <sys/mbuf.h>
 
 #include <machine/bus.h>

Modified: head/sys/dev/xdma/xdma_queue.c
==============================================================================
--- head/sys/dev/xdma/xdma_queue.c      Wed May  8 15:20:16 2019        
(r347336)
+++ head/sys/dev/xdma/xdma_queue.c      Wed May  8 15:22:27 2019        
(r347337)
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2018 Ruslan Bukin <b...@bsdpad.com>
+ * Copyright (c) 2018-2019 Ruslan Bukin <b...@bsdpad.com>
  * All rights reserved.
  *
  * This software was developed by SRI International and the University of
@@ -37,7 +37,6 @@ __FBSDID("$FreeBSD$");
 #include <sys/bus.h>
 #include <sys/kernel.h>
 #include <sys/malloc.h>
-#include <sys/sx.h>
 
 #include <machine/bus.h>
 

Modified: head/sys/dev/xdma/xdma_sg.c
==============================================================================
--- head/sys/dev/xdma/xdma_sg.c Wed May  8 15:20:16 2019        (r347336)
+++ head/sys/dev/xdma/xdma_sg.c Wed May  8 15:22:27 2019        (r347337)
@@ -1,7 +1,8 @@
 /*-
- * Copyright (c) 2018 Ruslan Bukin <b...@bsdpad.com>
- * All rights reserved.
+ * SPDX-License-Identifier: BSD-2-Clause
  *
+ * Copyright (c) 2018-2019 Ruslan Bukin <b...@bsdpad.com>
+ *
  * This software was developed by SRI International and the University of
  * Cambridge Computer Laboratory under DARPA/AFRL contract FA8750-10-C-0237
  * ("CTSRD"), as part of the DARPA CRASH research programme.
@@ -38,10 +39,14 @@ __FBSDID("$FreeBSD$");
 #include <sys/kernel.h>
 #include <sys/malloc.h>
 #include <sys/mbuf.h>
-#include <sys/sx.h>
+#include <sys/rwlock.h>
 
 #include <machine/bus.h>
 
+#include <vm/vm.h>
+#include <vm/vm_extern.h>
+#include <vm/vm_page.h>
+
 #ifdef FDT
 #include <dev/fdt/fdt_common.h>
 #include <dev/ofw/ofw_bus.h>
@@ -58,25 +63,71 @@ struct seg_load_request {
        uint32_t error;
 };
 
+static void
+xchan_bufs_free_reserved(xdma_channel_t *xchan)
+{
+       struct xdma_request *xr;
+       vm_size_t size;
+       int i;
+
+       for (i = 0; i < xchan->xr_num; i++) {
+               xr = &xchan->xr_mem[i];
+               size = xr->buf.size;
+               if (xr->buf.vaddr) {
+                       pmap_kremove_device(xr->buf.vaddr, size);
+                       kva_free(xr->buf.vaddr, size);
+                       xr->buf.vaddr = 0;
+               }
+               if (xr->buf.paddr) {
+                       vmem_free(xchan->vmem, xr->buf.paddr, size);
+                       xr->buf.paddr = 0;
+               }
+               xr->buf.size = 0;
+       }
+}
+
 static int
-_xchan_bufs_alloc(xdma_channel_t *xchan)
+xchan_bufs_alloc_reserved(xdma_channel_t *xchan)
 {
        xdma_controller_t *xdma;
        struct xdma_request *xr;
+       vmem_addr_t addr;
+       vm_size_t size;
        int i;
 
        xdma = xchan->xdma;
 
+       if (xchan->vmem == NULL)
+               return (ENOBUFS);
+
        for (i = 0; i < xchan->xr_num; i++) {
                xr = &xchan->xr_mem[i];
-               /* TODO: bounce buffer */
+               size = round_page(xchan->maxsegsize);
+               if (vmem_alloc(xchan->vmem, size,
+                   M_BESTFIT | M_NOWAIT, &addr)) {
+                       device_printf(xdma->dev,
+                           "%s: Can't allocate memory\n", __func__);
+                       xchan_bufs_free_reserved(xchan);
+                       return (ENOMEM);
+               }
+               
+               xr->buf.size = size;
+               xr->buf.paddr = addr;
+               xr->buf.vaddr = kva_alloc(size);
+               if (xr->buf.vaddr == 0) {
+                       device_printf(xdma->dev,
+                           "%s: Can't allocate KVA\n", __func__);
+                       xchan_bufs_free_reserved(xchan);
+                       return (ENOMEM);
+               }
+               pmap_kenter_device(xr->buf.vaddr, size, addr);
        }
 
        return (0);
 }
 
 static int
-_xchan_bufs_alloc_busdma(xdma_channel_t *xchan)
+xchan_bufs_alloc_busdma(xdma_channel_t *xchan)
 {
        xdma_controller_t *xdma;
        struct xdma_request *xr;
@@ -138,9 +189,10 @@ xchan_bufs_alloc(xdma_channel_t *xchan)
        }
 
        if (xchan->caps & XCHAN_CAP_BUSDMA)
-               ret = _xchan_bufs_alloc_busdma(xchan);
-       else
-               ret = _xchan_bufs_alloc(xchan);
+               ret = xchan_bufs_alloc_busdma(xchan);
+       else {
+               ret = xchan_bufs_alloc_reserved(xchan);
+       }
        if (ret != 0) {
                device_printf(xdma->dev,
                    "%s: Can't allocate bufs.\n", __func__);
@@ -169,12 +221,8 @@ xchan_bufs_free(xdma_channel_t *xchan)
                        bus_dmamap_destroy(xchan->dma_tag_bufs, b->map);
                }
                bus_dma_tag_destroy(xchan->dma_tag_bufs);
-       } else {
-               for (i = 0; i < xchan->xr_num; i++) {
-                       xr = &xchan->xr_mem[i];
-                       /* TODO: bounce buffer */
-               }
-       }
+       } else
+               xchan_bufs_free_reserved(xchan);
 
        xchan->flags &= ~XCHAN_BUFS_ALLOCATED;
 
@@ -296,6 +344,11 @@ xchan_seg_done(xdma_channel_t *xchan,
                                bus_dmamap_sync(xchan->dma_tag_bufs, b->map, 
                                    BUS_DMASYNC_POSTREAD);
                        bus_dmamap_unload(xchan->dma_tag_bufs, b->map);
+               } else {
+                       if (xr->req_type == XR_TYPE_MBUF &&
+                           xr->direction == XDMA_DEV_TO_MEM)
+                               m_copyback(xr->m, 0, st->transferred,
+                                   (void *)xr->buf.vaddr);
                }
                xr->status.error = st->error;
                xr->status.transferred = st->transferred;
@@ -437,7 +490,13 @@ _xdma_load_data(xdma_channel_t *xchan, struct xdma_req
 
        switch (xr->req_type) {
        case XR_TYPE_MBUF:
-               seg[0].ds_addr = mtod(m, bus_addr_t);
+               if ((xchan->caps & XCHAN_CAP_NOBUFS) == 0) {
+                       if (xr->direction == XDMA_MEM_TO_DEV)
+                               m_copydata(m, 0, m->m_pkthdr.len,
+                                   (void *)xr->buf.vaddr);
+                       seg[0].ds_addr = (bus_addr_t)xr->buf.paddr;
+               } else
+                       seg[0].ds_addr = mtod(m, bus_addr_t);
                seg[0].ds_len = m->m_pkthdr.len;
                break;
        case XR_TYPE_BIO:
@@ -494,6 +553,7 @@ xdma_process(xdma_channel_t *xchan,
        xdma = xchan->xdma;
 
        n = 0;
+       c = 0;
 
        ret = XDMA_CHANNEL_CAPACITY(xdma->dma_dev, xchan, &capacity);
        if (ret != 0) {
_______________________________________________
svn-src-head@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-head
To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org"

Reply via email to