Hi,
I have some doubts in registering different types of memory for dma using vfio.
Currently I have 4.15 kernel and have attached P4800X to vfio.
I followed the examples in the vfio kernel doc at
(https://www.kernel.org/doc/Documentation/vfio.txt). Everything is fine with
anonymous mapped memory in the example. I also tried to register two other
types of memory:
1. Mapped from device dax.
2. Kernel memory memory exposed to userspace using remap_pfn_range.
```
switch (mem_type) {
case MEM_TYPE_DEVDAX:
// https://github.com/axboe/fio/blob/master/engines/dev-dax.c
fd = open("/dev/dax0.0", O_RDWR, 0666);
assert(fd != -1);
ptr = mmap(NULL, map_size, PROT_READ | PROT_WRITE,
MAP_SHARED, fd, 0);
assert(ptr != MAP_FAILED);
// memset(ptr, 0xff, map_size);
break;
case MEM_TYPE_DUMMYMAP:
fd = open("/dev/dummymap", O_RDWR, 0666); // see below for dummymap kernel
module
assert(fd != -1);
ptr = mmap(target_addr, map_size, PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_FIXED, fd, 0);
assert(ptr != MAP_FAILED);
// memset(ptr, 0xff, map_size);
break;
case MEM_TYPE_ANON_REGULAR:
default:
ptr = mmap(target_addr, map_size, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, 0, 0);
assert(ptr != MAP_FAILED);
}
dma_map.vaddr = ptr;
dma_map.size = map_size;
dma_map.iova = 0x900000000; /* 1MB starting at 0x900000000 from device view */
dma_map.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE;
if (0 > ioctl(container, VFIO_IOMMU_MAP_DMA, &dma_map)) {
PERR("VFIO_IO_MMU_MAP_DMA failed with errno %s", strerror(errno));
return -1;
} else {
PINF("DMA memory setup succeed!");
}
```
The "dummy map" is a kernel module which allocates kernel memory and map it to
user space using remap_pfn_range:
```
static int fop_mmap(struct file *file, struct vm_area_struct *vma) {
size_t map_size;
void *dma_virtaddr;
// dma_addr_t dma_handle;
map_size = vma->vm_end - vma->vm_start;
printk(KERN_INFO "dummy map mmap called!\n");
/*dma_virtaddr = dma_alloc_coherent(dummymap_dev.this_device, map_size,*/
/*&dma_handle, GFP_KERNEL);*/
dma_virtaddr = kmalloc(map_size, GFP_KERNEL);
if (!dma_virtaddr) {
printk(KERN_ERR "dummy map failed in memory allocation");
return -ENOMEM;
}
vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
if (remap_pfn_range(vma, vma->vm_start,
virt_to_phys(dma_virtaddr) >> PAGE_SHIFT, map_size,
vma->vm_page_prot))
return -ENOMEM;
vma->vm_ops = &dummymap_fops;
vm_open(vma);
pr_info("dummy map succeed");
return 0;
}
```
What happened is
1. the "devdax" registration seems to succeed. I will do further tests on
actual DMA. (I hope I am doing the right thing, am I?)
2. The "dummy map" approach failed at the VFIO_IOMMU_MAP_DMA ioctl (with
errno = -14), I guess it violated some memory requirements for DMA using vfio,
does anyone have any suggestions on this?.
I tracked into the kernel source and found out that the vaddr_get_pfn requires
Pg_reserved bit set for VM_PFNMAP mappings(which is set from
remap_pfn_range)... So I guess I needs somehow allocate memory with Pg_reserved
set, but how?
I also left my code in
https://github.com/fengggli/dumymap/commit/c562cd863ec0f6d921a7ae42ecfad0dbfc9d9d0b
, if anyone wants to have a try.
Thanks
Feng Li
_______________________________________________
vfio-users mailing list
vfio-users@redhat.com
https://www.redhat.com/mailman/listinfo/vfio-users
