Hi, I have some doubts in registering different types of memory for dma using vfio. Currently I have 4.15 kernel and have attached P4800X to vfio. I followed the examples in the vfio kernel doc at (https://www.kernel.org/doc/Documentation/vfio.txt). Everything is fine with anonymous mapped memory in the example. I also tried to register two other types of memory:
1. Mapped from device dax. 2. Kernel memory memory exposed to userspace using remap_pfn_range. ``` switch (mem_type) { case MEM_TYPE_DEVDAX: // https://github.com/axboe/fio/blob/master/engines/dev-dax.c fd = open("/dev/dax0.0", O_RDWR, 0666); assert(fd != -1); ptr = mmap(NULL, map_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); assert(ptr != MAP_FAILED); // memset(ptr, 0xff, map_size); break; case MEM_TYPE_DUMMYMAP: fd = open("/dev/dummymap", O_RDWR, 0666); // see below for dummymap kernel module assert(fd != -1); ptr = mmap(target_addr, map_size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, fd, 0); assert(ptr != MAP_FAILED); // memset(ptr, 0xff, map_size); break; case MEM_TYPE_ANON_REGULAR: default: ptr = mmap(target_addr, map_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, 0, 0); assert(ptr != MAP_FAILED); } dma_map.vaddr = ptr; dma_map.size = map_size; dma_map.iova = 0x900000000; /* 1MB starting at 0x900000000 from device view */ dma_map.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE; if (0 > ioctl(container, VFIO_IOMMU_MAP_DMA, &dma_map)) { PERR("VFIO_IO_MMU_MAP_DMA failed with errno %s", strerror(errno)); return -1; } else { PINF("DMA memory setup succeed!"); } ``` The "dummy map" is a kernel module which allocates kernel memory and map it to user space using remap_pfn_range: ``` static int fop_mmap(struct file *file, struct vm_area_struct *vma) { size_t map_size; void *dma_virtaddr; // dma_addr_t dma_handle; map_size = vma->vm_end - vma->vm_start; printk(KERN_INFO "dummy map mmap called!\n"); /*dma_virtaddr = dma_alloc_coherent(dummymap_dev.this_device, map_size,*/ /*&dma_handle, GFP_KERNEL);*/ dma_virtaddr = kmalloc(map_size, GFP_KERNEL); if (!dma_virtaddr) { printk(KERN_ERR "dummy map failed in memory allocation"); return -ENOMEM; } vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); if (remap_pfn_range(vma, vma->vm_start, virt_to_phys(dma_virtaddr) >> PAGE_SHIFT, map_size, vma->vm_page_prot)) return -ENOMEM; vma->vm_ops = &dummymap_fops; vm_open(vma); pr_info("dummy map succeed"); return 0; } ``` What happened is 1. the "devdax" registration seems to succeed. I will do further tests on actual DMA. (I hope I am doing the right thing, am I?) 2. The "dummy map" approach failed at the VFIO_IOMMU_MAP_DMA ioctl (with errno = -14), I guess it violated some memory requirements for DMA using vfio, does anyone have any suggestions on this?. I tracked into the kernel source and found out that the vaddr_get_pfn requires Pg_reserved bit set for VM_PFNMAP mappings(which is set from remap_pfn_range)... So I guess I needs somehow allocate memory with Pg_reserved set, but how? I also left my code in https://github.com/fengggli/dumymap/commit/c562cd863ec0f6d921a7ae42ecfad0dbfc9d9d0b , if anyone wants to have a try. Thanks Feng Li
_______________________________________________ vfio-users mailing list vfio-users@redhat.com https://www.redhat.com/mailman/listinfo/vfio-users