Hi Sergio, On Friday 14 July 2017 01:13 PM, Sergio Gonzalez Monroy wrote:
> On 13/07/2017 09:23, santosh wrote: >> Hi Sergio, >> >> On Wednesday 12 July 2017 01:50 PM, Sergio Gonzalez Monroy wrote: >> >>> On 11/07/2017 07:16, Santosh Shukla wrote: >>>> Get iommu class of PCI device on the bus and returns preferred iova >>>> mapping mode for that bus. >>>> >>>> Algorithm for iova scheme selection for PCI bus: >>>> 0. Look for device attached to vfio kdrv and has .drv_flag set >>>> to RTE_PCI_DRV_NEED_IOVA_VA. >>>> 1. Look for any device attached to UIO class of driver. >>>> 2. Check for vfio-noiommu mode enabled. >>>> >>>> If 1) & 2) is false and 0) is true then select >>>> mapping scheme as iova=va. Otherwise use default >>>> mapping scheme (iova_pa). >>>> >>>> Signed-off-by: Santosh Shukla <santosh.shu...@caviumnetworks.com> >>>> Signed-off-by: Jerin Jacob <jerin.ja...@caviumnetworks.com> >>>> --- >>>> v1 --> v2: >>>> - Removed Linux version check in vfio_noiommu func. Refer [1]. >>>> - Extending autodetction logic for _iommu_class. >>>> Refer [2]. >>>> >>>> [1] https://www.mail-archive.com/dev@dpdk.org/msg70108.html >>>> [2] https://www.mail-archive.com/dev@dpdk.org/msg70279.html >>> Just wondering how it all works with device hotplug. >>> Correct me if I am wrong but if EAL decides to use IOVA_AS_VA scheme, >>> if we were to attach a device that needs IOVA_AS_PA, it will not work and >>> should fail to attach, right? >>> >> It will work for igb_uio case, and won't work for vfio-noiommu hotplug >> case(Invalid case). > > Why are those two cases (igb_uio, vfio-noiommu) different? do they not have > the same requirements, ie. need IOVA_PA sheme? > Behavior remains same. For vfio-noiommu case in the context of hot-plugging - Rest of the VFIO(/iommu) devices will be functionally effected thats why mentioned invalid case. > Thanks, > Sergio > >> Yes, we can dictate iova awareness to hotplug/unplug area. >> >>> Thanks, >>> Sergio >>> >>>> lib/librte_eal/linuxapp/eal/eal_pci.c | 66 >>>> +++++++++++++++++++++++++ >>>> lib/librte_eal/linuxapp/eal/eal_vfio.c | 19 +++++++ >>>> lib/librte_eal/linuxapp/eal/eal_vfio.h | 4 ++ >>>> lib/librte_eal/linuxapp/eal/rte_eal_version.map | 1 + >>>> 4 files changed, 90 insertions(+) >>>> >>>> diff --git a/lib/librte_eal/linuxapp/eal/eal_pci.c >>>> b/lib/librte_eal/linuxapp/eal/eal_pci.c >>>> index 7d9e1a99b..573caa000 100644 >>>> --- a/lib/librte_eal/linuxapp/eal/eal_pci.c >>>> +++ b/lib/librte_eal/linuxapp/eal/eal_pci.c >>>> @@ -45,6 +45,7 @@ >>>> #include "eal_filesystem.h" >>>> #include "eal_private.h" >>>> #include "eal_pci_init.h" >>>> +#include "eal_vfio.h" >>>> /** >>>> * @file >>>> @@ -488,6 +489,71 @@ rte_pci_scan(void) >>>> return -1; >>>> } >>>> +/* >>>> + * Any one of the device bound to uio >>>> + */ >>>> +static inline int >>>> +pci_device_bound_uio(void) >>>> +{ >>>> + struct rte_pci_device *dev = NULL; >>>> + >>>> + FOREACH_DEVICE_ON_PCIBUS(dev) { >>>> + if (dev->kdrv == RTE_KDRV_IGB_UIO || >>>> + dev->kdrv == RTE_KDRV_UIO_GENERIC) { >>>> + return 1; >>>> + } >>>> + } >>>> + return 0; >>>> +} >>>> + >>>> +/* >>>> + * Any one of the device has iova as va >>>> + */ >>>> +static inline int >>>> +pci_device_has_iova_va(void) >>>> +{ >>>> + struct rte_pci_device *dev = NULL; >>>> + struct rte_pci_driver *drv = NULL; >>>> + >>>> + FOREACH_DRIVER_ON_PCIBUS(drv) { >>>> + if (drv && drv->drv_flags & RTE_PCI_DRV_NEED_IOVA_VA) { >>>> + FOREACH_DEVICE_ON_PCIBUS(dev) { >>>> + if (dev->kdrv == RTE_KDRV_VFIO && >>>> + rte_pci_match(drv, dev)) >>>> + return 1; >>>> + } >>>> + } >>>> + } >>>> + return 0; >>>> +} >>>> + >>>> +/* >>>> + * Get iommu class of PCI devices on the bus. >>>> + */ >>>> +enum rte_iova_mode >>>> +rte_pci_get_iommu_class(void) >>>> +{ >>>> + bool is_vfio_noiommu_enabled; >>>> + bool has_iova_va; >>>> + bool is_bound_uio; >>>> + >>>> + has_iova_va = pci_device_has_iova_va(); >>>> + is_bound_uio = pci_device_bound_uio(); >>>> + is_vfio_noiommu_enabled = vfio_noiommu_is_enabled() == 1 ? 1 : 0; >>>> + >>>> + if (has_iova_va && !is_bound_uio && !is_vfio_noiommu_enabled) >>>> + return RTE_IOVA_VA; >>>> + >>>> + if (has_iova_va) { >>>> + if (is_vfio_noiommu_enabled) >>>> + RTE_LOG(WARNING, EAL, "vfio-noiommu mode configured\n"); >>>> + if (is_bound_uio) >>>> + RTE_LOG(WARNING, EAL, "Some device attached to UIO\n"); >>>> + } >>>> + >>>> + return RTE_IOVA_PA; >>>> +} >>>> + >>>> /* Read PCI config space. */ >>>> int rte_pci_read_config(const struct rte_pci_device *device, >>>> void *buf, size_t len, off_t offset) >>>> diff --git a/lib/librte_eal/linuxapp/eal/eal_vfio.c >>>> b/lib/librte_eal/linuxapp/eal/eal_vfio.c >>>> index 946df7e31..c8a97b7e7 100644 >>>> --- a/lib/librte_eal/linuxapp/eal/eal_vfio.c >>>> +++ b/lib/librte_eal/linuxapp/eal/eal_vfio.c >>>> @@ -816,4 +816,23 @@ vfio_noiommu_dma_map(int __rte_unused >>>> vfio_container_fd) >>>> return 0; >>>> } >>>> +int >>>> +vfio_noiommu_is_enabled(void) >>>> +{ >>>> + int fd, ret, cnt __rte_unused; >>>> + char c; >>>> + >>>> + ret = -1; >>>> + fd = open(VFIO_NOIOMMU_MODE, O_RDONLY); >>>> + if (fd < 0) >>>> + return -1; >>>> + >>>> + cnt = read(fd, &c, 1); >>>> + if (c == 'Y') >>>> + ret = 1; >>>> + >>>> + close(fd); >>>> + return ret; >>>> +} >>>> + >>>> #endif >>>> diff --git a/lib/librte_eal/linuxapp/eal/eal_vfio.h >>>> b/lib/librte_eal/linuxapp/eal/eal_vfio.h >>>> index 5ff63e5d7..26ea8e119 100644 >>>> --- a/lib/librte_eal/linuxapp/eal/eal_vfio.h >>>> +++ b/lib/librte_eal/linuxapp/eal/eal_vfio.h >>>> @@ -150,6 +150,8 @@ struct vfio_config { >>>> #define VFIO_NOIOMMU_GROUP_FMT "/dev/vfio/noiommu-%u" >>>> #define VFIO_GET_REGION_ADDR(x) ((uint64_t) x << 40ULL) >>>> #define VFIO_GET_REGION_IDX(x) (x >> 40) >>>> +#define VFIO_NOIOMMU_MODE \ >>>> + "/sys/module/vfio/parameters/enable_unsafe_noiommu_mode" >>>> /* DMA mapping function prototype. >>>> * Takes VFIO container fd as a parameter. >>>> @@ -210,6 +212,8 @@ int pci_vfio_is_enabled(void); >>>> int vfio_mp_sync_setup(void); >>>> +int vfio_noiommu_is_enabled(void); >>>> + >>>> #define SOCKET_REQ_CONTAINER 0x100 >>>> #define SOCKET_REQ_GROUP 0x200 >>>> #define SOCKET_CLR_GROUP 0x300 >>>> diff --git a/lib/librte_eal/linuxapp/eal/rte_eal_version.map >>>> b/lib/librte_eal/linuxapp/eal/rte_eal_version.map >>>> index c91dd44c4..044f89c7c 100644 >>>> --- a/lib/librte_eal/linuxapp/eal/rte_eal_version.map >>>> +++ b/lib/librte_eal/linuxapp/eal/rte_eal_version.map >>>> @@ -206,6 +206,7 @@ DPDK_17.08 { >>>> rte_bus_find_by_device; >>>> rte_bus_find_by_name; >>>> rte_pci_match; >>>> + rte_pci_get_iommu_class; >>>> } DPDK_17.05; >>>> >>> >
