Xu Yilun <yilun...@linux.intel.com> writes: > When device is TSM Bound, some of its MMIO regions are controlled by > secure firmware. E.g. TDX Connect would require these MMIO regions > mappeed in S-EPT and never unmapped until device Unbound. Zapping bars > irrespective of TSM Bound state may cause unexpected secure firmware > errors. It is always safe to do TSM Unbind first, transiting the device > to shared, then do whatever needed as before. > > Signed-off-by: Xu Yilun <yilun...@linux.intel.com> > --- > drivers/vfio/pci/vfio_pci_config.c | 4 +++ > drivers/vfio/pci/vfio_pci_core.c | 41 +++++++++++++++++++----------- > drivers/vfio/pci/vfio_pci_priv.h | 3 +++ > 3 files changed, 33 insertions(+), 15 deletions(-) > > diff --git a/drivers/vfio/pci/vfio_pci_config.c > b/drivers/vfio/pci/vfio_pci_config.c > index 7ac062bd5044..4ffe661c9e59 100644 > --- a/drivers/vfio/pci/vfio_pci_config.c > +++ b/drivers/vfio/pci/vfio_pci_config.c > @@ -590,6 +590,7 @@ static int vfio_basic_config_write(struct > vfio_pci_core_device *vdev, int pos, > new_mem = !!(new_cmd & PCI_COMMAND_MEMORY); > > if (!new_mem) { > + vfio_pci_tsm_unbind(vdev); > vfio_pci_zap_and_down_write_memory_lock(vdev); > vfio_pci_dma_buf_move(vdev, true); >
Don't we need to re-bind the vdev with tsm_bind for the continued use of TDI? > } else { > @@ -712,6 +713,7 @@ static void vfio_lock_and_set_power_state(struct > vfio_pci_core_device *vdev, > pci_power_t state) > { > if (state >= PCI_D3hot) { > + vfio_pci_tsm_unbind(vdev); > vfio_pci_zap_and_down_write_memory_lock(vdev); > vfio_pci_dma_buf_move(vdev, true); > } else { > @@ -907,6 +909,7 @@ static int vfio_exp_config_write(struct > vfio_pci_core_device *vdev, int pos, > &cap); > > if (!ret && (cap & PCI_EXP_DEVCAP_FLR)) { > + vfio_pci_tsm_unbind(vdev); > vfio_pci_zap_and_down_write_memory_lock(vdev); > vfio_pci_dma_buf_move(vdev, true); > pci_try_reset_function(vdev->pdev); > @@ -992,6 +995,7 @@ static int vfio_af_config_write(struct > vfio_pci_core_device *vdev, int pos, > &cap); > > if (!ret && (cap & PCI_AF_CAP_FLR) && (cap & PCI_AF_CAP_TP)) { > + vfio_pci_tsm_unbind(vdev); > vfio_pci_zap_and_down_write_memory_lock(vdev); > vfio_pci_dma_buf_move(vdev, true); > pci_try_reset_function(vdev->pdev); > diff --git a/drivers/vfio/pci/vfio_pci_core.c > b/drivers/vfio/pci/vfio_pci_core.c > index 92544e54c9c3..a8437fcecca1 100644 > --- a/drivers/vfio/pci/vfio_pci_core.c > +++ b/drivers/vfio/pci/vfio_pci_core.c > @@ -286,6 +286,7 @@ static int vfio_pci_runtime_pm_entry(struct > vfio_pci_core_device *vdev, > * The vdev power related flags are protected with 'memory_lock' > * semaphore. > */ > + vfio_pci_tsm_unbind(vdev); > vfio_pci_zap_and_down_write_memory_lock(vdev); > vfio_pci_dma_buf_move(vdev, true); > > @@ -693,11 +694,7 @@ void vfio_pci_core_close_device(struct vfio_device > *core_vdev) > eeh_dev_release(vdev->pdev); > #endif > > - if (vdev->is_tsm_bound) { > - vfio_iommufd_tsm_unbind(&vdev->vdev); > - pci_release_regions(vdev->pdev); > - vdev->is_tsm_bound = false; > - } > + __vfio_pci_tsm_unbind(vdev); > > vfio_pci_core_disable(vdev); > > @@ -1222,6 +1219,7 @@ static int vfio_pci_ioctl_reset(struct > vfio_pci_core_device *vdev, > if (!vdev->reset_works) > return -EINVAL; > > + vfio_pci_tsm_unbind(vdev); > vfio_pci_zap_and_down_write_memory_lock(vdev); > > /* > @@ -1491,12 +1489,32 @@ static int vfio_pci_ioctl_tsm_bind(struct > vfio_pci_core_device *vdev, > return ret; > } > > +void __vfio_pci_tsm_unbind(struct vfio_pci_core_device *vdev) > +{ > + struct pci_dev *pdev = vdev->pdev; > + > + lockdep_assert_held(&vdev->vdev.dev_set->lock); > + > + if (!vdev->is_tsm_bound) > + return; > + > + vfio_iommufd_tsm_unbind(&vdev->vdev); > + pci_release_regions(pdev); > + vdev->is_tsm_bound = false; > Do we really need to check vdev->is_tsm_bound? The tsm_ops lock already ensures that concurrent TSM operations can't happen, and repeated calls to bind()/unbind() seem to be handled safely by pci_tsm_bind and pci_tsm_unbind. > +} > + > +void vfio_pci_tsm_unbind(struct vfio_pci_core_device *vdev) > +{ > + mutex_lock(&vdev->vdev.dev_set->lock); > + __vfio_pci_tsm_unbind(vdev); > + mutex_unlock(&vdev->vdev.dev_set->lock); > +} > If is_tsm_bound is no longer needed, and pci_release_regions / request_region_exclusive are now handled within pci_tsm_unbind / bind, do we still need mutex_lock() to guard this path? > + > static int vfio_pci_ioctl_tsm_unbind(struct vfio_pci_core_device *vdev, > void __user *arg) > { > unsigned long minsz = offsetofend(struct vfio_pci_tsm_unbind, flags); > struct vfio_pci_tsm_unbind tsm_unbind; > - struct pci_dev *pdev = vdev->pdev; > > if (copy_from_user(&tsm_unbind, arg, minsz)) > return -EFAULT; > @@ -1504,15 +1522,7 @@ static int vfio_pci_ioctl_tsm_unbind(struct > vfio_pci_core_device *vdev, > if (tsm_unbind.argsz < minsz || tsm_unbind.flags) > return -EINVAL; > > - mutex_lock(&vdev->vdev.dev_set->lock); > - > - if (!vdev->is_tsm_bound) > - return 0; > - > - vfio_iommufd_tsm_unbind(&vdev->vdev); > - pci_release_regions(pdev); > - vdev->is_tsm_bound = false; > - mutex_unlock(&vdev->vdev.dev_set->lock); > + vfio_pci_tsm_unbind(vdev); > > return 0; > } > @@ -2526,6 +2536,7 @@ static int vfio_pci_dev_set_hot_reset(struct > vfio_device_set *dev_set, > break; > } > > + __vfio_pci_tsm_unbind(vdev); > /* > * Take the memory write lock for each device and zap BAR > * mappings to prevent the user accessing the device while in > diff --git a/drivers/vfio/pci/vfio_pci_priv.h > b/drivers/vfio/pci/vfio_pci_priv.h > index 6f3e8eafdc35..e5bf27f46a73 100644 > --- a/drivers/vfio/pci/vfio_pci_priv.h > +++ b/drivers/vfio/pci/vfio_pci_priv.h > @@ -130,4 +130,7 @@ static inline void vfio_pci_dma_buf_move(struct > vfio_pci_core_device *vdev, > } > #endif > > +void __vfio_pci_tsm_unbind(struct vfio_pci_core_device *vdev); > +void vfio_pci_tsm_unbind(struct vfio_pci_core_device *vdev); > + > #endif > -- > 2.25.1