[RFC PATCH 19/19] drm/xe: HAX: Use pcie p2p dma to test fast interconnect

[Thomas Hellström]

Knowing that this is not the correct way to support pcie_p2p over hmm
+ the dma api, pretend that pcie_p2p is a driver-private fast
interconnect to demonstrate how multi-device SVM can be done.

This has been used to test SVM on a BMG client with a pagemap
created on a DG1 GPU over pcie p2p.

Signed-off-by: Thomas Hellström <thomas.hellst...@linux.intel.com>
---
 drivers/gpu/drm/xe/xe_svm.c | 50 ++++++++++++++++++++++++++++++++++---
 drivers/gpu/drm/xe/xe_svm.h |  1 +
 2 files changed, 48 insertions(+), 3 deletions(-)

diff --git a/drivers/gpu/drm/xe/xe_svm.c b/drivers/gpu/drm/xe/xe_svm.c
index 56c2c731be27..0b562b411fa4 100644
--- a/drivers/gpu/drm/xe/xe_svm.c
+++ b/drivers/gpu/drm/xe/xe_svm.c
@@ -3,6 +3,8 @@
  * Copyright © 2024 Intel Corporation
  */
 
+#include <linux/pci-p2pdma.h>
+
 #include <drm/drm_drv.h>
 #include <drm/drm_managed.h>
 #include <drm/drm_pagemap.h>
@@ -379,6 +381,25 @@ static u64 xe_page_to_dpa(struct page *page)
        return dpa;
 }
 
+static u64 xe_page_to_pcie(struct page *page)
+{
+       struct xe_pagemap *xpagemap = xe_page_to_pagemap(page);
+       struct xe_vram_region *vr = xe_pagemap_to_vr(xpagemap);
+       u64 hpa_base = xpagemap->hpa_base;
+       u64 ioaddr;
+       struct xe_tile *tile = xe_vr_to_tile(vr);
+       u64 pfn = page_to_pfn(page);
+       u64 offset;
+
+       xe_tile_assert(tile, is_device_private_page(page));
+       xe_tile_assert(tile, (pfn << PAGE_SHIFT) >= hpa_base);
+
+       offset = (pfn << PAGE_SHIFT) - hpa_base;
+       ioaddr = vr->io_start + offset;
+
+       return ioaddr;
+}
+
 enum xe_svm_copy_dir {
        XE_SVM_COPY_TO_VRAM,
        XE_SVM_COPY_TO_SRAM,
@@ -940,13 +961,27 @@ xe_drm_pagemap_device_map(struct drm_pagemap *dpagemap,
                addr = xe_page_to_dpa(page);
                prot = XE_INTERCONNECT_VRAM;
        } else {
-               addr = DMA_MAPPING_ERROR;
-               prot = 0;
+               addr = dma_map_resource(dev,
+                                       xe_page_to_pcie(page),
+                                       PAGE_SIZE << order, dir,
+                                       DMA_ATTR_SKIP_CPU_SYNC);
+               prot = XE_INTERCONNECT_P2P;
        }
 
        return drm_pagemap_device_addr_encode(addr, prot, order, dir);
 }
 
+static void xe_drm_pagemap_device_unmap(struct drm_pagemap *dpagemap,
+                                       struct device *dev,
+                                       struct drm_pagemap_device_addr addr)
+{
+       if (addr.proto != XE_INTERCONNECT_P2P)
+               return;
+
+       dma_unmap_resource(dev, addr.addr, PAGE_SIZE << addr.order,
+                          addr.dir, DMA_ATTR_SKIP_CPU_SYNC);
+}
+
 static void xe_pagemap_fini(struct xe_pagemap *xpagemap)
 {
        struct dev_pagemap *pagemap = &xpagemap->pagemap;
@@ -1004,13 +1039,22 @@ static bool xe_has_interconnect(struct drm_pagemap_peer 
*peer1,
        struct device *dev1 = xpagemap1->dpagemap.drm->dev;
        struct device *dev2 = xpagemap2->dpagemap.drm->dev;
 
-       return dev1 == dev2;
+       if (dev1 == dev2)
+               return true;
+
+       /* Define this if your system can correctly identify pci_p2p capability 
*/
+#ifdef XE_P2P_CAPABLE
+       return pci_p2pdma_distance(to_pci_dev(dev1), dev2, true) >= 0;
+#else
+       return true;
+#endif
 }
 
 static DRM_PAGEMAP_OWNER_LIST_DEFINE(xe_owner_list);
 
 static const struct drm_pagemap_ops xe_drm_pagemap_ops = {
        .device_map = xe_drm_pagemap_device_map,
+       .device_unmap = xe_drm_pagemap_device_unmap,
        .populate_mm = xe_drm_pagemap_populate_mm,
        .destroy = xe_pagemap_destroy,
 };
diff --git a/drivers/gpu/drm/xe/xe_svm.h b/drivers/gpu/drm/xe/xe_svm.h
index 7c076c36c1c5..59b7a46f2bd9 100644
--- a/drivers/gpu/drm/xe/xe_svm.h
+++ b/drivers/gpu/drm/xe/xe_svm.h
@@ -13,6 +13,7 @@
 #include <drm/drm_pagemap_util.h>
 
 #define XE_INTERCONNECT_VRAM DRM_INTERCONNECT_DRIVER
+#define XE_INTERCONNECT_P2P (XE_INTERCONNECT_VRAM + 1)
 
 struct drm_device;
 struct drm_file;
-- 
2.48.1