From: Matthew Brost <matthew.br...@intel.com>
Update GPU SVM and Xe to use two-pass MMU notifiers, enabling pipelined
TLB invalidations across VMs or multiple devices.
The driver-side (Xe) implementation is not yet implemented.
Signed-off-by: Matthew Brost <matthew.br...@intel.com>
---
drivers/gpu/drm/drm_gpusvm.c | 18 +++++++++++-------
drivers/gpu/drm/xe/xe_svm.c | 9 +++++----
include/drm/drm_gpusvm.h | 11 +++++++----
3 files changed, 23 insertions(+), 15 deletions(-)
diff --git a/drivers/gpu/drm/drm_gpusvm.c b/drivers/gpu/drm/drm_gpusvm.c
index 661306da6b2d..92dc7d2bd6cf 100644
--- a/drivers/gpu/drm/drm_gpusvm.c
+++ b/drivers/gpu/drm/drm_gpusvm.c
@@ -374,10 +374,13 @@ notifier_iter_first(struct rb_root_cached *root, unsigned
long start,
(notifier__) = (next__), (next__) =
__drm_gpusvm_notifier_next(notifier__))
/**
- * drm_gpusvm_notifier_invalidate() - Invalidate a GPU SVM notifier.
+ * drm_gpusvm_notifier_invalidate_twopass() - Invalidate a GPU SVM notifie,
+ * fist pass.
+ *
* @mni: Pointer to the mmu_interval_notifier structure.
* @mmu_range: Pointer to the mmu_notifier_range structure.
* @cur_seq: Current sequence number.
+ * @pass: First pass of MMU notifier
*
* This function serves as a generic MMU notifier for GPU SVM. It sets the MMU
* notifier sequence number and calls the driver invalidate vfunc under
@@ -386,9 +389,10 @@ notifier_iter_first(struct rb_root_cached *root, unsigned
long start,
* Return: true if the operation succeeds, false otherwise.
*/
static bool
-drm_gpusvm_notifier_invalidate(struct mmu_interval_notifier *mni,
- const struct mmu_notifier_range *mmu_range,
- unsigned long cur_seq)
+drm_gpusvm_notifier_invalidate_twopass(struct mmu_interval_notifier *mni,
+ const struct mmu_notifier_range
*mmu_range,
+ unsigned long cur_seq,
+ struct mmu_interval_notifier_pass **pass)
{
struct drm_gpusvm_notifier *notifier =
container_of(mni, typeof(*notifier), notifier);
@@ -399,7 +403,7 @@ drm_gpusvm_notifier_invalidate(struct mmu_interval_notifier
*mni,
down_write(&gpusvm->notifier_lock);
mmu_interval_set_seq(mni, cur_seq);
- gpusvm->ops->invalidate(gpusvm, notifier, mmu_range);
+ gpusvm->ops->invalidate_twopass(gpusvm, notifier, mmu_range, pass);
up_write(&gpusvm->notifier_lock);
return true;
@@ -409,7 +413,7 @@ drm_gpusvm_notifier_invalidate(struct mmu_interval_notifier
*mni,
* drm_gpusvm_notifier_ops - MMU interval notifier operations for GPU SVM
*/
static const struct mmu_interval_notifier_ops drm_gpusvm_notifier_ops = {
- .invalidate = drm_gpusvm_notifier_invalidate,
+ .invalidate_twopass = drm_gpusvm_notifier_invalidate_twopass,
};
/**
@@ -440,7 +444,7 @@ int drm_gpusvm_init(struct drm_gpusvm *gpusvm,
const struct drm_gpusvm_ops *ops,
const unsigned long *chunk_sizes, int num_chunks)
{
- if (!ops->invalidate || !num_chunks)
+ if (!ops->invalidate_twopass || !num_chunks)
return -EINVAL;
gpusvm->name = name;
diff --git a/drivers/gpu/drm/xe/xe_svm.c b/drivers/gpu/drm/xe/xe_svm.c
index e35c6d4def20..23c5b363261c 100644
--- a/drivers/gpu/drm/xe/xe_svm.c
+++ b/drivers/gpu/drm/xe/xe_svm.c
@@ -171,9 +171,10 @@ xe_svm_range_notifier_event_end(struct xe_vm *vm, struct
drm_gpusvm_range *r,
mmu_range);
}
-static void xe_svm_invalidate(struct drm_gpusvm *gpusvm,
- struct drm_gpusvm_notifier *notifier,
- const struct mmu_notifier_range *mmu_range)
+static void xe_svm_invalidate_twopass(struct drm_gpusvm *gpusvm,
+ struct drm_gpusvm_notifier *notifier,
+ const struct mmu_notifier_range
*mmu_range,
+ struct mmu_interval_notifier_pass **p)
{
struct xe_vm *vm = gpusvm_to_vm(gpusvm);
struct xe_device *xe = vm->xe;
@@ -553,7 +554,7 @@ static const struct drm_pagemap_devmem_ops
dpagemap_devmem_ops = {
static const struct drm_gpusvm_ops gpusvm_ops = {
.range_alloc = xe_svm_range_alloc,
.range_free = xe_svm_range_free,
- .invalidate = xe_svm_invalidate,
+ .invalidate_twopass = xe_svm_invalidate_twopass,
};
static const unsigned long fault_chunk_sizes[] = {
diff --git a/include/drm/drm_gpusvm.h b/include/drm/drm_gpusvm.h
index 8d613e9b2690..8b5e159857fc 100644
--- a/include/drm/drm_gpusvm.h
+++ b/include/drm/drm_gpusvm.h
@@ -63,17 +63,20 @@ struct drm_gpusvm_ops {
void (*range_free)(struct drm_gpusvm_range *range);
/**
- * @invalidate: Invalidate GPU SVM notifier (required)
+ * @invalidate_twopass: Invalidate first pass GPU SVM notifier
(required)
* @gpusvm: Pointer to the GPU SVM
* @notifier: Pointer to the GPU SVM notifier
* @mmu_range: Pointer to the mmu_notifier_range structure
+ * @pass: Pass of MMU notifier, optionally populated driver side
+ * if a second pass of MMU notifier is desired
*
* Invalidate the GPU page tables. It can safely walk the notifier range
* RB tree/list in this function. Called while holding the notifier
lock.
*/
- void (*invalidate)(struct drm_gpusvm *gpusvm,
- struct drm_gpusvm_notifier *notifier,
- const struct mmu_notifier_range *mmu_range);
+ void (*invalidate_twopass)(struct drm_gpusvm *gpusvm,
+ struct drm_gpusvm_notifier *notifier,
+ const struct mmu_notifier_range *mmu_range,
+ struct mmu_interval_notifier_pass **pass);
};
/**
--
2.50.1
