Convert the DRM scheduler to use kthread_work instead of a workqueue for job submission.
Goal is to build a priority inversion avoidance scheme on top of the kthread_work, and by doing so improve worst case latency of elevated priority clients such as compositors ie. improve the smoothness of the user interface. Or in other words, avoid an effective priority inversion where high priority clients have their work processed by a normal priority workqueues which compete with random background tasks. As some DRM scheduler users create per userspace client workqueues we implement a trivial round-robin kthread_worker pool with a small maximum which can be improved upon later. Signed-off-by: Tvrtko Ursulin <[email protected]> Cc: Danilo Krummrich <[email protected]> Cc: Matthew Brost <[email protected]> Cc: Philipp Stanner <[email protected]> --- drivers/gpu/drm/scheduler/sched_main.c | 80 +++++++++++++++++++++++++- include/drm/gpu_scheduler.h | 75 +++++++++++++++++++----- 2 files changed, 140 insertions(+), 15 deletions(-) diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c index 1e728838a4a2..fff908d0f4c7 100644 --- a/drivers/gpu/drm/scheduler/sched_main.c +++ b/drivers/gpu/drm/scheduler/sched_main.c @@ -1246,7 +1246,85 @@ EXPORT_SYMBOL(drm_sched_job_is_signaled); void drm_sched_destroy_worker(struct drm_sched_worker *worker) { - destroy_workqueue(worker->worker); + unsigned int i; + + for (i = 0; i < worker->num; i++) { + if (worker->worker[i]) + kthread_destroy_worker(worker->worker[i]); + } kfree(worker); } EXPORT_SYMBOL(drm_sched_destroy_worker); + +static void drm_sched_kworkfn(struct kthread_work *work_) +{ + struct drm_sched_work *work = + container_of(work_, typeof(*work), work); + unsigned long flags; + + BUILD_BUG_ON(offsetof(struct drm_sched_work, work)); + + spin_lock_irqsave(&work->lock, flags); + work->state = DRM_SCHED_WORK_RUNNING; + spin_unlock_irqrestore(&work->lock, flags); + + work->func(work); + + spin_lock_irqsave(&work->lock, flags); + switch (work->state) { + case DRM_SCHED_WORK_MIGRATING: + /* + * drm_sched_queue_work re-quested a re-queue while we were + * running, and has potentially changed the assigned worker so + * lets handle it. + */ + work->state = DRM_SCHED_WORK_QUEUED; + kthread_init_work(&work->work, drm_sched_kworkfn); + kthread_queue_work(work->worker, &work->work); + break; + default: + work->state = DRM_SCHED_WORK_IDLE; + }; + spin_unlock_irqrestore(&work->lock, flags); +} + +void drm_sched_queue_work(struct drm_sched_worker *worker, + struct drm_sched_work *work) +{ + struct kthread_worker *kw; + unsigned long flags; + unsigned int i = 0; + + if (worker->num > 1) + i = atomic_inc_return(&worker->last) & (worker->num - 1); + + kw = worker->worker[i]; + + spin_lock_irqsave(&work->lock, flags); + switch (work->state) { + case DRM_SCHED_WORK_IDLE: + work->worker = kw; + work->state = DRM_SCHED_WORK_QUEUED; + kthread_init_work(&work->work, drm_sched_kworkfn); + kthread_queue_work(work->worker, &work->work); + break; + case DRM_SCHED_WORK_RUNNING: + /* + * We must ensure only once instance of the work is running so + * flag to change the worker and self re-queue from the worker + * when it is safe to do so. + */ + work->worker = kw; + work->state = DRM_SCHED_WORK_MIGRATING; + break; + case DRM_SCHED_WORK_QUEUED: + case DRM_SCHED_WORK_MIGRATING: + /* + * Do nothing if the work is already queued but not started yet, + * or if self re-queue was already marked. + */ + break; + }; + spin_unlock_irqrestore(&work->lock, flags); +} +EXPORT_SYMBOL(drm_sched_queue_work); diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h index 391663360426..37e824d9483e 100644 --- a/include/drm/gpu_scheduler.h +++ b/include/drm/gpu_scheduler.h @@ -28,6 +28,7 @@ #include <linux/average.h> #include <linux/dma-fence.h> #include <linux/completion.h> +#include <linux/kthread.h> #include <linux/xarray.h> #include <linux/workqueue.h> @@ -78,17 +79,34 @@ enum drm_sched_priority { struct drm_sched_entity_stats; struct drm_sched_worker { - struct workqueue_struct *worker; + unsigned int num; + atomic_t last; + struct kthread_worker *worker[4]; }; -#define drm_sched_work work_struct +enum drm_sched_work_state { + DRM_SCHED_WORK_IDLE = 0, + DRM_SCHED_WORK_QUEUED, + DRM_SCHED_WORK_RUNNING, + DRM_SCHED_WORK_MIGRATING, +}; + +struct drm_sched_work { + struct kthread_work work; + spinlock_t lock; + enum drm_sched_work_state state; + struct kthread_worker *worker; + void (*func)(struct drm_sched_work *); +}; #define drm_sched_create_ordered_worker(fmt, ...) \ ({ \ struct drm_sched_worker *worker_; \ - worker_ = kmalloc_obj(typeof(*worker_)); \ + worker_ = kzalloc_obj(typeof(*worker_)); \ if (worker_) { \ - worker_->worker = alloc_ordered_workqueue((fmt), WQ_MEM_RECLAIM, ## __VA_ARGS__); \ + atomic_set(&worker_->last, 0); \ + worker_->num = 1; \ + worker_->worker[0] = kthread_run_worker(PF_MEMALLOC, (fmt), ## __VA_ARGS__); \ if (IS_ERR(worker_->worker)) { \ kfree(worker_); \ worker_ = ERR_CAST(worker_->worker); \ @@ -102,12 +120,22 @@ struct drm_sched_worker { #define drm_sched_create_concurrent_worker(fmt, ...) \ ({ \ struct drm_sched_worker *worker_; \ - worker_ = kmalloc_obj(typeof(*worker_)); \ + worker_ = kzalloc_obj(typeof(*worker_)); \ if (worker_) { \ - worker_->worker = alloc_workqueue((fmt), WQ_UNBOUND, 0, ## __VA_ARGS__); \ - if (IS_ERR(worker_->worker)) { \ - kfree(worker_); \ - worker_ = ERR_CAST(worker_->worker); \ + size_t i_, cpus_; \ + atomic_set(&worker_->last, 0); \ + cpus_ = num_possible_cpus(); \ + BUILD_BUG_ON(!is_power_of_2(ARRAY_SIZE(worker_->worker))); \ + worker_->num = roundup_pow_of_two(min_t(unsigned int, cpus_, ARRAY_SIZE(worker_->worker))); \ + for (i_ = 0; i_ < worker_->num; i_++) { \ + worker_->worker[i_] = kthread_run_worker(0, (fmt), ## __VA_ARGS__); \ + if (IS_ERR(worker_->worker[i_])) { \ + int ret_ = PTR_ERR(worker_->worker[i_]); \ + while (i_ > 0) \ + kthread_destroy_worker(worker_->worker[--i_]); \ + kfree(worker_); \ + worker_ = ERR_PTR(ret_); \ + } \ } \ } else { \ worker_ = ERR_PTR(-ENOMEM); \ @@ -117,11 +145,30 @@ struct drm_sched_worker { void drm_sched_destroy_worker(struct drm_sched_worker *worker); -#define drm_sched_init_work INIT_WORK -#define drm_sched_queue_work(worker_, work_) \ - queue_work((struct workqueue_struct *)(worker_), (work_)) -#define drm_sched_flush_work flush_work -#define drm_sched_cancel_work_sync cancel_work_sync +void drm_sched_queue_work(struct drm_sched_worker *worker, + struct drm_sched_work *work); + +static inline void +drm_sched_init_work(struct drm_sched_work *work, + void (*func)(struct drm_sched_work *)) +{ + work->state = DRM_SCHED_WORK_IDLE; + work->worker = NULL; + work->func = func; + + spin_lock_init(&work->lock); +} + + +static inline void drm_sched_flush_work(struct drm_sched_work *work) +{ + kthread_flush_work(&work->work); +} + +static inline bool drm_sched_cancel_work_sync(struct drm_sched_work *work) +{ + return kthread_cancel_work_sync(&work->work); +} /** * struct drm_sched_entity - A wrapper around a job queue (typically -- 2.54.0
