From: Hao Xiang <hao.xi...@linux.dev>
* Use a safe thread queue for DSA task enqueue/dequeue.
* Implement DSA task submission.
* Implement DSA batch task submission.
Signed-off-by: Hao Xiang <hao.xi...@linux.dev>
Signed-off-by: Yichen Wang <yichen.w...@bytedance.com>
Reviewed-by: Fabiano Rosas <faro...@suse.de>
---
include/qemu/dsa.h | 29 +++++++
util/dsa.c | 186 ++++++++++++++++++++++++++++++++++++++++++++-
2 files changed, 214 insertions(+), 1 deletion(-)
diff --git a/include/qemu/dsa.h b/include/qemu/dsa.h
index fbf0dcd692..a32b75c387 100644
--- a/include/qemu/dsa.h
+++ b/include/qemu/dsa.h
@@ -27,6 +27,17 @@
#include <linux/idxd.h>
#include "x86intrin.h"
+typedef enum QemuDsaTaskType {
+ QEMU_DSA_TASK = 0,
+ QEMU_DSA_BATCH_TASK
+} QemuDsaTaskType;
+
+typedef enum QemuDsaTaskStatus {
+ QEMU_DSA_TASK_READY = 0,
+ QEMU_DSA_TASK_PROCESSING,
+ QEMU_DSA_TASK_COMPLETION
+} QemuDsaTaskStatus;
+
typedef struct {
void *work_queue;
} QemuDsaDevice;
@@ -44,6 +55,24 @@ typedef struct {
QemuDsaTaskQueue task_queue;
} QemuDsaDeviceGroup;
+typedef void (*qemu_dsa_completion_fn)(void *);
+
+typedef struct QemuDsaBatchTask {
+ struct dsa_hw_desc batch_descriptor;
+ struct dsa_hw_desc *descriptors;
+ struct dsa_completion_record batch_completion __attribute__((aligned(32)));
+ struct dsa_completion_record *completions;
+ QemuDsaDeviceGroup *group;
+ QemuDsaDevice *device;
+ qemu_dsa_completion_fn completion_callback;
+ QemuSemaphore sem_task_complete;
+ QemuDsaTaskType task_type;
+ QemuDsaTaskStatus status;
+ int batch_size;
+ QSIMPLEQ_ENTRY(QemuDsaBatchTask) entry;
+} QemuDsaBatchTask;
+
+
/**
* @brief Initializes DSA devices.
*
diff --git a/util/dsa.c b/util/dsa.c
index 57f1cbe68f..8e78215903 100644
--- a/util/dsa.c
+++ b/util/dsa.c
@@ -31,6 +31,7 @@
#include "x86intrin.h"
#define DSA_WQ_PORTAL_SIZE 4096
+#define DSA_WQ_DEPTH 128
#define MAX_DSA_DEVICES 16
uint32_t max_retry_count;
@@ -210,6 +211,182 @@ dsa_device_group_get_next_device(QemuDsaDeviceGroup
*group)
return &group->dsa_devices[current];
}
+/**
+ * @brief Empties out the DSA task queue.
+ *
+ * @param group A pointer to the DSA device group.
+ */
+static void
+dsa_empty_task_queue(QemuDsaDeviceGroup *group)
+{
+ qemu_mutex_lock(&group->task_queue_lock);
+ QemuDsaTaskQueue *task_queue = &group->task_queue;
+ while (!QSIMPLEQ_EMPTY(task_queue)) {
+ QSIMPLEQ_REMOVE_HEAD(task_queue, entry);
+ }
+ qemu_mutex_unlock(&group->task_queue_lock);
+}
+
+/**
+ * @brief Adds a task to the DSA task queue.
+ *
+ * @param group A pointer to the DSA device group.
+ * @param task A pointer to the DSA task to enqueue.
+ *
+ * @return int Zero if successful, otherwise a proper error code.
+ */
+static int
+dsa_task_enqueue(QemuDsaDeviceGroup *group,
+ QemuDsaBatchTask *task)
+{
+ bool notify = false;
+
+ qemu_mutex_lock(&group->task_queue_lock);
+
+ if (!group->running) {
+ error_report("DSA: Tried to queue task to stopped device queue.");
+ qemu_mutex_unlock(&group->task_queue_lock);
+ return -1;
+ }
+
+ /* The queue is empty. This enqueue operation is a 0->1 transition. */
+ if (QSIMPLEQ_EMPTY(&group->task_queue)) {
+ notify = true;
+ }
+
+ QSIMPLEQ_INSERT_TAIL(&group->task_queue, task, entry);
+
+ /* We need to notify the waiter for 0->1 transitions. */
+ if (notify) {
+ qemu_cond_signal(&group->task_queue_cond);
+ }
+
+ qemu_mutex_unlock(&group->task_queue_lock);
+
+ return 0;
+}
+
+/**
+ * @brief Takes a DSA task out of the task queue.
+ *
+ * @param group A pointer to the DSA device group.
+ * @return QemuDsaBatchTask* The DSA task being dequeued.
+ */
+__attribute__((unused))
+static QemuDsaBatchTask *
+dsa_task_dequeue(QemuDsaDeviceGroup *group)
+{
+ QemuDsaBatchTask *task = NULL;
+
+ qemu_mutex_lock(&group->task_queue_lock);
+
+ while (true) {
+ if (!group->running) {
+ goto exit;
+ }
+ task = QSIMPLEQ_FIRST(&group->task_queue);
+ if (task != NULL) {
+ break;
+ }
+ qemu_cond_wait(&group->task_queue_cond, &group->task_queue_lock);
+ }
+
+ QSIMPLEQ_REMOVE_HEAD(&group->task_queue, entry);
+
+exit:
+ qemu_mutex_unlock(&group->task_queue_lock);
+ return task;
+}
+
+/**
+ * @brief Submits a DSA work item to the device work queue.
+ *
+ * @param wq A pointer to the DSA work queue's device memory.
+ * @param descriptor A pointer to the DSA work item descriptor.
+ *
+ * @return Zero if successful, non-zero otherwise.
+ */
+static int
+submit_wi_int(void *wq, struct dsa_hw_desc *descriptor)
+{
+ uint32_t retry = 0;
+
+ _mm_sfence();
+
+ while (true) {
+ if (_enqcmd(wq, descriptor) == 0) {
+ break;
+ }
+ retry++;
+ if (retry > max_retry_count) {
+ error_report("Submit work retry %u times.", retry);
+ return -1;
+ }
+ }
+
+ return 0;
+}
+
+/**
+ * @brief Asynchronously submits a DSA work item to the
+ * device work queue.
+ *
+ * @param task A pointer to the task.
+ *
+ * @return int Zero if successful, non-zero otherwise.
+ */
+__attribute__((unused))
+static int
+submit_wi_async(QemuDsaBatchTask *task)
+{
+ QemuDsaDeviceGroup *device_group = task->group;
+ QemuDsaDevice *device_instance = task->device;
+ int ret;
+
+ assert(task->task_type == QEMU_DSA_TASK);
+
+ task->status = QEMU_DSA_TASK_PROCESSING;
+
+ ret = submit_wi_int(device_instance->work_queue,
+ &task->descriptors[0]);
+ if (ret != 0) {
+ return ret;
+ }
+
+ return dsa_task_enqueue(device_group, task);
+}
+
+/**
+ * @brief Asynchronously submits a DSA batch work item to the
+ * device work queue.
+ *
+ * @param batch_task A pointer to the batch task.
+ *
+ * @return int Zero if successful, non-zero otherwise.
+ */
+__attribute__((unused))
+static int
+submit_batch_wi_async(QemuDsaBatchTask *batch_task)
+{
+ QemuDsaDeviceGroup *device_group = batch_task->group;
+ QemuDsaDevice *device_instance = batch_task->device;
+ int ret;
+
+ assert(batch_task->task_type == QEMU_DSA_BATCH_TASK);
+ assert(batch_task->batch_descriptor.desc_count <= batch_task->batch_size);
+ assert(batch_task->status == QEMU_DSA_TASK_READY);
+
+ batch_task->status = QEMU_DSA_TASK_PROCESSING;
+
+ ret = submit_wi_int(device_instance->work_queue,
+ &batch_task->batch_descriptor);
+ if (ret != 0) {
+ return ret;
+ }
+
+ return dsa_task_enqueue(device_group, batch_task);
+}
+
/**
* @brief Check if DSA is running.
*
@@ -223,7 +400,12 @@ bool qemu_dsa_is_running(void)
static void
dsa_globals_init(void)
{
- max_retry_count = UINT32_MAX;
+ /*
+ * This value follows a reference example by Intel. The POLL_RETRY_MAX is
+ * defined to 10000, so here we used the max WQ depth * 100 for the the max
+ * polling retry count.
+ */
+ max_retry_count = DSA_WQ_DEPTH * 100;
}
/**
@@ -266,6 +448,8 @@ void qemu_dsa_stop(void)
if (!group->running) {
return;
}
+
+ dsa_empty_task_queue(group);
}
/**
--
Yichen Wang
