* Add a DSA task completion callback.
* DSA completion thread will call the tasks's completion callback
on every task/batch task completion.
* DSA submission path to wait for completion.
* Implement CPU fallback if DSA is not able to complete the task.
Signed-off-by: Hao Xiang <hao.xi...@linux.dev>
Signed-off-by: Bryan Zhang <bryan.zh...@bytedance.com>
---
include/qemu/dsa.h | 14 +++++
util/dsa.c | 147 ++++++++++++++++++++++++++++++++++++++++++++-
2 files changed, 158 insertions(+), 3 deletions(-)
diff --git a/include/qemu/dsa.h b/include/qemu/dsa.h
index 645e6fc367..e002652879 100644
--- a/include/qemu/dsa.h
+++ b/include/qemu/dsa.h
@@ -91,6 +91,20 @@ buffer_zero_batch_task_init(struct dsa_batch_task *task,
*/
void buffer_zero_batch_task_destroy(struct dsa_batch_task *task);
+/**
+ * @brief Performs buffer zero comparison on a DSA batch task asynchronously.
+ *
+ * @param batch_task A pointer to the batch task.
+ * @param buf An array of memory buffers.
+ * @param count The number of buffers in the array.
+ * @param len The buffer length.
+ *
+ * @return Zero if successful, otherwise non-zero.
+ */
+int
+buffer_is_zero_dsa_batch_async(struct dsa_batch_task *batch_task,
+ const void **buf, size_t count, size_t len);
+
#else
static inline bool dsa_is_running(void)
diff --git a/util/dsa.c b/util/dsa.c
index 9db4cfcf1d..5a2bf33651 100644
--- a/util/dsa.c
+++ b/util/dsa.c
@@ -473,6 +473,57 @@ poll_completion(struct dsa_completion_record *completion,
return 0;
}
+/**
+ * @brief Helper function to use CPU to complete a single
+ * zero page checking task.
+ *
+ * @param completion A pointer to a DSA task completion record.
+ * @param descriptor A pointer to a DSA task descriptor.
+ * @param result A pointer to the result of a zero page checking.
+ */
+static void
+task_cpu_fallback_int(struct dsa_completion_record *completion,
+ struct dsa_hw_desc *descriptor, bool *result)
+{
+ const uint8_t *buf;
+ size_t len;
+
+ if (completion->status == DSA_COMP_SUCCESS) {
+ return;
+ }
+
+ /*
+ * DSA was able to partially complete the operation. Check the
+ * result. If we already know this is not a zero page, we can
+ * return now.
+ */
+ if (completion->bytes_completed != 0 && completion->result != 0) {
+ *result = false;
+ return;
+ }
+
+ /* Let's fallback to use CPU to complete it. */
+ buf = (const uint8_t *)descriptor->src_addr;
+ len = descriptor->xfer_size;
+ *result = buffer_is_zero(buf + completion->bytes_completed,
+ len - completion->bytes_completed);
+}
+
+/**
+ * @brief Use CPU to complete a single zero page checking task.
+ *
+ * @param task A pointer to the task.
+ */
+static void
+task_cpu_fallback(struct dsa_batch_task *task)
+{
+ assert(task->task_type == DSA_TASK);
+
+ task_cpu_fallback_int(&task->completions[0],
+ &task->descriptors[0],
+ &task->results[0]);
+}
+
/**
* @brief Complete a single DSA task in the batch task.
*
@@ -574,6 +625,47 @@ exit:
return ret;
}
+/**
+ * @brief Use CPU to complete the zero page checking batch task.
+ *
+ * @param batch_task A pointer to the batch task.
+ */
+static void
+batch_task_cpu_fallback(struct dsa_batch_task *batch_task)
+{
+ assert(batch_task->task_type == DSA_BATCH_TASK);
+
+ struct dsa_completion_record *batch_completion =
+ &batch_task->batch_completion;
+ struct dsa_completion_record *completion;
+ uint8_t status;
+ bool *results = batch_task->results;
+ uint32_t count = batch_task->batch_descriptor.desc_count;
+
+ /* DSA is able to complete the entire batch task. */
+ if (batch_completion->status == DSA_COMP_SUCCESS) {
+ assert(count == batch_completion->bytes_completed);
+ return;
+ }
+
+ /*
+ * DSA encounters some error and is not able to complete
+ * the entire batch task. Use CPU fallback.
+ */
+ for (int i = 0; i < count; i++) {
+
+ completion = &batch_task->completions[i];
+ status = completion->status;
+
+ assert(status == DSA_COMP_SUCCESS ||
+ status == DSA_COMP_PAGE_FAULT_NOBOF);
+
+ task_cpu_fallback_int(completion,
+ &batch_task->descriptors[i],
+ &results[i]);
+ }
+}
+
/**
* @brief Handles an asynchronous DSA batch task completion.
*
@@ -861,7 +953,6 @@ buffer_zero_batch_task_set(struct dsa_batch_task
*batch_task,
*
* @return int Zero if successful, otherwise an appropriate error code.
*/
-__attribute__((unused))
static int
buffer_zero_dsa_async(struct dsa_batch_task *task,
const void *buf, size_t len)
@@ -880,7 +971,6 @@ buffer_zero_dsa_async(struct dsa_batch_task *task,
* @param count The number of buffers.
* @param len The buffer length.
*/
-__attribute__((unused))
static int
buffer_zero_dsa_batch_async(struct dsa_batch_task *batch_task,
const void **buf, size_t count, size_t len)
@@ -911,13 +1001,29 @@ buffer_zero_dsa_completion(void *context)
*
* @param batch_task A pointer to the buffer zero comparison batch task.
*/
-__attribute__((unused))
static void
buffer_zero_dsa_wait(struct dsa_batch_task *batch_task)
{
qemu_sem_wait(&batch_task->sem_task_complete);
}
+/**
+ * @brief Use CPU to complete the zero page checking task if DSA
+ * is not able to complete it.
+ *
+ * @param batch_task A pointer to the batch task.
+ */
+static void
+buffer_zero_cpu_fallback(struct dsa_batch_task *batch_task)
+{
+ if (batch_task->task_type == DSA_TASK) {
+ task_cpu_fallback(batch_task);
+ } else {
+ assert(batch_task->task_type == DSA_BATCH_TASK);
+ batch_task_cpu_fallback(batch_task);
+ }
+}
+
/**
* @brief Check if DSA is running.
*
@@ -990,5 +1096,40 @@ void dsa_cleanup(void)
dsa_device_group_cleanup(&dsa_group);
}
+/**
+ * @brief Performs buffer zero comparison on a DSA batch task asynchronously.
+ *
+ * @param batch_task A pointer to the batch task.
+ * @param buf An array of memory buffers.
+ * @param count The number of buffers in the array.
+ * @param len The buffer length.
+ *
+ * @return Zero if successful, otherwise non-zero.
+ */
+int
+buffer_is_zero_dsa_batch_async(struct dsa_batch_task *batch_task,
+ const void **buf, size_t count, size_t len)
+{
+ if (count <= 0 || count > batch_task->batch_size) {
+ return -1;
+ }
+
+ assert(batch_task != NULL);
+ assert(len != 0);
+ assert(buf != NULL);
+
+ if (count == 1) {
+ /* DSA doesn't take batch operation with only 1 task. */
+ buffer_zero_dsa_async(batch_task, buf[0], len);
+ } else {
+ buffer_zero_dsa_batch_async(batch_task, buf, count, len);
+ }
+
+ buffer_zero_dsa_wait(batch_task);
+ buffer_zero_cpu_fallback(batch_task);
+
+ return 0;
+}
+
#endif
--
2.30.2
