[PATCH RFC 10/14] migration/rdma: Wait for all multifd to complete registration

[Zhimin Feng]

Signed-off-by: Zhimin Feng <fengzhim...@huawei.com>
---
 migration/multifd.c |  6 ++++++
 migration/rdma.c    | 17 +++++++++++++++++
 2 files changed, 23 insertions(+)

diff --git a/migration/multifd.c b/migration/multifd.c
index 4ae25fc88f..c986d4c247 100644
--- a/migration/multifd.c
+++ b/migration/multifd.c
@@ -389,6 +389,7 @@ static void multifd_send_terminate_threads(Error *err)
         qemu_mutex_lock(&p->mutex);
         p->quit = true;
         if (migrate_use_rdma()) {
+            qemu_sem_post(&p->sem);
             qemu_sem_post(&p->sem_sync);
         } else {
             qemu_sem_post(&p->sem);
@@ -502,6 +503,11 @@ static void *multifd_rdma_send_thread(void *opaque)
     if (qemu_rdma_registration(p->rdma) < 0) {
         goto out;
     }
+    /*
+     * Inform the main RDMA thread to run when multifd
+     * RDMA thread have completed registration.
+     */
+    qemu_sem_post(&p->sem);
 
     while (true) {
         qemu_sem_wait(&p->sem_sync);
diff --git a/migration/rdma.c b/migration/rdma.c
index 5de3a29712..4c48e9832c 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -3733,6 +3733,23 @@ static int qemu_rdma_registration_stop(QEMUFile *f, void 
*opaque,
                     rdma->dest_blocks[i].remote_host_addr;
             local->block[i].remote_rkey = rdma->dest_blocks[i].remote_rkey;
         }
+
+        /* Wait for all multifd channels to complete registration */
+        if (migrate_use_multifd()) {
+            int i;
+            int thread_count = migrate_multifd_channels();
+            MultiFDSendParams *multifd_send_param = NULL;
+            for (i = 0; i < thread_count; i++) {
+                ret = get_multifd_send_param(i, &multifd_send_param);
+                if (ret) {
+                    ERROR(errp, "rdma: error"
+                          "getting multifd_send_param(%d)", i);
+                    return ret;
+                }
+
+                qemu_sem_wait(&multifd_send_param->sem);
+            }
+        }
     }
 
     trace_qemu_rdma_registration_stop(flags);
-- 
2.19.1