iommufd: register container for cpr

Steven Sistare Mon, 19 May 2025 08:53:28 -0700

On 5/16/2025 6:23 AM, Duan, Zhenzhong wrote:
 >> -----Original Message-----

From: Steve Sistare <steven.sist...@oracle.com>
Subject: [PATCH V3 35/42] vfio/iommufd: register container for cpr


Register a vfio iommufd container and device for CPR, replacing the generic
CPR register call with a more specific iommufd register call.  Add a
blocker if the kernel does not support IOMMU_IOAS_CHANGE_PROCESS.

This is mostly boiler plate.  The fields to to saved and restored are added
in subsequent patches.

Signed-off-by: Steve Sistare <steven.sist...@oracle.com>
---
hw/vfio/cpr-iommufd.c      | 97
++++++++++++++++++++++++++++++++++++++++++++++
hw/vfio/iommufd.c          |  6 ++-
hw/vfio/meson.build        |  1 +
hw/vfio/vfio-iommufd.h     |  1 +
include/hw/vfio/vfio-cpr.h |  8 ++++
5 files changed, 111 insertions(+), 2 deletions(-)
create mode 100644 hw/vfio/cpr-iommufd.c

diff --git a/hw/vfio/cpr-iommufd.c b/hw/vfio/cpr-iommufd.c
new file mode 100644
index 0000000..46f2006
--- /dev/null
+++ b/hw/vfio/cpr-iommufd.c
@@ -0,0 +1,97 @@
+/*
+ * Copyright (c) 2024-2025 Oracle and/or its affiliates.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "hw/vfio/vfio-cpr.h"
+#include "migration/blocker.h"
+#include "migration/cpr.h"
+#include "migration/migration.h"
+#include "migration/vmstate.h"
+#include "system/iommufd.h"
+#include "vfio-iommufd.h"
+
+static bool vfio_cpr_supported(VFIOIOMMUFDContainer *container, Error
**errp)
+{
+    if (!iommufd_change_process_capable(container->be)) {
+        error_setg(errp,
+                   "VFIO container does not support 
IOMMU_IOAS_CHANGE_PROCESS");
+        return false;
+    }
+    return true;
+}
+
+static const VMStateDescription vfio_container_vmstate = {
+    .name = "vfio-iommufd-container",
+    .version_id = 0,
+    .minimum_version_id = 0,
+    .needed = cpr_needed_for_reuse,
+    .fields = (VMStateField[]) {
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static const VMStateDescription iommufd_cpr_vmstate = {
+    .name = "iommufd",
+    .version_id = 0,
+    .minimum_version_id = 0,
+    .needed = cpr_needed_for_reuse,
+    .fields = (VMStateField[]) {
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+bool vfio_iommufd_cpr_register_container(VFIOIOMMUFDContainer
*container,
+                                         Error **errp)
+{
+    VFIOContainerBase *bcontainer = &container->bcontainer;
+    Error **cpr_blocker = &container->cpr_blocker;
+
+    migration_add_notifier_mode(&bcontainer->cpr_reboot_notifier,
+                                vfio_cpr_reboot_notifier,
+                                MIG_MODE_CPR_REBOOT);
+
+    if (!vfio_cpr_supported(container, cpr_blocker)) {
+        return migrate_add_blocker_modes(cpr_blocker, errp,
+                                         MIG_MODE_CPR_TRANSFER, -1) == 0;
+    }
+
+    vmstate_register(NULL, -1, &vfio_container_vmstate, container);
+    vmstate_register(NULL, -1, &iommufd_cpr_vmstate, container->be);


Will this register iommufd be multiple times if multiple containers under one 
iommufd?
Maybe introduce a cpr_register_iommufd()?


I thought iommufd:container is 1:1 because of this logic in iommufd_cdev_attach:

    QLIST_FOREACH(bcontainer, &space->containers, next) {
        container = container_of(bcontainer, VFIOIOMMUFDContainer, bcontainer);
        if (VFIO_IOMMU_GET_CLASS(bcontainer) != iommufd_vioc ||
            vbasedev->iommufd != container->be) {
            continue;
    }
    /* Need to allocate a new dedicated container */

- Steve

+
+    return true;
+}
+
+void vfio_iommufd_cpr_unregister_container(VFIOIOMMUFDContainer
*container)
+{
+    VFIOContainerBase *bcontainer = &container->bcontainer;
+
+    vmstate_unregister(NULL, &iommufd_cpr_vmstate, container->be);
+    vmstate_unregister(NULL, &vfio_container_vmstate, container);
+    migrate_del_blocker(&container->cpr_blocker);
+    migration_remove_notifier(&bcontainer->cpr_reboot_notifier);
+}
+
+static const VMStateDescription vfio_device_vmstate = {
+    .name = "vfio-iommufd-device",
+    .version_id = 0,
+    .minimum_version_id = 0,
+    .needed = cpr_needed_for_reuse,
+    .fields = (VMStateField[]) {
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+void vfio_iommufd_cpr_register_device(VFIODevice *vbasedev)
+{
+    vmstate_register(NULL, -1, &vfio_device_vmstate, vbasedev);
+}
+
+void vfio_iommufd_cpr_unregister_device(VFIODevice *vbasedev)
+{
+    vmstate_unregister(NULL, &vfio_device_vmstate, vbasedev);
+}
diff --git a/hw/vfio/iommufd.c b/hw/vfio/iommufd.c
index ea99b8d..dabb948 100644
--- a/hw/vfio/iommufd.c
+++ b/hw/vfio/iommufd.c
@@ -460,7 +460,7 @@ static void
iommufd_cdev_container_destroy(VFIOIOMMUFDContainer *container)
     if (!QLIST_EMPTY(&bcontainer->device_list)) {
         return;
     }
-    vfio_cpr_unregister_container(bcontainer);
+    vfio_iommufd_cpr_unregister_container(container);
     vfio_listener_unregister(bcontainer);
     iommufd_backend_free_id(container->be, container->ioas_id);
     object_unref(container);
@@ -611,7 +611,7 @@ static bool iommufd_cdev_attach(const char *name,
VFIODevice *vbasedev,
         goto err_listener_register;
     }

-    if (!vfio_cpr_register_container(bcontainer, errp)) {
+    if (!vfio_iommufd_cpr_register_container(container, errp)) {
         goto err_listener_register;
     }

@@ -633,6 +633,7 @@ found_container:
     }

     vfio_device_prepare(vbasedev, bcontainer, &dev_info);
+    vfio_iommufd_cpr_register_device(vbasedev);

     trace_iommufd_cdev_device_info(vbasedev->name, devfd, vbasedev-

num_irqs,

                                    vbasedev->num_regions, vbasedev->flags);
@@ -671,6 +672,7 @@ static void iommufd_cdev_detach(VFIODevice *vbasedev)
     vfio_address_space_put(space);

     migrate_del_blocker(&vbasedev->cpr.id_blocker);
+    vfio_iommufd_cpr_unregister_device(vbasedev);
     iommufd_cdev_unbind_and_disconnect(vbasedev);
     close(vbasedev->fd);
}
diff --git a/hw/vfio/meson.build b/hw/vfio/meson.build
index 73d29f9..a158fd8 100644
--- a/hw/vfio/meson.build
+++ b/hw/vfio/meson.build
@@ -21,6 +21,7 @@ system_ss.add(when: 'CONFIG_VFIO_XGMAC', if_true:
files('calxeda-xgmac.c'))
system_ss.add(when: 'CONFIG_VFIO_AMD_XGBE', if_true: files('amd-xgbe.c'))
system_ss.add(when: 'CONFIG_VFIO', if_true: files(
   'cpr.c',
+  'cpr-iommufd.c',
   'cpr-legacy.c',
   'device.c',
   'migration.c',
diff --git a/hw/vfio/vfio-iommufd.h b/hw/vfio/vfio-iommufd.h
index 5615dcd..cc57a05 100644
--- a/hw/vfio/vfio-iommufd.h
+++ b/hw/vfio/vfio-iommufd.h
@@ -25,6 +25,7 @@ typedef struct IOMMUFDBackend IOMMUFDBackend;
typedef struct VFIOIOMMUFDContainer {
     VFIOContainerBase bcontainer;
     IOMMUFDBackend *be;
+    Error *cpr_blocker;
     uint32_t ioas_id;
     QLIST_HEAD(, VFIOIOASHwpt) hwpt_list;
} VFIOIOMMUFDContainer;
diff --git a/include/hw/vfio/vfio-cpr.h b/include/hw/vfio/vfio-cpr.h
index d06d117..1379b20 100644
--- a/include/hw/vfio/vfio-cpr.h
+++ b/include/hw/vfio/vfio-cpr.h
@@ -31,6 +31,7 @@ struct VFIOContainerBase;
struct VFIOGroup;
struct VFIOPCIDevice;
struct VFIODevice;
+struct VFIOIOMMUFDContainer;

bool vfio_legacy_cpr_register_container(struct VFIOContainer *container,
                                         Error **errp);
@@ -43,6 +44,13 @@ bool vfio_cpr_register_container(struct
VFIOContainerBase *bcontainer,
                                  Error **errp);
void vfio_cpr_unregister_container(struct VFIOContainerBase *bcontainer);

+bool vfio_iommufd_cpr_register_container(struct VFIOIOMMUFDContainer
*container,
+                                         Error **errp);
+void vfio_iommufd_cpr_unregister_container(
+    struct VFIOIOMMUFDContainer *container);
+void vfio_iommufd_cpr_register_device(struct VFIODevice *vbasedev);
+void vfio_iommufd_cpr_unregister_device(struct VFIODevice *vbasedev);
+
bool vfio_cpr_container_match(struct VFIOContainer *container,
                               struct VFIOGroup *group, int *fd);

--
1.8.3.1

Re: [PATCH V3 35/42] vfio/iommufd: register container for cpr

Reply via email to