>-----Original Message-----
>From: Steve Sistare <steven.sist...@oracle.com>
>Subject: [PATCH V3 35/42] vfio/iommufd: register container for cpr
>
>Register a vfio iommufd container and device for CPR, replacing the generic
>CPR register call with a more specific iommufd register call. Add a
>blocker if the kernel does not support IOMMU_IOAS_CHANGE_PROCESS.
>
>This is mostly boiler plate. The fields to to saved and restored are added
>in subsequent patches.
>
>Signed-off-by: Steve Sistare <steven.sist...@oracle.com>
>---
> hw/vfio/cpr-iommufd.c | 97
>++++++++++++++++++++++++++++++++++++++++++++++
> hw/vfio/iommufd.c | 6 ++-
> hw/vfio/meson.build | 1 +
> hw/vfio/vfio-iommufd.h | 1 +
> include/hw/vfio/vfio-cpr.h | 8 ++++
> 5 files changed, 111 insertions(+), 2 deletions(-)
> create mode 100644 hw/vfio/cpr-iommufd.c
>
>diff --git a/hw/vfio/cpr-iommufd.c b/hw/vfio/cpr-iommufd.c
>new file mode 100644
>index 0000000..46f2006
>--- /dev/null
>+++ b/hw/vfio/cpr-iommufd.c
>@@ -0,0 +1,97 @@
>+/*
>+ * Copyright (c) 2024-2025 Oracle and/or its affiliates.
>+ *
>+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
>+ * See the COPYING file in the top-level directory.
>+ */
>+
>+#include "qemu/osdep.h"
>+#include "qapi/error.h"
>+#include "hw/vfio/vfio-cpr.h"
>+#include "migration/blocker.h"
>+#include "migration/cpr.h"
>+#include "migration/migration.h"
>+#include "migration/vmstate.h"
>+#include "system/iommufd.h"
>+#include "vfio-iommufd.h"
>+
>+static bool vfio_cpr_supported(VFIOIOMMUFDContainer *container, Error
>**errp)
>+{
>+ if (!iommufd_change_process_capable(container->be)) {
>+ error_setg(errp,
>+ "VFIO container does not support
>IOMMU_IOAS_CHANGE_PROCESS");
>+ return false;
>+ }
>+ return true;
>+}
>+
>+static const VMStateDescription vfio_container_vmstate = {
>+ .name = "vfio-iommufd-container",
>+ .version_id = 0,
>+ .minimum_version_id = 0,
>+ .needed = cpr_needed_for_reuse,
>+ .fields = (VMStateField[]) {
>+ VMSTATE_END_OF_LIST()
>+ }
>+};
>+
>+static const VMStateDescription iommufd_cpr_vmstate = {
>+ .name = "iommufd",
>+ .version_id = 0,
>+ .minimum_version_id = 0,
>+ .needed = cpr_needed_for_reuse,
>+ .fields = (VMStateField[]) {
>+ VMSTATE_END_OF_LIST()
>+ }
>+};
>+
>+bool vfio_iommufd_cpr_register_container(VFIOIOMMUFDContainer
>*container,
>+ Error **errp)
>+{
>+ VFIOContainerBase *bcontainer = &container->bcontainer;
>+ Error **cpr_blocker = &container->cpr_blocker;
>+
>+ migration_add_notifier_mode(&bcontainer->cpr_reboot_notifier,
>+ vfio_cpr_reboot_notifier,
>+ MIG_MODE_CPR_REBOOT);
>+
>+ if (!vfio_cpr_supported(container, cpr_blocker)) {
>+ return migrate_add_blocker_modes(cpr_blocker, errp,
>+ MIG_MODE_CPR_TRANSFER, -1) == 0;
>+ }
>+
>+ vmstate_register(NULL, -1, &vfio_container_vmstate, container);
>+ vmstate_register(NULL, -1, &iommufd_cpr_vmstate, container->be);
Will this register iommufd be multiple times if multiple containers under one
iommufd?
Maybe introduce a cpr_register_iommufd()?
>+
>+ return true;
>+}
>+
>+void vfio_iommufd_cpr_unregister_container(VFIOIOMMUFDContainer
>*container)
>+{
>+ VFIOContainerBase *bcontainer = &container->bcontainer;
>+
>+ vmstate_unregister(NULL, &iommufd_cpr_vmstate, container->be);
>+ vmstate_unregister(NULL, &vfio_container_vmstate, container);
>+ migrate_del_blocker(&container->cpr_blocker);
>+ migration_remove_notifier(&bcontainer->cpr_reboot_notifier);
>+}
>+
>+static const VMStateDescription vfio_device_vmstate = {
>+ .name = "vfio-iommufd-device",
>+ .version_id = 0,
>+ .minimum_version_id = 0,
>+ .needed = cpr_needed_for_reuse,
>+ .fields = (VMStateField[]) {
>+ VMSTATE_END_OF_LIST()
>+ }
>+};
>+
>+void vfio_iommufd_cpr_register_device(VFIODevice *vbasedev)
>+{
>+ vmstate_register(NULL, -1, &vfio_device_vmstate, vbasedev);
>+}
>+
>+void vfio_iommufd_cpr_unregister_device(VFIODevice *vbasedev)
>+{
>+ vmstate_unregister(NULL, &vfio_device_vmstate, vbasedev);
>+}
>diff --git a/hw/vfio/iommufd.c b/hw/vfio/iommufd.c
>index ea99b8d..dabb948 100644
>--- a/hw/vfio/iommufd.c
>+++ b/hw/vfio/iommufd.c
>@@ -460,7 +460,7 @@ static void
>iommufd_cdev_container_destroy(VFIOIOMMUFDContainer *container)
> if (!QLIST_EMPTY(&bcontainer->device_list)) {
> return;
> }
>- vfio_cpr_unregister_container(bcontainer);
>+ vfio_iommufd_cpr_unregister_container(container);
> vfio_listener_unregister(bcontainer);
> iommufd_backend_free_id(container->be, container->ioas_id);
> object_unref(container);
>@@ -611,7 +611,7 @@ static bool iommufd_cdev_attach(const char *name,
>VFIODevice *vbasedev,
> goto err_listener_register;
> }
>
>- if (!vfio_cpr_register_container(bcontainer, errp)) {
>+ if (!vfio_iommufd_cpr_register_container(container, errp)) {
> goto err_listener_register;
> }
>
>@@ -633,6 +633,7 @@ found_container:
> }
>
> vfio_device_prepare(vbasedev, bcontainer, &dev_info);
>+ vfio_iommufd_cpr_register_device(vbasedev);
>
> trace_iommufd_cdev_device_info(vbasedev->name, devfd, vbasedev-
>>num_irqs,
> vbasedev->num_regions, vbasedev->flags);
>@@ -671,6 +672,7 @@ static void iommufd_cdev_detach(VFIODevice *vbasedev)
> vfio_address_space_put(space);
>
> migrate_del_blocker(&vbasedev->cpr.id_blocker);
>+ vfio_iommufd_cpr_unregister_device(vbasedev);
> iommufd_cdev_unbind_and_disconnect(vbasedev);
> close(vbasedev->fd);
> }
>diff --git a/hw/vfio/meson.build b/hw/vfio/meson.build
>index 73d29f9..a158fd8 100644
>--- a/hw/vfio/meson.build
>+++ b/hw/vfio/meson.build
>@@ -21,6 +21,7 @@ system_ss.add(when: 'CONFIG_VFIO_XGMAC', if_true:
>files('calxeda-xgmac.c'))
> system_ss.add(when: 'CONFIG_VFIO_AMD_XGBE', if_true: files('amd-xgbe.c'))
> system_ss.add(when: 'CONFIG_VFIO', if_true: files(
> 'cpr.c',
>+ 'cpr-iommufd.c',
> 'cpr-legacy.c',
> 'device.c',
> 'migration.c',
>diff --git a/hw/vfio/vfio-iommufd.h b/hw/vfio/vfio-iommufd.h
>index 5615dcd..cc57a05 100644
>--- a/hw/vfio/vfio-iommufd.h
>+++ b/hw/vfio/vfio-iommufd.h
>@@ -25,6 +25,7 @@ typedef struct IOMMUFDBackend IOMMUFDBackend;
> typedef struct VFIOIOMMUFDContainer {
> VFIOContainerBase bcontainer;
> IOMMUFDBackend *be;
>+ Error *cpr_blocker;
> uint32_t ioas_id;
> QLIST_HEAD(, VFIOIOASHwpt) hwpt_list;
> } VFIOIOMMUFDContainer;
>diff --git a/include/hw/vfio/vfio-cpr.h b/include/hw/vfio/vfio-cpr.h
>index d06d117..1379b20 100644
>--- a/include/hw/vfio/vfio-cpr.h
>+++ b/include/hw/vfio/vfio-cpr.h
>@@ -31,6 +31,7 @@ struct VFIOContainerBase;
> struct VFIOGroup;
> struct VFIOPCIDevice;
> struct VFIODevice;
>+struct VFIOIOMMUFDContainer;
>
> bool vfio_legacy_cpr_register_container(struct VFIOContainer *container,
> Error **errp);
>@@ -43,6 +44,13 @@ bool vfio_cpr_register_container(struct
>VFIOContainerBase *bcontainer,
> Error **errp);
> void vfio_cpr_unregister_container(struct VFIOContainerBase *bcontainer);
>
>+bool vfio_iommufd_cpr_register_container(struct VFIOIOMMUFDContainer
>*container,
>+ Error **errp);
>+void vfio_iommufd_cpr_unregister_container(
>+ struct VFIOIOMMUFDContainer *container);
>+void vfio_iommufd_cpr_register_device(struct VFIODevice *vbasedev);
>+void vfio_iommufd_cpr_unregister_device(struct VFIODevice *vbasedev);
>+
> bool vfio_cpr_container_match(struct VFIOContainer *container,
> struct VFIOGroup *group, int *fd);
>
>--
>1.8.3.1
