Yichen Wang <yichen.w...@bytedance.com> writes:
> From: Hao Xiang <hao.xi...@linux.dev>
>
> Intel DSA offloading is an optional feature that turns on if
> proper hardware and software stack is available. To turn on
> DSA offloading in multifd live migration:
>
> dsa-accel-path="[dsa_dev_path1] [dsa_dev_path2] ... [dsa_dev_pathX]"
>
> This feature is turned off by default.
>
> Signed-off-by: Hao Xiang <hao.xi...@linux.dev>
> Signed-off-by: Yichen Wang <yichen.w...@bytedance.com>
> ---
> migration/migration-hmp-cmds.c | 15 ++++++++++-
> migration/options.c | 47 ++++++++++++++++++++++++++++++++++
> migration/options.h | 1 +
> qapi/migration.json | 32 ++++++++++++++++++++---
> 4 files changed, 90 insertions(+), 5 deletions(-)
>
> diff --git a/migration/migration-hmp-cmds.c b/migration/migration-hmp-cmds.c
> index 7d608d26e1..c422db4ecd 100644
> --- a/migration/migration-hmp-cmds.c
> +++ b/migration/migration-hmp-cmds.c
> @@ -312,7 +312,16 @@ void hmp_info_migrate_parameters(Monitor *mon, const
> QDict *qdict)
> monitor_printf(mon, "%s: '%s'\n",
> MigrationParameter_str(MIGRATION_PARAMETER_TLS_AUTHZ),
> params->tls_authz);
> -
> + if (params->has_dsa_accel_path) {
> + strList *dsa_accel_path = params->dsa_accel_path;
> + monitor_printf(mon, "%s:",
> + MigrationParameter_str(MIGRATION_PARAMETER_DSA_ACCEL_PATH));
> + while (dsa_accel_path) {
> + monitor_printf(mon, " %s", dsa_accel_path->value);
> + dsa_accel_path = dsa_accel_path->next;
> + }
> + monitor_printf(mon, "\n");
> + }
> if (params->has_block_bitmap_mapping) {
> const BitmapMigrationNodeAliasList *bmnal;
>
> @@ -563,6 +572,10 @@ void hmp_migrate_set_parameter(Monitor *mon, const QDict
> *qdict)
> p->has_x_checkpoint_delay = true;
> visit_type_uint32(v, param, &p->x_checkpoint_delay, &err);
> break;
> + case MIGRATION_PARAMETER_DSA_ACCEL_PATH:
> + p->has_dsa_accel_path = true;
> + visit_type_strList(v, param, &p->dsa_accel_path, &err);
> + break;
> case MIGRATION_PARAMETER_MULTIFD_CHANNELS:
> p->has_multifd_channels = true;
> visit_type_uint8(v, param, &p->multifd_channels, &err);
> diff --git a/migration/options.c b/migration/options.c
> index 645f55003d..f839493016 100644
> --- a/migration/options.c
> +++ b/migration/options.c
> @@ -29,6 +29,7 @@
> #include "ram.h"
> #include "options.h"
> #include "sysemu/kvm.h"
> +#include <cpuid.h>
>
> /* Maximum migrate downtime set to 2000 seconds */
> #define MAX_MIGRATE_DOWNTIME_SECONDS 2000
> @@ -162,6 +163,10 @@ Property migration_properties[] = {
> DEFINE_PROP_ZERO_PAGE_DETECTION("zero-page-detection", MigrationState,
> parameters.zero_page_detection,
> ZERO_PAGE_DETECTION_MULTIFD),
> + /* DEFINE_PROP_ARRAY("dsa-accel-path", MigrationState, x, */
> + /* parameters.dsa_accel_path, qdev_prop_string, char
> *), */
> + /* DEFINE_PROP_STRING("dsa-accel-path", MigrationState, */
> + /* parameters.dsa_accel_path), */
>
> /* Migration capabilities */
> DEFINE_PROP_MIG_CAP("x-xbzrle", MIGRATION_CAPABILITY_XBZRLE),
> @@ -815,6 +820,13 @@ const char *migrate_tls_creds(void)
> return s->parameters.tls_creds;
> }
>
> +const strList *migrate_dsa_accel_path(void)
> +{
> + MigrationState *s = migrate_get_current();
> +
> + return s->parameters.dsa_accel_path;
> +}
> +
> const char *migrate_tls_hostname(void)
> {
> MigrationState *s = migrate_get_current();
> @@ -926,6 +938,7 @@ MigrationParameters *qmp_query_migrate_parameters(Error
> **errp)
> params->zero_page_detection = s->parameters.zero_page_detection;
> params->has_direct_io = true;
> params->direct_io = s->parameters.direct_io;
> + params->dsa_accel_path = QAPI_CLONE(strList,
> s->parameters.dsa_accel_path);
>
> return params;
> }
> @@ -934,6 +947,7 @@ void migrate_params_init(MigrationParameters *params)
> {
> params->tls_hostname = g_strdup("");
> params->tls_creds = g_strdup("");
> + params->dsa_accel_path = NULL;
>
> /* Set has_* up only for parameter checks */
> params->has_throttle_trigger_threshold = true;
> @@ -1137,6 +1151,22 @@ bool migrate_params_check(MigrationParameters *params,
> Error **errp)
> return false;
> }
>
> + if (params->has_zero_page_detection &&
> + params->zero_page_detection == ZERO_PAGE_DETECTION_DSA_ACCEL) {
> +#ifdef CONFIG_DSA_OPT
> + unsigned int eax, ebx, ecx, edx;
> + /* ENQCMD is indicated by bit 29 of ecx in CPUID leaf 7, subleaf 0.
> */
> + if (!__get_cpuid_count(7, 0, &eax, &ebx, &ecx, &edx) ||
> + !(ecx & (1 << 29))) {
> + error_setg(errp, "DSA acceleration is not supported by CPU");
> + return false;
> + }
This should be a function along with the others in dsa.h, then you
wouldn't need the ifdef here.
> +#else
> + error_setg(errp, "DSA acceleration is not enabled");
> + return false;
> +#endif
> + }
> +
> return true;
> }
>
> @@ -1247,6 +1277,11 @@ static void
> migrate_params_test_apply(MigrateSetParameters *params,
> if (params->has_direct_io) {
> dest->direct_io = params->direct_io;
> }
> +
> + if (params->has_dsa_accel_path) {
> + dest->has_dsa_accel_path = true;
> + dest->dsa_accel_path = params->dsa_accel_path;
> + }
> }
>
> static void migrate_params_apply(MigrateSetParameters *params, Error **errp)
> @@ -1376,6 +1411,12 @@ static void migrate_params_apply(MigrateSetParameters
> *params, Error **errp)
> if (params->has_direct_io) {
> s->parameters.direct_io = params->direct_io;
> }
> + if (params->has_dsa_accel_path) {
> + qapi_free_strList(s->parameters.dsa_accel_path);
> + s->parameters.has_dsa_accel_path = true;
> + s->parameters.dsa_accel_path =
> + QAPI_CLONE(strList, params->dsa_accel_path);
> + }
> }
>
> void qmp_migrate_set_parameters(MigrateSetParameters *params, Error **errp)
> @@ -1401,6 +1442,12 @@ void qmp_migrate_set_parameters(MigrateSetParameters
> *params, Error **errp)
> params->tls_authz->type = QTYPE_QSTRING;
> params->tls_authz->u.s = strdup("");
> }
> + /* if (params->dsa_accel_path */
> + /* && params->dsa_accel_path->type == QTYPE_QNULL) { */
> + /* qobject_unref(params->dsa_accel_path->u.n); */
> + /* params->dsa_accel_path->type = QTYPE_QLIST; */
> + /* params->dsa_accel_path->u.s = strdup(""); */
> + /* } */
>
> migrate_params_test_apply(params, &tmp);
>
> diff --git a/migration/options.h b/migration/options.h
> index a2397026db..78b9e4080b 100644
> --- a/migration/options.h
> +++ b/migration/options.h
> @@ -85,6 +85,7 @@ const char *migrate_tls_creds(void);
> const char *migrate_tls_hostname(void);
> uint64_t migrate_xbzrle_cache_size(void);
> ZeroPageDetection migrate_zero_page_detection(void);
> +const strList *migrate_dsa_accel_path(void);
>
> /* parameters helpers */
>
> diff --git a/qapi/migration.json b/qapi/migration.json
> index 1234bef888..ff41780347 100644
> --- a/qapi/migration.json
> +++ b/qapi/migration.json
> @@ -619,10 +619,14 @@
> # multifd migration is enabled, else in the main migration thread
> # as for @legacy.
> #
> +# @dsa-accel: Perform zero page checking with the DSA accelerator
> +# offloading in multifd sender thread if multifd migration is
> +# enabled, else in the main migration thread as for @legacy.
> +#
> # Since: 9.0
> ##
> { 'enum': 'ZeroPageDetection',
> - 'data': [ 'none', 'legacy', 'multifd' ] }
> + 'data': [ 'none', 'legacy', 'multifd', 'dsa-accel' ] }
>
> ##
> # @BitmapMigrationBitmapAliasTransform:
> @@ -825,6 +829,12 @@
> # See description in @ZeroPageDetection. Default is 'multifd'.
> # (since 9.0)
> #
> +# @dsa-accel-path: If enabled, use DSA accelerator offloading for
> +# certain memory operations. Enable DSA accelerator for zero
> +# page detection offloading by setting the @zero-page-detection
> +# to dsa-accel. This parameter defines the dsa device path, and
> +# defaults to an empty list. (since 9.2)
> +#
> # @direct-io: Open migration files with O_DIRECT when possible. This
> # only has effect if the @mapped-ram capability is enabled.
> # (Since 9.1)
> @@ -843,7 +853,7 @@
> 'cpu-throttle-initial', 'cpu-throttle-increment',
> 'cpu-throttle-tailslow',
> 'tls-creds', 'tls-hostname', 'tls-authz', 'max-bandwidth',
> - 'avail-switchover-bandwidth', 'downtime-limit',
> + 'avail-switchover-bandwidth', 'downtime-limit', 'dsa-accel-path',
> { 'name': 'x-checkpoint-delay', 'features': [ 'unstable' ] },
> 'multifd-channels',
> 'xbzrle-cache-size', 'max-postcopy-bandwidth',
> @@ -1000,6 +1010,12 @@
> # See description in @ZeroPageDetection. Default is 'multifd'.
> # (since 9.0)
> #
> +# @dsa-accel-path: If enabled, use DSA accelerator offloading for
> +# certain memory operations. Enable DSA accelerator for zero
> +# page detection offloading by setting the @zero-page-detection
> +# to dsa-accel. This parameter defines the dsa device path, and
> +# defaults to an empty list. (since 9.2)
> +#
> # @direct-io: Open migration files with O_DIRECT when possible. This
> # only has effect if the @mapped-ram capability is enabled.
> # (Since 9.1)
> @@ -1044,7 +1060,8 @@
> '*vcpu-dirty-limit': 'uint64',
> '*mode': 'MigMode',
> '*zero-page-detection': 'ZeroPageDetection',
> - '*direct-io': 'bool' } }
> + '*direct-io': 'bool',
> + '*dsa-accel-path': ['str'] } }
>
> ##
> # @migrate-set-parameters:
> @@ -1204,6 +1221,12 @@
> # See description in @ZeroPageDetection. Default is 'multifd'.
> # (since 9.0)
> #
> +# @dsa-accel-path: If enabled, use DSA accelerator offloading for
> +# certain memory operations. Enable DSA accelerator for zero
> +# page detection offloading by setting the @zero-page-detection
> +# to dsa-accel. This parameter defines the dsa device path, and
> +# defaults to an empty list. (since 9.2)
> +#
> # @direct-io: Open migration files with O_DIRECT when possible. This
> # only has effect if the @mapped-ram capability is enabled.
> # (Since 9.1)
> @@ -1245,7 +1268,8 @@
> '*vcpu-dirty-limit': 'uint64',
> '*mode': 'MigMode',
> '*zero-page-detection': 'ZeroPageDetection',
> - '*direct-io': 'bool' } }
> + '*direct-io': 'bool',
> + '*dsa-accel-path': ['str'] } }
>
> ##
> # @query-migrate-parameters:
