From: Michael Baum <michae...@nvidia.com>
Add option to probe common device using import CTX/PD functions instead
of create functions.
This option requires accepting the context FD and the PD handle as
devargs.
Signed-off-by: Michael Baum <michae...@nvidia.com>
---
drivers/common/mlx5/linux/mlx5_common_os.c | 160 +++++++++++++++++--
drivers/common/mlx5/linux/mlx5_common_os.h | 6 -
drivers/common/mlx5/mlx5_common.c | 32 ++--
drivers/common/mlx5/mlx5_common.h | 6 +-
drivers/common/mlx5/windows/mlx5_common_os.c | 29 +++-
drivers/common/mlx5/windows/mlx5_common_os.h | 1 -
6 files changed, 196 insertions(+), 38 deletions(-)
diff --git a/drivers/common/mlx5/linux/mlx5_common_os.c
b/drivers/common/mlx5/linux/mlx5_common_os.c
index 0d3e24e04e..a8226e987f 100644
--- a/drivers/common/mlx5/linux/mlx5_common_os.c
+++ b/drivers/common/mlx5/linux/mlx5_common_os.c
@@ -416,7 +416,7 @@ mlx5_glue_constructor(void)
* @return
* 0 on success, a negative errno value otherwise and rte_errno is set.
*/
-int
+static int
mlx5_os_pd_create(struct mlx5_common_device *cdev)
{
#ifdef HAVE_IBV_FLOW_DV_SUPPORT
@@ -450,6 +450,65 @@ mlx5_os_pd_create(struct mlx5_common_device *cdev)
#endif /* HAVE_IBV_FLOW_DV_SUPPORT */
}
+/**
+ * Import Protection Domain object according to given pdn.
+ *
+ * @param[out] cdev
+ * Pointer to the mlx5 device.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx5_os_pd_import(struct mlx5_common_device *cdev)
+{
+ struct mlx5_common_dev_config *config = &cdev->config;
+
+ cdev->pd = mlx5_glue->import_pd(cdev->ctx, config->pd_handle);
+ if (cdev->pd == NULL) {
+ DRV_LOG(ERR, "Failed to import PD.");
+ return errno ? -errno : -ENOMEM;
+ }
+ cdev->pdn = config->pd_handle;
+ return 0;
+}
+
+/**
+ * Prepare Protection Domain object and extract its pdn using DV API.
+ *
+ * @param[out] cdev
+ * Pointer to the mlx5 device.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_os_pd_prepare(struct mlx5_common_device *cdev)
+{
+ if (cdev->config.pd_handle == MLX5_ARG_UNSET)
+ return mlx5_os_pd_create(cdev);
+ else
+ return mlx5_os_pd_import(cdev);
+}
+
+/**
+ * Release Protection Domain object.
+ *
+ * @param[out] cdev
+ * Pointer to the mlx5 device.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise.
+ */
+int
+mlx5_os_pd_release(struct mlx5_common_device *cdev)
+{
+ if (cdev->config.pd_handle == MLX5_ARG_UNSET)
+ return mlx5_glue->dealloc_pd(cdev->pd);
+ mlx5_glue->unimport_pd(cdev->pd);
+ return 0;
+}
+
static struct ibv_device *
mlx5_os_get_ibv_device(const struct rte_pci_addr *addr)
{
@@ -648,28 +707,28 @@ mlx5_restore_doorbell_mapping_env(int value)
/**
* Function API to open IB device.
*
- *
* @param cdev
* Pointer to the mlx5 device.
* @param classes
* Chosen classes come from device arguments.
*
* @return
- * 0 on success, a negative errno value otherwise and rte_errno is set.
+ * Pointer to ibv_context on success, NULL otherwise and rte_errno is set.
*/
-int
-mlx5_os_open_device(struct mlx5_common_device *cdev, uint32_t classes)
+static struct ibv_context *
+mlx5_open_device(struct mlx5_common_device *cdev, uint32_t classes)
{
struct ibv_device *ibv;
struct ibv_context *ctx = NULL;
int dbmap_env;
+ MLX5_ASSERT(cdev->config.device_fd == MLX5_ARG_UNSET);
if (classes & MLX5_CLASS_VDPA)
ibv = mlx5_vdpa_get_ibv_dev(cdev->dev);
else
ibv = mlx5_os_get_ibv_dev(cdev->dev);
if (!ibv)
- return -rte_errno;
+ return NULL;
DRV_LOG(INFO, "Dev information matches for device \"%s\".", ibv->name);
/*
* Configure environment variable "MLX5_BF_SHUT_UP" before the device
@@ -682,29 +741,104 @@ mlx5_os_open_device(struct mlx5_common_device *cdev,
uint32_t classes)
ctx = mlx5_glue->dv_open_device(ibv);
if (ctx) {
cdev->config.devx = 1;
- DRV_LOG(DEBUG, "DevX is supported.");
} else if (classes == MLX5_CLASS_ETH) {
/* The environment variable is still configured. */
ctx = mlx5_glue->open_device(ibv);
if (ctx == NULL)
goto error;
- DRV_LOG(DEBUG, "DevX is NOT supported.");
} else {
goto error;
}
/* The device is created, no need for environment. */
mlx5_restore_doorbell_mapping_env(dbmap_env);
- /* Hint libmlx5 to use PMD allocator for data plane resources */
- mlx5_set_context_attr(cdev->dev, ctx);
- cdev->ctx = ctx;
- return 0;
+ return ctx;
error:
rte_errno = errno ? errno : ENODEV;
/* The device creation is failed, no need for environment. */
mlx5_restore_doorbell_mapping_env(dbmap_env);
DRV_LOG(ERR, "Failed to open IB device \"%s\".", ibv->name);
- return -rte_errno;
+ return NULL;
}
+
+/**
+ * Function API to import IB device.
+ *
+ * @param cdev
+ * Pointer to the mlx5 device.
+ *
+ * @return
+ * Pointer to ibv_context on success, NULL otherwise and rte_errno is set.
+ */
+static struct ibv_context *
+mlx5_import_device(struct mlx5_common_device *cdev)
+{
+ struct ibv_context *ctx = NULL;
+
+ MLX5_ASSERT(cdev->config.device_fd != MLX5_ARG_UNSET);
+ ctx = mlx5_glue->import_device(cdev->config.device_fd);
+ if (!ctx) {
+ DRV_LOG(ERR, "Failed to import device for fd=%d.",
+ cdev->config.device_fd);
+ rte_errno = errno;
+ }
+ return ctx;
+}
+
+/**
+ * Function API to prepare IB device.
+ *
+ * @param cdev
+ * Pointer to the mlx5 device.
+ * @param classes
+ * Chosen classes come from device arguments.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_os_open_device(struct mlx5_common_device *cdev, uint32_t classes)
+{
+
+ struct ibv_context *ctx = NULL;
+
+ if (cdev->config.device_fd == MLX5_ARG_UNSET)
+ ctx = mlx5_open_device(cdev, classes);
+ else
+ ctx = mlx5_import_device(cdev);
+ if (ctx == NULL)
+ return -rte_errno;
+ /* Hint libmlx5 to use PMD allocator for data plane resources */
+ mlx5_set_context_attr(cdev->dev, ctx);
+ cdev->ctx = ctx;
+ return 0;
+}
+
+/**
+ * Query HCA attributes.
+ * For remote context, it is check if DevX is supported.
+ *
+ * @param cdev
+ * Pointer to mlx5 device structure.
+ *
+ * @return
+ * 0 on success, a negative value otherwise.
+ */
+int
+mlx5_os_query_hca_attr(struct mlx5_common_device *cdev)
+{
+ int ret;
+
+ ret = mlx5_devx_cmd_query_hca_attr(cdev->ctx, &cdev->config.hca_attr);
+ if (ret) {
+ if (cdev->config.device_fd == MLX5_ARG_UNSET) {
+ rte_errno = ENOTSUP;
+ return -rte_errno;
+ }
+ cdev->config.devx = 0;
+ }
+ return 0;
+}
+
int
mlx5_get_device_guid(const struct rte_pci_addr *dev, uint8_t *guid, size_t len)
{
diff --git a/drivers/common/mlx5/linux/mlx5_common_os.h
b/drivers/common/mlx5/linux/mlx5_common_os.h
index 83066e752d..246e8b2784 100644
--- a/drivers/common/mlx5/linux/mlx5_common_os.h
+++ b/drivers/common/mlx5/linux/mlx5_common_os.h
@@ -203,12 +203,6 @@ mlx5_os_get_devx_uar_page_id(void *uar)
#endif
}
-static inline int
-mlx5_os_dealloc_pd(void *pd)
-{
- return mlx5_glue->dealloc_pd(pd);
-}
-
__rte_internal
static inline void *
mlx5_os_umem_reg(void *ctx, void *addr, size_t size, uint32_t access)
diff --git a/drivers/common/mlx5/mlx5_common.c
b/drivers/common/mlx5/mlx5_common.c
index f1650f94c6..bd7f0021d5 100644
--- a/drivers/common/mlx5/mlx5_common.c
+++ b/drivers/common/mlx5/mlx5_common.c
@@ -133,6 +133,10 @@ mlx5_common_args_check_handler(const char *key, const char
*val, void *opaque)
config->mr_mempool_reg_en = !!tmp;
} else if (strcmp(key, "sys_mem_en") == 0) {
config->sys_mem_en = !!tmp;
+ } else if (strcmp(key, "device_fd") == 0) {
+ config->device_fd = !!tmp;
+ } else if (strcmp(key, "pd_handle") == 0) {
+ config->pd_handle = !!tmp;
}
return 0;
}
@@ -160,6 +164,8 @@ mlx5_common_config_get(struct rte_devargs *devargs,
config->mr_mempool_reg_en = 1;
config->sys_mem_en = 0;
config->dbnc = MLX5_ARG_UNSET;
+ config->device_fd = MLX5_ARG_UNSET;
+ config->pd_handle = MLX5_ARG_UNSET;
if (devargs == NULL)
return 0;
kvlist = rte_kvargs_parse(devargs->args, NULL);
@@ -492,7 +498,7 @@ static void
mlx5_dev_hw_global_release(struct mlx5_common_device *cdev)
{
if (cdev->pd != NULL) {
- claim_zero(mlx5_os_dealloc_pd(cdev->pd));
+ claim_zero(mlx5_os_pd_release(cdev));
cdev->pd = NULL;
}
if (cdev->ctx != NULL) {
@@ -522,19 +528,23 @@ mlx5_dev_hw_global_prepare(struct mlx5_common_device
*cdev, uint32_t classes)
if (ret < 0)
return ret;
/* Allocate Protection Domain object and extract its pdn. */
- ret = mlx5_os_pd_create(cdev);
+ ret = mlx5_os_pd_prepare(cdev);
if (ret)
goto error;
- /* All actions taken below are relevant only when DevX is supported */
- if (cdev->config.devx == 0)
- return 0;
- /* Query HCA attributes. */
- ret = mlx5_devx_cmd_query_hca_attr(cdev->ctx, &cdev->config.hca_attr);
- if (ret) {
- DRV_LOG(ERR, "Unable to read HCA capabilities.");
- rte_errno = ENOTSUP;
- goto error;
+ /*
+ * TODO: write it again...
+ * All actions taken below are relevant only when DevX is supported
+ */
+ if (cdev->config.devx || cdev->config.device_fd != MLX5_ARG_UNSET) {
+ /* Query HCA attributes. */
+ ret = mlx5_os_query_hca_attr(cdev);
+ if (ret) {
+ DRV_LOG(ERR, "Unable to read HCA capabilities.");
+ rte_errno = ENOTSUP;
+ goto error;
+ }
}
+ DRV_LOG(DEBUG, "DevX is %ssupported.", cdev->config.devx ? "" : "NOT ");
return 0;
error:
mlx5_dev_hw_global_release(cdev);
diff --git a/drivers/common/mlx5/mlx5_common.h
b/drivers/common/mlx5/mlx5_common.h
index e8809844af..78c8a29740 100644
--- a/drivers/common/mlx5/mlx5_common.h
+++ b/drivers/common/mlx5/mlx5_common.h
@@ -414,6 +414,8 @@ void mlx5_common_init(void);
struct mlx5_common_dev_config {
struct mlx5_hca_attr hca_attr; /* HCA attributes. */
int dbnc; /* Skip doorbell register write barrier. */
+ int device_fd; /* Device file descriptor for importation. */
+ int pd_handle; /* Protection Domain handle for importation. */
unsigned int devx:1; /* Whether devx interface is available or not. */
unsigned int sys_mem_en:1; /* The default memory allocator. */
unsigned int mr_mempool_reg_en:1;
@@ -507,7 +509,9 @@ mlx5_devx_uar_release(struct mlx5_uar *uar);
/* mlx5_common_os.c */
int mlx5_os_open_device(struct mlx5_common_device *cdev, uint32_t classes);
-int mlx5_os_pd_create(struct mlx5_common_device *cdev);
+int mlx5_os_query_hca_attr(struct mlx5_common_device *cdev);
+int mlx5_os_pd_prepare(struct mlx5_common_device *cdev);
+int mlx5_os_pd_release(struct mlx5_common_device *cdev);
/* mlx5 PMD wrapped MR struct. */
struct mlx5_pmd_wrapped_mr {
diff --git a/drivers/common/mlx5/windows/mlx5_common_os.c
b/drivers/common/mlx5/windows/mlx5_common_os.c
index 162c7476cc..d8410f57c1 100644
--- a/drivers/common/mlx5/windows/mlx5_common_os.c
+++ b/drivers/common/mlx5/windows/mlx5_common_os.c
@@ -25,21 +25,38 @@ mlx5_glue_constructor(void)
{
}
+/**
+ * Query HCA attributes.
+ *
+ * @param cdev
+ * Pointer to mlx5 device structure.
+ *
+ * @return
+ * 0 on success, a negative value otherwise.
+ */
+int
+mlx5_os_query_hca_attr(struct mlx5_common_device *cdev)
+{
+ return mlx5_devx_cmd_query_hca_attr(cdev->ctx, &cdev->config.hca_attr);
+}
+
/**
* Release PD. Releases a given mlx5_pd object
*
- * @param[in] pd
- * Pointer to mlx5_pd.
+ * @param[in] cdev
+ * Pointer to the mlx5 device.
*
* @return
* Zero if pd is released successfully, negative number otherwise.
*/
int
-mlx5_os_dealloc_pd(void *pd)
+mlx5_os_pd_release(struct mlx5_common_device *cdev)
{
+ struct mlx5_pd *pd = cdev->pd;
+
if (!pd)
return -EINVAL;
- mlx5_devx_cmd_destroy(((struct mlx5_pd *)pd)->obj);
+ mlx5_devx_cmd_destroy(pd->obj);
mlx5_free(pd);
return 0;
}
@@ -47,14 +64,14 @@ mlx5_os_dealloc_pd(void *pd)
/**
* Allocate Protection Domain object and extract its pdn using DV API.
*
- * @param[out] dev
+ * @param[out] cdev
* Pointer to the mlx5 device.
*
* @return
* 0 on success, a negative value otherwise.
*/
int
-mlx5_os_pd_create(struct mlx5_common_device *cdev)
+mlx5_os_pd_prepare(struct mlx5_common_device *cdev)
{
struct mlx5_pd *pd;
diff --git a/drivers/common/mlx5/windows/mlx5_common_os.h
b/drivers/common/mlx5/windows/mlx5_common_os.h
index 3afce56cd9..a6a1b6890f 100644
--- a/drivers/common/mlx5/windows/mlx5_common_os.h
+++ b/drivers/common/mlx5/windows/mlx5_common_os.h
@@ -248,7 +248,6 @@ mlx5_os_devx_subscribe_devx_event(void *eventc,
return -ENOTSUP;
}
-int mlx5_os_dealloc_pd(void *pd);
__rte_internal
void *mlx5_os_umem_reg(void *ctx, void *addr, size_t size, uint32_t access);
__rte_internal
--
2.25.1
