Thursday, March 21, 2019 10:11 AM, Viacheslav Ovsiienko:
> Subject: [PATCH 10/14] net/mlx5: switch to the shared IB device context
>
> The code is updated to use the shared IB device context and device handles.
> The IB device context is shared between reprentors created over the single
> multiport IB device. All Verbs and DevX objects will be created whithin this
> shared context.
>
> Signed-off-by: Viacheslav Ovsiienko <viachesl...@mellanox.com>
Acked-by: Shahaf Shuler <shah...@mellanox.com>
> ---
> drivers/net/mlx5/mlx5.c | 7 +++----
> drivers/net/mlx5/mlx5.h | 1 -
> drivers/net/mlx5/mlx5_ethdev.c | 13 +++++++------
> drivers/net/mlx5/mlx5_flow_dv.c | 9 +++++----
> drivers/net/mlx5/mlx5_flow_verbs.c | 6 ++++--
> drivers/net/mlx5/mlx5_rxq.c | 23 ++++++++++++-----------
> drivers/net/mlx5/mlx5_socket.c | 4 ++--
> drivers/net/mlx5/mlx5_trigger.c | 2 +-
> drivers/net/mlx5/mlx5_txq.c | 4 ++--
> 9 files changed, 36 insertions(+), 33 deletions(-)
>
> diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index
> a0f224b..312c42b 100644
> --- a/drivers/net/mlx5/mlx5.c
> +++ b/drivers/net/mlx5/mlx5.c
> @@ -407,7 +407,7 @@ struct mlx5_dev_spawn_data {
>
> DRV_LOG(DEBUG, "port %u closing device \"%s\"",
> dev->data->port_id,
> - ((priv->ctx != NULL) ? priv->ctx->device->name : ""));
> + ((priv->sh->ctx != NULL) ? priv->sh->ctx->device->name :
> ""));
> /* In case mlx5_dev_stop() has not been called. */
> mlx5_dev_interrupt_handler_uninstall(dev);
> mlx5_traffic_disable(dev);
> @@ -1111,7 +1111,6 @@ struct mlx5_dev_spawn_data {
> goto error;
> }
> priv->sh = sh;
> - priv->ctx = sh->ctx;
> priv->ibv_port = spawn->ibv_port;
> priv->mtu = ETHER_MTU;
> #ifndef RTE_ARCH_64
> @@ -1349,7 +1348,7 @@ struct mlx5_dev_spawn_data {
> }
> }
> #ifdef HAVE_MLX5DV_DR
> - ns = mlx5dv_dr_create_ns(ctx,
> + ns = mlx5dv_dr_create_ns(sh->ctx,
>
> MLX5DV_DR_NS_DOMAIN_INGRESS_BYPASS);
> if (ns == NULL) {
> DRV_LOG(ERR, "mlx5dv_dr_create_ns failed"); @@ -
> 1357,7 +1356,7 @@ struct mlx5_dev_spawn_data {
> goto error;
> }
> priv->rx_ns = ns;
> - ns = mlx5dv_dr_create_ns(ctx,
> + ns = mlx5dv_dr_create_ns(sh->ctx,
>
> MLX5DV_DR_NS_DOMAIN_EGRESS_BYPASS);
> if (ns == NULL) {
> DRV_LOG(ERR, "mlx5dv_dr_create_ns failed"); diff --
> git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index
> efa3a3c..d816d24 100644
> --- a/drivers/net/mlx5/mlx5.h
> +++ b/drivers/net/mlx5/mlx5.h
> @@ -226,7 +226,6 @@ struct mlx5_priv {
> struct rte_eth_dev_data *dev_data; /* Pointer to device data. */
> struct mlx5_ibv_shared *sh; /* Shared IB device context. */
> uint32_t ibv_port; /* IB device port number. */
> - struct ibv_context *ctx; /* Verbs context. */
> struct ether_addr mac[MLX5_MAX_MAC_ADDRESSES]; /* MAC
> addresses. */
> BITFIELD_DECLARE(mac_own, uint64_t,
> MLX5_MAX_MAC_ADDRESSES);
> /* Bit-field of MAC addresses owned by the PMD. */ diff --git
> a/drivers/net/mlx5/mlx5_ethdev.c b/drivers/net/mlx5/mlx5_ethdev.c index
> 69f238b..1b2173b 100644
> --- a/drivers/net/mlx5/mlx5_ethdev.c
> +++ b/drivers/net/mlx5/mlx5_ethdev.c
> @@ -1055,7 +1055,7 @@ int mlx5_fw_version_get(struct rte_eth_dev *dev,
> char *fw_ver, size_t fw_size)
> }
> /* Read all message and acknowledge them. */
> for (;;) {
> - if (mlx5_glue->get_async_event(priv->ctx, &event))
> + if (mlx5_glue->get_async_event(priv->sh->ctx, &event))
> break;
> if ((event.event_type == IBV_EVENT_PORT_ACTIVE ||
> event.event_type == IBV_EVENT_PORT_ERR) &&
> @@ -1142,12 +1142,13 @@ int mlx5_fw_version_get(struct rte_eth_dev
> *dev, char *fw_ver, size_t fw_size)
> mlx5_dev_interrupt_handler_install(struct rte_eth_dev *dev) {
> struct mlx5_priv *priv = dev->data->dev_private;
> + struct ibv_context *ctx = priv->sh->ctx;
> int ret;
> int flags;
>
> - assert(priv->ctx->async_fd > 0);
> - flags = fcntl(priv->ctx->async_fd, F_GETFL);
> - ret = fcntl(priv->ctx->async_fd, F_SETFL, flags | O_NONBLOCK);
> + assert(ctx->async_fd > 0);
> + flags = fcntl(ctx->async_fd, F_GETFL);
> + ret = fcntl(ctx->async_fd, F_SETFL, flags | O_NONBLOCK);
> if (ret) {
> DRV_LOG(INFO,
> "port %u failed to change file descriptor async event"
> @@ -1158,7 +1159,7 @@ int mlx5_fw_version_get(struct rte_eth_dev *dev,
> char *fw_ver, size_t fw_size)
> }
> if (dev->data->dev_conf.intr_conf.lsc ||
> dev->data->dev_conf.intr_conf.rmv) {
> - priv->intr_handle.fd = priv->ctx->async_fd;
> + priv->intr_handle.fd = ctx->async_fd;
> priv->intr_handle.type = RTE_INTR_HANDLE_EXT;
> rte_intr_callback_register(&priv->intr_handle,
> mlx5_dev_interrupt_handler, dev);
> @@ -1303,7 +1304,7 @@ int mlx5_fw_version_get(struct rte_eth_dev *dev,
> char *fw_ver, size_t fw_size)
> struct ibv_device_attr device_attr;
> struct mlx5_priv *priv = dev->data->dev_private;
>
> - if (mlx5_glue->query_device(priv->ctx, &device_attr) == EIO)
> + if (mlx5_glue->query_device(priv->sh->ctx, &device_attr) == EIO)
> return 1;
> return 0;
> }
> diff --git a/drivers/net/mlx5/mlx5_flow_dv.c
> b/drivers/net/mlx5/mlx5_flow_dv.c index 44b36ee..b8943da 100644
> --- a/drivers/net/mlx5/mlx5_flow_dv.c
> +++ b/drivers/net/mlx5/mlx5_flow_dv.c
> @@ -840,7 +840,7 @@ struct field_modify_info modify_tcp[] = {
> *cache_resource = *resource;
> cache_resource->verbs_action =
> mlx5_glue->dv_create_flow_action_packet_reformat
> - (priv->ctx, cache_resource->reformat_type,
> + (priv->sh->ctx, cache_resource->reformat_type,
> cache_resource->ft_type, ns, cache_resource-
> >flags,
> cache_resource->size,
> (cache_resource->size ? cache_resource->buf :
> NULL)); @@ -1569,7 +1569,8 @@ struct field_modify_info modify_tcp[] = {
> *cache_resource = *resource;
> cache_resource->verbs_action =
> mlx5_glue->dv_create_flow_action_modify_header
> - (priv->ctx, cache_resource->ft_type,
> + (priv->sh->ctx,
> + cache_resource->ft_type,
> ns, 0,
> cache_resource->actions_num *
> sizeof(cache_resource->actions[0]),
> @@ -1629,7 +1630,7 @@ struct field_modify_info modify_tcp[] = {
> ret = -ENOMEM;
> goto error_exit;
> }
> - ret = mlx5_devx_cmd_flow_counter_alloc(priv->ctx, dcs);
> + ret = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, dcs);
> if (ret)
> goto error_exit;
> struct mlx5_flow_counter tmpl = {
> @@ -2982,7 +2983,7 @@ struct field_modify_info modify_tcp[] = {
> if (matcher->egress)
> dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS;
> cache_matcher->matcher_object =
> - mlx5_glue->dv_create_flow_matcher(priv->ctx, &dv_attr,
> + mlx5_glue->dv_create_flow_matcher(priv->sh->ctx,
> &dv_attr,
> tbl->obj);
> if (!cache_matcher->matcher_object) {
> rte_free(cache_matcher);
> diff --git a/drivers/net/mlx5/mlx5_flow_verbs.c
> b/drivers/net/mlx5/mlx5_flow_verbs.c
> index 6c4f52f..9548d12 100644
> --- a/drivers/net/mlx5/mlx5_flow_verbs.c
> +++ b/drivers/net/mlx5/mlx5_flow_verbs.c
> @@ -56,10 +56,11 @@
> {
> #if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
> struct mlx5_priv *priv = dev->data->dev_private;
> + struct ibv_context ctx = priv->sh->ctx;
> struct ibv_counter_set_init_attr init = {
> .counter_set_id = counter->id};
>
> - counter->cs = mlx5_glue->create_counter_set(priv->ctx, &init);
> + counter->cs = mlx5_glue->create_counter_set(ctx, &init);
> if (!counter->cs) {
> rte_errno = ENOTSUP;
> return -ENOTSUP;
> @@ -67,12 +68,13 @@
> return 0;
> #elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
> struct mlx5_priv *priv = dev->data->dev_private;
> + struct ibv_context *ctx = priv->sh->ctx;
> struct ibv_counters_init_attr init = {0};
> struct ibv_counter_attach_attr attach;
> int ret;
>
> memset(&attach, 0, sizeof(attach));
> - counter->cs = mlx5_glue->create_counters(priv->ctx, &init);
> + counter->cs = mlx5_glue->create_counters(ctx, &init);
> if (!counter->cs) {
> rte_errno = ENOTSUP;
> return -ENOTSUP;
> diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
> index fd1c3a2..dcb97c2 100644
> --- a/drivers/net/mlx5/mlx5_rxq.c
> +++ b/drivers/net/mlx5/mlx5_rxq.c
> @@ -799,7 +799,7 @@ struct mlx5_rxq_ibv *
> }
> tmpl->rxq_ctrl = rxq_ctrl;
> if (rxq_ctrl->irq) {
> - tmpl->channel = mlx5_glue->create_comp_channel(priv-
> >ctx);
> + tmpl->channel = mlx5_glue->create_comp_channel(priv-
> >sh->ctx);
> if (!tmpl->channel) {
> DRV_LOG(ERR, "port %u: comp channel creation
> failure",
> dev->data->port_id);
> @@ -848,7 +848,7 @@ struct mlx5_rxq_ibv *
> }
> #endif
> tmpl->cq = mlx5_glue->cq_ex_to_cq
> - (mlx5_glue->dv_create_cq(priv->ctx, &attr.cq.ibv,
> + (mlx5_glue->dv_create_cq(priv->sh->ctx, &attr.cq.ibv,
> &attr.cq.mlx5));
> if (tmpl->cq == NULL) {
> DRV_LOG(ERR, "port %u Rx queue %u CQ creation failure",
> @@ -905,10 +905,10 @@ struct mlx5_rxq_ibv *
> .two_byte_shift_en =
> MLX5_MPRQ_TWO_BYTE_SHIFT,
> };
> }
> - tmpl->wq = mlx5_glue->dv_create_wq(priv->ctx, &attr.wq.ibv,
> + tmpl->wq = mlx5_glue->dv_create_wq(priv->sh->ctx, &attr.wq.ibv,
> &attr.wq.mlx5);
> #else
> - tmpl->wq = mlx5_glue->create_wq(priv->ctx, &attr.wq.ibv);
> + tmpl->wq = mlx5_glue->create_wq(priv->sh->ctx, &attr.wq.ibv);
> #endif
> if (tmpl->wq == NULL) {
> DRV_LOG(ERR, "port %u Rx queue %u WQ creation failure",
> @@ -1643,7 +1643,7 @@ struct mlx5_ind_table_ibv *
> for (j = 0; i != (unsigned int)(1 << wq_n); ++i, ++j)
> wq[i] = wq[j];
> ind_tbl->ind_table = mlx5_glue->create_rwq_ind_table
> - (priv->ctx,
> + (priv->sh->ctx,
> &(struct ibv_rwq_ind_table_init_attr){
> .log_ind_tbl_size = wq_n,
> .ind_tbl = wq,
> @@ -1817,7 +1817,7 @@ struct mlx5_hrxq *
> }
> #endif
> qp = mlx5_glue->dv_create_qp
> - (priv->ctx,
> + (priv->sh->ctx,
> &(struct ibv_qp_init_attr_ex){
> .qp_type = IBV_QPT_RAW_PACKET,
> .comp_mask =
> @@ -1836,7 +1836,7 @@ struct mlx5_hrxq *
> &qp_init_attr);
> #else
> qp = mlx5_glue->create_qp_ex
> - (priv->ctx,
> + (priv->sh->ctx,
> &(struct ibv_qp_init_attr_ex){
> .qp_type = IBV_QPT_RAW_PACKET,
> .comp_mask =
> @@ -1988,20 +1988,21 @@ struct mlx5_rxq_ibv *
> mlx5_rxq_ibv_drop_new(struct rte_eth_dev *dev) {
> struct mlx5_priv *priv = dev->data->dev_private;
> + struct ibv_context *ctx = priv->sh->ctx;
> struct ibv_cq *cq;
> struct ibv_wq *wq = NULL;
> struct mlx5_rxq_ibv *rxq;
>
> if (priv->drop_queue.rxq)
> return priv->drop_queue.rxq;
> - cq = mlx5_glue->create_cq(priv->ctx, 1, NULL, NULL, 0);
> + cq = mlx5_glue->create_cq(ctx, 1, NULL, NULL, 0);
> if (!cq) {
> DEBUG("port %u cannot allocate CQ for drop queue",
> dev->data->port_id);
> rte_errno = errno;
> goto error;
> }
> - wq = mlx5_glue->create_wq(priv->ctx,
> + wq = mlx5_glue->create_wq(ctx,
> &(struct ibv_wq_init_attr){
> .wq_type = IBV_WQT_RQ,
> .max_wr = 1,
> @@ -2078,7 +2079,7 @@ struct mlx5_ind_table_ibv *
> if (!rxq)
> return NULL;
> tmpl.ind_table = mlx5_glue->create_rwq_ind_table
> - (priv->ctx,
> + (priv->sh->ctx,
> &(struct ibv_rwq_ind_table_init_attr){
> .log_ind_tbl_size = 0,
> .ind_tbl = &rxq->wq,
> @@ -2145,7 +2146,7 @@ struct mlx5_hrxq *
> ind_tbl = mlx5_ind_table_ibv_drop_new(dev);
> if (!ind_tbl)
> return NULL;
> - qp = mlx5_glue->create_qp_ex(priv->ctx,
> + qp = mlx5_glue->create_qp_ex(priv->sh->ctx,
> &(struct ibv_qp_init_attr_ex){
> .qp_type = IBV_QPT_RAW_PACKET,
> .comp_mask =
> diff --git a/drivers/net/mlx5/mlx5_socket.c
> b/drivers/net/mlx5/mlx5_socket.c index 41cac3c..8fa6430 100644
> --- a/drivers/net/mlx5/mlx5_socket.c
> +++ b/drivers/net/mlx5/mlx5_socket.c
> @@ -185,9 +185,9 @@
> assert(cmsg != NULL);
> cmsg->cmsg_level = SOL_SOCKET;
> cmsg->cmsg_type = SCM_RIGHTS;
> - cmsg->cmsg_len = CMSG_LEN(sizeof(priv->ctx->cmd_fd));
> + cmsg->cmsg_len = CMSG_LEN(sizeof(priv->sh->ctx->cmd_fd));
> fd = (int *)CMSG_DATA(cmsg);
> - *fd = priv->ctx->cmd_fd;
> + *fd = priv->sh->ctx->cmd_fd;
> ret = sendmsg(conn_sock, &msg, 0);
> if (ret < 0)
> DRV_LOG(WARNING, "port %u cannot send response", diff -
> -git a/drivers/net/mlx5/mlx5_trigger.c b/drivers/net/mlx5/mlx5_trigger.c
> index 2137bdc..d13a1a1 100644
> --- a/drivers/net/mlx5/mlx5_trigger.c
> +++ b/drivers/net/mlx5/mlx5_trigger.c
> @@ -58,7 +58,7 @@
> goto error;
> }
> }
> - ret = mlx5_tx_uar_remap(dev, priv->ctx->cmd_fd);
> + ret = mlx5_tx_uar_remap(dev, priv->sh->ctx->cmd_fd);
> if (ret) {
> /* Adjust index for rollback. */
> i = priv->txqs_n - 1;
> diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c
> index 10a3040..25305b6 100644
> --- a/drivers/net/mlx5/mlx5_txq.c
> +++ b/drivers/net/mlx5/mlx5_txq.c
> @@ -392,7 +392,7 @@ struct mlx5_txq_ibv *
> ((desc / MLX5_TX_COMP_THRESH) - 1) : 1;
> if (is_empw_burst_func(tx_pkt_burst))
> cqe_n += MLX5_TX_COMP_THRESH_INLINE_DIV;
> - tmpl.cq = mlx5_glue->create_cq(priv->ctx, cqe_n, NULL, NULL, 0);
> + tmpl.cq = mlx5_glue->create_cq(priv->sh->ctx, cqe_n, NULL, NULL,
> 0);
> if (tmpl.cq == NULL) {
> DRV_LOG(ERR, "port %u Tx queue %u CQ creation failure",
> dev->data->port_id, idx);
> @@ -435,7 +435,7 @@ struct mlx5_txq_ibv *
> attr.init.max_tso_header = txq_ctrl->max_tso_header;
> attr.init.comp_mask |=
> IBV_QP_INIT_ATTR_MAX_TSO_HEADER;
> }
> - tmpl.qp = mlx5_glue->create_qp_ex(priv->ctx, &attr.init);
> + tmpl.qp = mlx5_glue->create_qp_ex(priv->sh->ctx, &attr.init);
> if (tmpl.qp == NULL) {
> DRV_LOG(ERR, "port %u Tx queue %u QP creation failure",
> dev->data->port_id, idx);
> --
> 1.8.3.1
