We are going to share the Direct Rules and Direct Verbs flow
device data structures between master and representors in the
E-Switch configurations over multiport IB device.
The code of initializing and destroying these data is
moved to dedicated routines, this is just a preparation
step for actual data sharing.
Signed-off-by: Viacheslav Ovsiienko <viachesl...@mellanox.com>
---
drivers/net/mlx5/mlx5.c | 117 +++++++++++++++++++++++++++++++++++++++++-------
drivers/net/mlx5/mlx5.h | 4 ++
2 files changed, 106 insertions(+), 15 deletions(-)
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 65aa9cf..62cba00 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -303,6 +303,101 @@ struct mlx5_dev_spawn_data {
}
/**
+ * Initialize DR related data within private structure.
+ * Routine checks the reference counter and does actual
+ * resources creation/iniialization only if counter is zero.
+ *
+ * @param[in] priv
+ * Pointer to the private device data structure.
+ *
+ * @return
+ * Zero on success, positive error code otherwise.
+ */
+static int
+mlx5_alloc_shared_dr(struct mlx5_priv *priv)
+{
+#ifdef HAVE_MLX5DV_DR
+ struct mlx5_ibv_shared *sh = priv->sh;
+ int err = 0;
+ void *ns;
+
+ assert(sh);
+ if (sh->dv_refcnt) {
+ /* Shared DV/DR structures is already initialized. */
+ sh->dv_refcnt++;
+ priv->dr_shared = 1;
+ return 0;
+ }
+ /* Reference counter is zero, we should initialize structures. */
+ ns = mlx5dv_dr_create_ns(sh->ctx, MLX5DV_DR_NS_DOMAIN_INGRESS_BYPASS);
+ if (!ns) {
+ DRV_LOG(ERR, "ingress mlx5dv_dr_create_ns failed");
+ err = errno;
+ goto error;
+ }
+ priv->rx_ns = ns;
+ ns = mlx5dv_dr_create_ns(sh->ctx, MLX5DV_DR_NS_DOMAIN_EGRESS_BYPASS);
+ if (!ns) {
+ DRV_LOG(ERR, "egress mlx5dv_dr_create_ns failed");
+ err = errno;
+ goto error;
+ }
+ priv->tx_ns = ns;
+ sh->dv_refcnt++;
+ priv->dr_shared = 1;
+ return 0;
+
+error:
+ /* Rollback the created objects. */
+ if (priv->rx_ns) {
+ mlx5dv_dr_destroy_ns(priv->rx_ns);
+ priv->rx_ns = NULL;
+ }
+ if (priv->tx_ns) {
+ mlx5dv_dr_destroy_ns(priv->tx_ns);
+ priv->tx_ns = NULL;
+ }
+ return err;
+#else
+ (void)priv;
+ return 0;
+#endif
+}
+
+/**
+ * Destroy DR related data within private structure.
+ *
+ * @param[in] priv
+ * Pointer to the private device data structure.
+ */
+static void
+mlx5_free_shared_dr(struct mlx5_priv *priv)
+{
+#ifdef HAVE_MLX5DV_DR
+ struct mlx5_ibv_shared *sh;
+
+ if (!priv->dr_shared)
+ return;
+ priv->dr_shared = 0;
+ sh = priv->sh;
+ assert(sh);
+ assert(sh->dv_refcnt);
+ if (sh->dv_refcnt && --sh->dv_refcnt)
+ return;
+ if (priv->rx_ns) {
+ mlx5dv_dr_destroy_ns(priv->rx_ns);
+ priv->rx_ns = NULL;
+ }
+ if (priv->tx_ns) {
+ mlx5dv_dr_destroy_ns(priv->tx_ns);
+ priv->tx_ns = NULL;
+ }
+#else
+ (void)priv;
+#endif
+}
+
+/**
* Initialize shared data between primary and secondary process.
*
* A memzone is reserved by primary process and secondary processes attach to
@@ -495,6 +590,7 @@ struct mlx5_dev_spawn_data {
mlx5_mprq_free_mp(dev);
mlx5_mr_release(dev);
assert(priv->sh);
+ mlx5_free_shared_dr(priv);
if (priv->sh)
mlx5_free_shared_ibctx(priv->sh);
priv->sh = NULL;
@@ -1483,22 +1579,11 @@ struct mlx5_dev_spawn_data {
priv->tcf_context = NULL;
}
}
-#ifdef HAVE_MLX5DV_DR
- priv->rx_ns = mlx5dv_dr_create_ns
- (sh->ctx, MLX5DV_DR_NS_DOMAIN_INGRESS_BYPASS);
- if (priv->rx_ns == NULL) {
- DRV_LOG(ERR, "mlx5dv_dr_create_ns failed");
- err = errno;
- goto error;
- }
- priv->tx_ns = mlx5dv_dr_create_ns(sh->ctx,
- MLX5DV_DR_NS_DOMAIN_EGRESS_BYPASS);
- if (priv->tx_ns == NULL) {
- DRV_LOG(ERR, "mlx5dv_dr_create_ns failed");
- err = errno;
+ if (config.dv_flow_en) {
+ err = mlx5_alloc_shared_dr(priv);
+ if (err)
goto error;
- }
-#endif
+ }
TAILQ_INIT(&priv->flows);
TAILQ_INIT(&priv->ctrl_flows);
/* Hint libmlx5 to use PMD allocator for data plane resources */
@@ -1550,6 +1635,8 @@ struct mlx5_dev_spawn_data {
return eth_dev;
error:
if (priv) {
+ if (priv->sh)
+ mlx5_free_shared_dr(priv);
if (priv->nl_socket_route >= 0)
close(priv->nl_socket_route);
if (priv->nl_socket_rdma >= 0)
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 5a7597e..a6f6ef4 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -253,6 +253,9 @@ struct mlx5_ibv_shared {
char ibdev_name[IBV_SYSFS_NAME_MAX]; /* IB device name. */
char ibdev_path[IBV_SYSFS_PATH_MAX]; /* IB device path for secondary */
struct ibv_device_attr_ex device_attr; /* Device properties. */
+ /* Shared DV/DR flow data section. */
+ uint32_t dv_refcnt; /* DV/DR data reference counter. */
+ /* Shared interrupt handler section. */
pthread_mutex_t intr_mutex; /* Interrupt config mutex. */
uint32_t intr_cnt; /* Interrupt handler reference counter. */
struct rte_intr_handle intr_handle; /* Interrupt handler for device. */
@@ -284,6 +287,7 @@ struct mlx5_priv {
unsigned int isolated:1; /* Whether isolated mode is enabled. */
unsigned int representor:1; /* Device is a port representor. */
unsigned int master:1; /* Device is a E-Switch master. */
+ unsigned int dr_shared:1; /* DV/DR data is shared. */
uint16_t domain_id; /* Switch domain identifier. */
uint16_t vport_id; /* Associated VF vport index (if any). */
int32_t representor_id; /* Port representor identifier. */
--
1.8.3.1
