From: Xueming Li <xuemi...@nvidia.com>
To support multi-thread flow insertion/removal, this patch uses thread
safe hash list API for flow table cache hash list.
Signed-off-by: Xueming Li <xuemi...@nvidia.com>
Acked-by: Matan Azrad <ma...@nvidia.com>
---
drivers/net/mlx5/mlx5.c | 102 ++++---------------------
drivers/net/mlx5/mlx5.h | 2 +-
drivers/net/mlx5/mlx5_flow.h | 17 +++++
drivers/net/mlx5/mlx5_flow_dv.c | 164 ++++++++++++++++++++--------------------
4 files changed, 116 insertions(+), 169 deletions(-)
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index da043e2..fa769cd 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -1003,7 +1003,7 @@ struct mlx5_dev_ctx_shared *
}
/**
- * Destroy table hash list and all the root entries per domain.
+ * Destroy table hash list.
*
* @param[in] priv
* Pointer to the private device data structure.
@@ -1012,46 +1012,9 @@ struct mlx5_dev_ctx_shared *
mlx5_free_table_hash_list(struct mlx5_priv *priv)
{
struct mlx5_dev_ctx_shared *sh = priv->sh;
- struct mlx5_flow_tbl_data_entry *tbl_data;
- union mlx5_flow_tbl_key table_key = {
- {
- .table_id = 0,
- .reserved = 0,
- .domain = 0,
- .direction = 0,
- }
- };
- struct mlx5_hlist_entry *pos;
if (!sh->flow_tbls)
return;
- pos = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL);
- if (pos) {
- tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry,
- entry);
- MLX5_ASSERT(tbl_data);
- mlx5_hlist_remove(sh->flow_tbls, pos);
- mlx5_free(tbl_data);
- }
- table_key.direction = 1;
- pos = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL);
- if (pos) {
- tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry,
- entry);
- MLX5_ASSERT(tbl_data);
- mlx5_hlist_remove(sh->flow_tbls, pos);
- mlx5_free(tbl_data);
- }
- table_key.direction = 0;
- table_key.domain = 1;
- pos = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL);
- if (pos) {
- tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry,
- entry);
- MLX5_ASSERT(tbl_data);
- mlx5_hlist_remove(sh->flow_tbls, pos);
- mlx5_free(tbl_data);
- }
mlx5_hlist_destroy(sh->flow_tbls);
}
@@ -1066,80 +1029,45 @@ struct mlx5_dev_ctx_shared *
* Zero on success, positive error code otherwise.
*/
int
-mlx5_alloc_table_hash_list(struct mlx5_priv *priv)
+mlx5_alloc_table_hash_list(struct mlx5_priv *priv __rte_unused)
{
+ int err = 0;
+ /* Tables are only used in DV and DR modes. */
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
struct mlx5_dev_ctx_shared *sh = priv->sh;
char s[MLX5_HLIST_NAMESIZE];
- int err = 0;
MLX5_ASSERT(sh);
snprintf(s, sizeof(s), "%s_flow_table", priv->sh->ibdev_name);
sh->flow_tbls = mlx5_hlist_create(s, MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE,
- 0, 0, NULL, NULL, NULL);
+ 0, 0, flow_dv_tbl_create_cb, NULL,
+ flow_dv_tbl_remove_cb);
if (!sh->flow_tbls) {
DRV_LOG(ERR, "flow tables with hash creation failed.");
err = ENOMEM;
return err;
}
+ sh->flow_tbls->ctx = sh;
#ifndef HAVE_MLX5DV_DR
+ struct rte_flow_error error;
+ struct rte_eth_dev *dev = &rte_eth_devices[priv->dev_data->port_id];
+
/*
* In case we have not DR support, the zero tables should be created
* because DV expect to see them even if they cannot be created by
* RDMA-CORE.
*/
- union mlx5_flow_tbl_key table_key = {
- {
- .table_id = 0,
- .reserved = 0,
- .domain = 0,
- .direction = 0,
- }
- };
- struct mlx5_flow_tbl_data_entry *tbl_data = mlx5_malloc(MLX5_MEM_ZERO,
- sizeof(*tbl_data), 0,
- SOCKET_ID_ANY);
-
- if (!tbl_data) {
- err = ENOMEM;
- goto error;
- }
- tbl_data->entry.key = table_key.v64;
- err = mlx5_hlist_insert(sh->flow_tbls, &tbl_data->entry);
- if (err)
- goto error;
- rte_atomic32_init(&tbl_data->tbl.refcnt);
- rte_atomic32_inc(&tbl_data->tbl.refcnt);
- table_key.direction = 1;
- tbl_data = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*tbl_data), 0,
- SOCKET_ID_ANY);
- if (!tbl_data) {
+ if (!flow_dv_tbl_resource_get(dev, 0, 0, 0, 1, &error) ||
+ !flow_dv_tbl_resource_get(dev, 0, 1, 0, 1, &error) ||
+ !flow_dv_tbl_resource_get(dev, 0, 0, 1, 1, &error)) {
err = ENOMEM;
goto error;
}
- tbl_data->entry.key = table_key.v64;
- err = mlx5_hlist_insert(sh->flow_tbls, &tbl_data->entry);
- if (err)
- goto error;
- rte_atomic32_init(&tbl_data->tbl.refcnt);
- rte_atomic32_inc(&tbl_data->tbl.refcnt);
- table_key.direction = 0;
- table_key.domain = 1;
- tbl_data = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*tbl_data), 0,
- SOCKET_ID_ANY);
- if (!tbl_data) {
- err = ENOMEM;
- goto error;
- }
- tbl_data->entry.key = table_key.v64;
- err = mlx5_hlist_insert(sh->flow_tbls, &tbl_data->entry);
- if (err)
- goto error;
- rte_atomic32_init(&tbl_data->tbl.refcnt);
- rte_atomic32_inc(&tbl_data->tbl.refcnt);
return err;
error:
mlx5_free_table_hash_list(priv);
#endif /* HAVE_MLX5DV_DR */
+#endif
return err;
}
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index ed16b0d..ae6d37d 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -496,7 +496,7 @@ struct mlx5_dev_shared_port {
struct {
/* Table ID should be at the lowest address. */
uint32_t table_id; /**< ID of the table. */
- uint16_t reserved; /**< must be zero for comparison. */
+ uint16_t dummy; /**< Dummy table for DV API. */
uint8_t domain; /**< 1 - FDB, 0 - NIC TX/RX. */
uint8_t direction; /**< 1 - egress, 0 - ingress. */
};
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index 3d325b2..8d12b5d 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -381,6 +381,13 @@ enum mlx5_flow_fate_type {
MLX5_FLOW_FATE_MAX,
};
+/* Hash list callback context */
+struct mlx5_flow_cb_ctx {
+ struct rte_eth_dev *dev;
+ struct rte_flow_error *error;
+ void *data;
+};
+
/* Matcher PRM representation */
struct mlx5_flow_dv_match_params {
size_t size;
@@ -1149,4 +1156,14 @@ int mlx5_flow_destroy_policer_rules(struct rte_eth_dev
*dev,
int mlx5_flow_meter_flush(struct rte_eth_dev *dev,
struct rte_mtr_error *error);
int mlx5_flow_dv_discover_counter_offset_support(struct rte_eth_dev *dev);
+
+/* Hash list callbacks for flow tables: */
+struct mlx5_hlist_entry *flow_dv_tbl_create_cb(struct mlx5_hlist *list,
+ uint64_t key, void *entry_ctx);
+void flow_dv_tbl_remove_cb(struct mlx5_hlist *list,
+ struct mlx5_hlist_entry *entry);
+struct mlx5_flow_tbl_resource *flow_dv_tbl_resource_get(struct rte_eth_dev
*dev,
+ uint32_t table_id, uint8_t egress, uint8_t transfer,
+ uint8_t dummy, struct rte_flow_error *error);
+
#endif /* RTE_PMD_MLX5_FLOW_H_ */
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 43d16b4..18a52b0 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -7800,54 +7800,18 @@ struct field_modify_info modify_tcp[] = {
}
-/**
- * Get a flow table.
- *
- * @param[in, out] dev
- * Pointer to rte_eth_dev structure.
- * @param[in] table_id
- * Table id to use.
- * @param[in] egress
- * Direction of the table.
- * @param[in] transfer
- * E-Switch or NIC flow.
- * @param[out] error
- * pointer to error structure.
- *
- * @return
- * Returns tables resource based on the index, NULL in case of failed.
- */
-static struct mlx5_flow_tbl_resource *
-flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
- uint32_t table_id, uint8_t egress,
- uint8_t transfer,
- struct rte_flow_error *error)
+struct mlx5_hlist_entry *
+flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *ctx)
{
- struct mlx5_priv *priv = dev->data->dev_private;
- struct mlx5_dev_ctx_shared *sh = priv->sh;
- struct mlx5_flow_tbl_resource *tbl;
- union mlx5_flow_tbl_key table_key = {
- {
- .table_id = table_id,
- .reserved = 0,
- .domain = !!transfer,
- .direction = !!egress,
- }
- };
- struct mlx5_hlist_entry *pos = mlx5_hlist_lookup(sh->flow_tbls,
- table_key.v64, NULL);
+ struct mlx5_dev_ctx_shared *sh = list->ctx;
struct mlx5_flow_tbl_data_entry *tbl_data;
+ struct rte_flow_error *error = ctx;
+ union mlx5_flow_tbl_key key = { .v64 = key64 };
+ struct mlx5_flow_tbl_resource *tbl;
+ void *domain;
uint32_t idx = 0;
int ret;
- void *domain;
- if (pos) {
- tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry,
- entry);
- tbl = &tbl_data->tbl;
- rte_atomic32_inc(&tbl->refcnt);
- return tbl;
- }
tbl_data = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_JUMP], &idx);
if (!tbl_data) {
rte_flow_error_set(error, ENOMEM,
@@ -7858,14 +7822,15 @@ struct field_modify_info modify_tcp[] = {
}
tbl_data->idx = idx;
tbl = &tbl_data->tbl;
- pos = &tbl_data->entry;
- if (transfer)
+ if (key.dummy)
+ return &tbl_data->entry;
+ if (key.domain)
domain = sh->fdb_domain;
- else if (egress)
+ else if (key.direction)
domain = sh->tx_domain;
else
domain = sh->rx_domain;
- ret = mlx5_flow_os_create_flow_tbl(domain, table_id, &tbl->obj);
+ ret = mlx5_flow_os_create_flow_tbl(domain, key.table_id, &tbl->obj);
if (ret) {
rte_flow_error_set(error, ENOMEM,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
@@ -7873,12 +7838,7 @@ struct field_modify_info modify_tcp[] = {
mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
return NULL;
}
- /*
- * No multi-threads now, but still better to initialize the reference
- * count before insert it into the hash list.
- */
- rte_atomic32_init(&tbl->refcnt);
- if (table_id) {
+ if (key.table_id) {
ret = mlx5_flow_os_create_flow_action_dest_flow_tbl
(tbl->obj, &tbl_data->jump.action);
if (ret) {
@@ -7891,17 +7851,69 @@ struct field_modify_info modify_tcp[] = {
return NULL;
}
}
- pos->key = table_key.v64;
- ret = !mlx5_hlist_insert(sh->flow_tbls, pos);
- if (ret < 0) {
- rte_flow_error_set(error, -ret,
+ return &tbl_data->entry;
+}
+
+/**
+ * Get a flow table.
+ *
+ * @param[in, out] dev
+ * Pointer to rte_eth_dev structure.
+ * @param[in] table_id
+ * Table id to use.
+ * @param[in] egress
+ * Direction of the table.
+ * @param[in] transfer
+ * E-Switch or NIC flow.
+ * @param[in] dummy
+ * Dummy entry for dv API.
+ * @param[out] error
+ * pointer to error structure.
+ *
+ * @return
+ * Returns tables resource based on the index, NULL in case of failed.
+ */
+struct mlx5_flow_tbl_resource *
+flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
+ uint32_t table_id, uint8_t egress,
+ uint8_t transfer, uint8_t dummy,
+ struct rte_flow_error *error)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+ union mlx5_flow_tbl_key table_key = {
+ {
+ .table_id = table_id,
+ .dummy = dummy,
+ .domain = !!transfer,
+ .direction = !!egress,
+ }
+ };
+ struct mlx5_hlist_entry *entry;
+ struct mlx5_flow_tbl_data_entry *tbl_data;
+
+ entry = mlx5_hlist_register(priv->sh->flow_tbls, table_key.v64, NULL);
+ if (!entry) {
+ rte_flow_error_set(error, ENOMEM,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
- "cannot insert flow table data entry");
- mlx5_flow_os_destroy_flow_tbl(tbl->obj);
- mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
+ "cannot get table");
+ return NULL;
}
- rte_atomic32_inc(&tbl->refcnt);
- return tbl;
+ tbl_data = container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
+ return &tbl_data->tbl;
+}
+
+void
+flow_dv_tbl_remove_cb(struct mlx5_hlist *list,
+ struct mlx5_hlist_entry *entry)
+{
+ struct mlx5_dev_ctx_shared *sh = list->ctx;
+ struct mlx5_flow_tbl_data_entry *tbl_data =
+ container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
+
+ MLX5_ASSERT(entry && sh);
+ if (tbl_data->tbl.obj)
+ mlx5_flow_os_destroy_flow_tbl(tbl_data->tbl.obj);
+ mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], tbl_data->idx);
}
/**
@@ -7926,18 +7938,7 @@ struct field_modify_info modify_tcp[] = {
if (!tbl)
return 0;
- if (rte_atomic32_dec_and_test(&tbl->refcnt)) {
- struct mlx5_hlist_entry *pos = &tbl_data->entry;
-
- mlx5_flow_os_destroy_flow_tbl(tbl->obj);
- tbl->obj = NULL;
- /* remove the entry from the hash list and free memory. */
- mlx5_hlist_remove(sh->flow_tbls, pos);
- mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_JUMP],
- tbl_data->idx);
- return 0;
- }
- return 1;
+ return mlx5_hlist_unregister(sh->flow_tbls, &tbl_data->entry);
}
/**
@@ -7976,7 +7977,7 @@ struct field_modify_info modify_tcp[] = {
int ret;
tbl = flow_dv_tbl_resource_get(dev, key->table_id, key->direction,
- key->domain, error);
+ key->domain, 0, error);
if (!tbl)
return -rte_errno; /* No need to refill the error info */
tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
@@ -8471,7 +8472,7 @@ struct field_modify_info modify_tcp[] = {
*cache_resource = *resource;
/* Create normal path table level */
tbl = flow_dv_tbl_resource_get(dev, next_ft_id,
- attr->egress, attr->transfer, error);
+ attr->egress, attr->transfer, 0, error);
if (!tbl) {
rte_flow_error_set(error, ENOMEM,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
@@ -9377,7 +9378,8 @@ struct field_modify_info modify_tcp[] = {
return ret;
tbl = flow_dv_tbl_resource_get(dev, table,
attr->egress,
- attr->transfer, error);
+ attr->transfer, 0,
+ error);
if (!tbl)
return rte_flow_error_set
(error, errno,
@@ -10771,7 +10773,7 @@ struct field_modify_info modify_tcp[] = {
dtb = &mtb->ingress;
/* Create the meter table with METER level. */
dtb->tbl = flow_dv_tbl_resource_get(dev, MLX5_FLOW_TABLE_LEVEL_METER,
- egress, transfer, &error);
+ egress, transfer, 0, &error);
if (!dtb->tbl) {
DRV_LOG(ERR, "Failed to create meter policer table.");
return -1;
@@ -10779,7 +10781,7 @@ struct field_modify_info modify_tcp[] = {
/* Create the meter suffix table with SUFFIX level. */
dtb->sfx_tbl = flow_dv_tbl_resource_get(dev,
MLX5_FLOW_TABLE_LEVEL_SUFFIX,
- egress, transfer, &error);
+ egress, transfer, 0, &error);
if (!dtb->sfx_tbl) {
DRV_LOG(ERR, "Failed to create meter suffix table.");
return -1;
@@ -11098,10 +11100,10 @@ struct field_modify_info modify_tcp[] = {
void *flow = NULL;
int i, ret = -1;
- tbl = flow_dv_tbl_resource_get(dev, 0, 0, 0, NULL);
+ tbl = flow_dv_tbl_resource_get(dev, 0, 0, 0, 0, NULL);
if (!tbl)
goto err;
- dest_tbl = flow_dv_tbl_resource_get(dev, 1, 0, 0, NULL);
+ dest_tbl = flow_dv_tbl_resource_get(dev, 1, 0, 0, 0, NULL);
if (!dest_tbl)
goto err;
dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0x4);
--
1.8.3.1
