From: Shun Hao <sh...@nvidia.com>
This fixes the meter statistics issue that when using multiple meters,
only one meter has stats value.
To match the correct meter in policer table, now the meter_id is also
used in its match criteria, so only one color and one drop matcher are
needed. And both meter_id and flow_id will be written to related registers
in meter prefix flow.
Fixes: 46a5e6bc6a ("net/mlx5: prepare meter flow tables")
Cc: sta...@dpdk.org
Signed-off-by: Shun Hao <sh...@nvidia.com>
---
drivers/net/mlx5/linux/mlx5_os.c | 9 +-
drivers/net/mlx5/mlx5.c | 7 +-
drivers/net/mlx5/mlx5.h | 25 +-
drivers/net/mlx5/mlx5_flow.c | 182 +++++++----
drivers/net/mlx5/mlx5_flow.h | 40 +--
drivers/net/mlx5/mlx5_flow_dv.c | 489 ++++++++++++++++++-----------
drivers/net/mlx5/mlx5_flow_meter.c | 53 +++-
7 files changed, 512 insertions(+), 293 deletions(-)
diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index 5e3ae9f10e..c3ae1972de 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -316,7 +316,13 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
}
sh->tx_domain = domain;
#ifdef HAVE_MLX5DV_DR_ESWITCH
+ sh->esw_drop_action = mlx5_glue->dr_create_flow_action_drop();
if (priv->config.dv_esw_en) {
+ if (!sh->esw_drop_action) {
+ DRV_LOG(ERR, "Failed to create eswitch drop action");
+ err = errno;
+ goto error;
+ }
domain = mlx5_glue->dr_create_domain
(sh->ctx, MLX5DV_DR_DOMAIN_TYPE_FDB);
if (!domain) {
@@ -325,7 +331,6 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
goto error;
}
sh->fdb_domain = domain;
- sh->esw_drop_action = mlx5_glue->dr_create_flow_action_drop();
}
#endif
if (!sh->tunnel_hub)
@@ -1225,7 +1230,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
- 1 + REG_C_0;
priv->mtr_en = 1;
priv->mtr_reg_share =
- config->hca_attr.qos.flow_meter;
+ config->hca_attr.qos.flow_meter;
DRV_LOG(DEBUG, "The REG_C meter uses is %d",
priv->mtr_color_reg);
}
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index d9372f0a00..a1744b1017 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -275,10 +275,13 @@ static const struct mlx5_indexed_pool_config
mlx5_ipool_cfg[] = {
},
#endif
[MLX5_IPOOL_MTR] = {
+ /**
+ * The ipool index should grow continually from small to big,
+ * for meter idx, so not set grow_trunk to avoid meter index
+ * not jump continually.
+ */
.size = sizeof(struct mlx5_flow_meter),
.trunk_size = 64,
- .grow_trunk = 3,
- .grow_shift = 2,
.need_lock = 1,
.release_mem_en = 1,
.malloc = mlx5_malloc,
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index e4963bd107..05b6a082f5 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -187,6 +187,16 @@ struct mlx5_stats_ctrl {
/* Maximal number of segments to split. */
#define MLX5_MAX_RXQ_NSEG (1u << MLX5_MAX_LOG_RQ_SEGS)
+/* The bit size of one register. */
+#define MLX5_REG_BITS 32
+
+/* Idle bits for non-color usage in color register. */
+#define MLX5_MTR_IDLE_BITS_IN_COLOR_REG (MLX5_REG_BITS - MLX5_MTR_COLOR_BITS)
+
+#define UINT32_T(x) ((uint32_t)(x))
+
+#define LS32_MASK(bits) ((UINT32_T(1) << (bits)) - 1)
+
/* LRO configurations structure. */
struct mlx5_lro_config {
uint32_t supported:1; /* Whether LRO is supported. */
@@ -928,9 +938,9 @@ struct mlx5_priv {
unsigned int representor:1; /* Device is a port representor. */
unsigned int master:1; /* Device is a E-Switch master. */
unsigned int txpp_en:1; /* Tx packet pacing enabled. */
+ unsigned int sampler_en:1; /* Whether support sampler. */
unsigned int mtr_en:1; /* Whether support meter. */
unsigned int mtr_reg_share:1; /* Whether support meter REG_C share. */
- unsigned int sampler_en:1; /* Whether support sampler. */
uint16_t domain_id; /* Switch domain identifier. */
uint16_t vport_id; /* Associated VF vport index (if any). */
uint32_t vport_meta_tag; /* Used for vport index match ove VF LAG. */
@@ -989,6 +999,10 @@ struct mlx5_priv {
uint32_t rss_shared_actions; /* RSS shared actions. */
struct mlx5_devx_obj *q_counters; /* DevX queue counter object. */
uint32_t counter_set_id; /* Queue counter ID to set in DevX objects. */
+ uint8_t max_mtr_bits;
+ /* Indicate how many bits are used by meter id at the most. */
+ uint8_t max_mtr_flow_bits;
+ /* Indicate how many bits are used by meter flow id at the most. */
};
#define PORT_ID(priv) ((priv)->dev_data->port_id)
@@ -1246,11 +1260,10 @@ int mlx5_pmd_socket_init(void);
int mlx5_flow_meter_ops_get(struct rte_eth_dev *dev, void *arg);
struct mlx5_flow_meter *mlx5_flow_meter_find(struct mlx5_priv *priv,
uint32_t meter_id);
-struct mlx5_flow_meter *mlx5_flow_meter_attach
- (struct mlx5_priv *priv,
- uint32_t meter_id,
- const struct rte_flow_attr *attr,
- struct rte_flow_error *error);
+int mlx5_flow_meter_attach(struct mlx5_priv *priv,
+ struct mlx5_flow_meter *fm,
+ const struct rte_flow_attr *attr,
+ struct rte_flow_error *error);
void mlx5_flow_meter_detach(struct mlx5_flow_meter *fm);
/* mlx5_os.c */
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index c347f8130e..cffd6129e8 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -764,9 +764,9 @@ mlx5_flow_get_reg_id(struct rte_eth_dev *dev,
return REG_C_0;
}
break;
- case MLX5_MTR_SFX:
+ case MLX5_MTR_ID:
/*
- * If meter color and flow match share one register, flow match
+ * If meter color and meter id share one register, flow match
* should use the meter color register for match.
*/
if (priv->mtr_reg_share)
@@ -3051,7 +3051,8 @@ flow_mreg_split_qrss_release(struct rte_eth_dev *dev,
SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles,
handle_idx, dev_handle, next)
- if (dev_handle->split_flow_id)
+ if (dev_handle->split_flow_id &&
+ !dev_handle->is_meter_flow_id)
mlx5_ipool_free(priv->sh->ipool
[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
dev_handle->split_flow_id);
@@ -3690,23 +3691,30 @@ flow_parse_metadata_split_actions_info(const struct
rte_flow_action actions[],
*
* @param[in] actions
* Pointer to the list of actions.
- * @param[out] mtr
+ * @param[out] has_mtr
* Pointer to the meter exist flag.
+ * @param[out] meter_id
+ * Pointer to the meter id.
*
* @return
* Total number of actions.
*/
static int
-flow_check_meter_action(const struct rte_flow_action actions[], uint32_t *mtr)
+flow_check_meter_action(const struct rte_flow_action actions[],
+ bool *has_mtr,
+ uint32_t *meter_id)
{
+ const struct rte_flow_action_meter *mtr = NULL;
int actions_n = 0;
- MLX5_ASSERT(mtr);
- *mtr = 0;
+ MLX5_ASSERT(has_mtr);
+ *has_mtr = false;
for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
switch (actions->type) {
case RTE_FLOW_ACTION_TYPE_METER:
- *mtr = 1;
+ mtr = actions->conf;
+ *meter_id = mtr->mtr_id;
+ *has_mtr = true;
break;
default:
break;
@@ -4363,8 +4371,10 @@ flow_create_split_inner(struct rte_eth_dev *dev,
* header will be in the prefix sub flow, as not to take the
* L3 tunnel header into account.
*
- * @param dev
+ * @param[in] dev
* Pointer to Ethernet device.
+ * @param[in] fm
+ * Pointer to flow meter structure.
* @param[in] items
* Pattern specification (list terminated by the END pattern item).
* @param[out] sfx_items
@@ -4379,29 +4389,38 @@ flow_create_split_inner(struct rte_eth_dev *dev,
* The pattern items for the suffix flow.
* @param[out] tag_sfx
* Pointer to suffix flow tag.
+ * @param[out] error
+ * Perform verbose error reporting if not NULL.
*
* @return
- * 0 on success.
+ * The flow id, 0 otherwise and rte_errno is set.
*/
-static int
+static uint32_t
flow_meter_split_prep(struct rte_eth_dev *dev,
- const struct rte_flow_item items[],
- struct rte_flow_item sfx_items[],
- const struct rte_flow_action actions[],
- struct rte_flow_action actions_sfx[],
- struct rte_flow_action actions_pre[])
+ struct mlx5_flow_meter *fm,
+ const struct rte_flow_item items[],
+ struct rte_flow_item sfx_items[],
+ const struct rte_flow_action actions[],
+ struct rte_flow_action actions_sfx[],
+ struct rte_flow_action actions_pre[],
+ struct rte_flow_error *error)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct rte_flow_action *tag_action = NULL;
struct rte_flow_item *tag_item;
struct mlx5_rte_flow_action_set_tag *set_tag;
- struct rte_flow_error error;
const struct rte_flow_action_raw_encap *raw_encap;
const struct rte_flow_action_raw_decap *raw_decap;
- struct mlx5_rte_flow_item_tag *tag_spec;
- struct mlx5_rte_flow_item_tag *tag_mask;
+ struct mlx5_rte_flow_item_tag *tag_item_spec;
+ struct mlx5_rte_flow_item_tag *tag_item_mask;
uint32_t tag_id = 0;
bool copy_vlan = false;
+ uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0;
+ uint8_t mtr_reg_bits = priv->mtr_reg_share ?
+ MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS;
+ uint8_t flow_id_bits = 0;
+ int shift;
+ uint32_t flow_id_val = 0;
/* Prepare the actions for prefix and suffix flow. */
for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
@@ -4410,10 +4429,7 @@ flow_meter_split_prep(struct rte_eth_dev *dev,
switch (actions->type) {
case RTE_FLOW_ACTION_TYPE_METER:
/* Add the extra tag action first. */
- tag_action = actions_pre;
- tag_action->type = (enum rte_flow_action_type)
- MLX5_RTE_FLOW_ACTION_TYPE_TAG;
- actions_pre++;
+ tag_action = actions_pre++;
action_cur = &actions_pre;
break;
case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
@@ -4446,23 +4462,22 @@ flow_meter_split_prep(struct rte_eth_dev *dev,
actions_sfx->type = RTE_FLOW_ACTION_TYPE_END;
actions_pre->type = RTE_FLOW_ACTION_TYPE_END;
actions_pre++;
- /* Set the tag. */
- set_tag = (void *)actions_pre;
- set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error);
- mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
- &tag_id);
- if (tag_id >= (1 << (sizeof(tag_id) * 8 - MLX5_MTR_COLOR_BITS))) {
- DRV_LOG(ERR, "Port %u meter flow id exceed max limit.",
- dev->data->port_id);
- mlx5_ipool_free(priv->sh->ipool
- [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], tag_id);
- return 0;
- } else if (!tag_id) {
- return 0;
+ /* Generate meter flow_id only if support multiple flows per meter. */
+ mlx5_ipool_malloc(fm->flow_ipool, &tag_id);
+ if (!tag_id)
+ return rte_flow_error_set(error, ENOMEM,
+ RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+ "Failed to allocate meter flow id.");
+ flow_id_bits = MLX5_REG_BITS - __builtin_clz(tag_id - 1);
+ flow_id_bits = flow_id_bits ? flow_id_bits : 1;
+ if ((flow_id_bits + priv->max_mtr_bits) > mtr_reg_bits) {
+ mlx5_ipool_free(fm->flow_ipool, tag_id);
+ return rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+ "Meter flow id exceeds max limit.");
}
- set_tag->data = tag_id << MLX5_MTR_COLOR_BITS;
- assert(tag_action);
- tag_action->conf = set_tag;
+ if (flow_id_bits > priv->max_mtr_flow_bits)
+ priv->max_mtr_flow_bits = flow_id_bits;
/* Prepare the suffix subflow items. */
tag_item = sfx_items++;
for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
@@ -4491,16 +4506,34 @@ flow_meter_split_prep(struct rte_eth_dev *dev,
}
sfx_items->type = RTE_FLOW_ITEM_TYPE_END;
sfx_items++;
- tag_spec = (struct mlx5_rte_flow_item_tag *)sfx_items;
- tag_spec->data = tag_id << MLX5_MTR_COLOR_BITS;
- tag_spec->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error);
- tag_mask = tag_spec + 1;
- tag_mask->data = 0xffffff00;
+ /* Build tag actions and items for meter_id/meter flow_id. */
+ assert(tag_action);
+ set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre;
+ tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items;
+ tag_item_mask = tag_item_spec + 1;
+ /*
+ * The color Reg bits used by flow_id are growing from
+ * msb to lsb, so must do bit reverse for flow_id val in RegC.
+ */
+ for (shift = 0; shift < flow_id_bits; shift++)
+ flow_id_val = (flow_id_val << 1) |
+ (((tag_id - 1) >> shift) & 0x1);
+ /* Both flow_id and meter_id share the same register. */
+ set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, 0, error);
+ set_tag->data =
+ (fm->idx | (flow_id_val << (mtr_reg_bits - flow_id_bits)))
+ << mtr_id_offset;
+ tag_item_spec->id = set_tag->id;
+ tag_item_spec->data = set_tag->data;
+ tag_item_mask->data = UINT32_MAX << mtr_id_offset;
+ tag_action->type = (enum rte_flow_action_type)
+ MLX5_RTE_FLOW_ACTION_TYPE_TAG;
+ tag_action->conf = set_tag;
tag_item->type = (enum rte_flow_item_type)
- MLX5_RTE_FLOW_ITEM_TYPE_TAG;
- tag_item->spec = tag_spec;
+ MLX5_RTE_FLOW_ITEM_TYPE_TAG;
+ tag_item->spec = tag_item_spec;
tag_item->last = NULL;
- tag_item->mask = tag_mask;
+ tag_item->mask = tag_item_mask;
return tag_id;
}
@@ -5200,25 +5233,49 @@ flow_create_split_meter(struct rte_eth_dev *dev,
struct rte_flow_error *error)
{
struct mlx5_priv *priv = dev->data->dev_private;
+ struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
struct rte_flow_action *sfx_actions = NULL;
struct rte_flow_action *pre_actions = NULL;
struct rte_flow_item *sfx_items = NULL;
struct mlx5_flow *dev_flow = NULL;
struct rte_flow_attr sfx_attr = *attr;
- uint32_t mtr = 0;
+ struct mlx5_flow_meter *fm = NULL;
+ bool has_mtr = false;
+ uint32_t meter_id;
uint32_t mtr_tag_id = 0;
size_t act_size;
size_t item_size;
int actions_n = 0;
- int ret;
+ int ret = 0;
if (priv->mtr_en)
- actions_n = flow_check_meter_action(actions, &mtr);
- if (mtr) {
- /* The five prefix actions: meter, decap, encap, tag, end. */
+ actions_n = flow_check_meter_action(actions, &has_mtr,
+ &meter_id);
+ if (has_mtr) {
+ if (flow->meter) {
+ fm = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR],
+ flow->meter);
+ if (!fm)
+ return rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+ NULL, "Meter not found.");
+ } else {
+ fm = mlx5_flow_meter_find(priv, meter_id);
+ if (!fm)
+ return rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+ NULL, "Meter not found.");
+ ret = mlx5_flow_meter_attach(priv, fm,
+ &sfx_attr, error);
+ if (ret)
+ return -rte_errno;
+ flow->meter = fm->idx;
+ }
+ wks->fm = fm;
+ /* The prefix actions: meter, decap, encap, tag, end. */
act_size = sizeof(struct rte_flow_action) * (actions_n + 5) +
sizeof(struct mlx5_rte_flow_action_set_tag);
- /* tag, vlan, port id, end. */
+ /* The suffix items: tag, vlan, port id, end. */
#define METER_SUFFIX_ITEM 4
item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM +
sizeof(struct mlx5_rte_flow_item_tag) * 2;
@@ -5232,9 +5289,9 @@ flow_create_split_meter(struct rte_eth_dev *dev,
sfx_items = (struct rte_flow_item *)((char *)sfx_actions +
act_size);
pre_actions = sfx_actions + actions_n;
- mtr_tag_id = flow_meter_split_prep(dev, items, sfx_items,
+ mtr_tag_id = flow_meter_split_prep(dev, fm, items, sfx_items,
actions, sfx_actions,
- pre_actions);
+ pre_actions, error);
if (!mtr_tag_id) {
ret = -rte_errno;
goto exit;
@@ -5245,10 +5302,12 @@ flow_create_split_meter(struct rte_eth_dev *dev,
attr, items, pre_actions,
flow_split_info, error);
if (ret) {
+ mlx5_ipool_free(fm->flow_ipool, mtr_tag_id);
ret = -rte_errno;
goto exit;
}
dev_flow->handle->split_flow_id = mtr_tag_id;
+ dev_flow->handle->is_meter_flow_id = 1;
/* Setting the sfx group atrr. */
sfx_attr.group = sfx_attr.transfer ?
(MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) :
@@ -6520,20 +6579,17 @@ mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused,
*
* @param[in] dev
* Pointer to Ethernet device.
- * @param[in] fm
- * Pointer to the flow meter.
*
* @return
* Pointer to table set on success, NULL otherwise.
*/
struct mlx5_meter_domains_infos *
-mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev,
- const struct mlx5_flow_meter *fm)
+mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev)
{
const struct mlx5_flow_driver_ops *fops;
fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
- return fops->create_mtr_tbls(dev, fm);
+ return fops->create_mtr_tbls(dev);
}
/**
@@ -6558,7 +6614,7 @@ mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev,
}
/**
- * Create policer rules.
+ * Prepare policer rules.
*
* @param[in] dev
* Pointer to Ethernet device.
@@ -6571,14 +6627,14 @@ mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev,
* 0 on success, -1 otherwise.
*/
int
-mlx5_flow_create_policer_rules(struct rte_eth_dev *dev,
+mlx5_flow_prepare_policer_rules(struct rte_eth_dev *dev,
struct mlx5_flow_meter *fm,
const struct rte_flow_attr *attr)
{
const struct mlx5_flow_driver_ops *fops;
fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV);
- return fops->create_policer_rules(dev, fm, attr);
+ return fops->prepare_policer_rules(dev, fm, attr);
}
/**
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index 8324e188e1..c92e746a04 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -79,7 +79,7 @@ enum mlx5_feature_name {
MLX5_APP_TAG,
MLX5_COPY_MARK,
MLX5_MTR_COLOR,
- MLX5_MTR_SFX,
+ MLX5_MTR_ID,
MLX5_ASO_FLOW_HIT,
};
@@ -655,7 +655,8 @@ struct mlx5_flow_handle {
uint64_t layers;
/**< Bit-fields of present layers, see MLX5_FLOW_LAYER_*. */
void *drv_flow; /**< pointer to driver flow object. */
- uint32_t split_flow_id:28; /**< Sub flow unique match flow id. */
+ uint32_t split_flow_id:27; /**< Sub flow unique match flow id. */
+ uint32_t is_meter_flow_id:1; /**< Indate if flow_id is for meter. */
uint32_t mark:1; /**< Metadate rxq mark flag. */
uint32_t fate_action:3; /**< Fate action type. */
union {
@@ -842,14 +843,16 @@ struct mlx5_meter_domain_info {
/**< Meter table. */
struct mlx5_flow_tbl_resource *sfx_tbl;
/**< Meter suffix table. */
- void *any_matcher;
- /**< Meter color not match default criteria. */
- void *color_matcher;
- /**< Meter color match criteria. */
+ struct mlx5_flow_dv_matcher *drop_matcher;
+ /**< Matcher for Drop. */
+ struct mlx5_flow_dv_matcher *color_matcher;
+ /**< Matcher for Color. */
void *jump_actn;
/**< Meter match action. */
- void *policer_rules[RTE_MTR_DROPPED + 1];
- /**< Meter policer for the match. */
+ void *green_rule;
+ /**< Meter green rule. */
+ void *drop_rule;
+ /**< Meter drop rule. */
};
/* Meter table set for TX RX FDB. */
@@ -862,10 +865,10 @@ struct mlx5_meter_domains_infos {
/**< RX meter table. */
struct mlx5_meter_domain_info transfer;
/**< FDB meter table. */
- void *drop_actn;
- /**< Drop action as not matched. */
- void *count_actns[RTE_MTR_DROPPED + 1];
- /**< Counters for match and unmatched statistics. */
+ void *green_count;
+ /**< Counters for green rule. */
+ void *drop_count;
+ /**< Counters for green rule. */
uint32_t fmp[MLX5_ST_SZ_DW(flow_meter_parameters)];
/**< Flow meter parameter. */
size_t fmp_size;
@@ -928,6 +931,8 @@ struct mlx5_flow_meter {
/**< Meter state. */
uint32_t shared:1;
/**< Meter shared or not. */
+ struct mlx5_indexed_pool *flow_ipool;
+ /**< Index pool for flow id. */
};
/* RFC2697 parameter structure. */
@@ -1148,6 +1153,7 @@ struct mlx5_flow_workspace {
struct mlx5_flow_rss_desc rss_desc;
uint32_t rssq_num; /* Allocated queue num in rss_desc. */
uint32_t flow_idx; /* Intermediate device flow index. */
+ struct mlx5_flow_meter *fm; /* Pointer to the meter in flow. */
};
struct mlx5_flow_split_info {
@@ -1188,8 +1194,7 @@ typedef int (*mlx5_flow_query_t)(struct rte_eth_dev *dev,
void *data,
struct rte_flow_error *error);
typedef struct mlx5_meter_domains_infos *(*mlx5_flow_create_mtr_tbls_t)
- (struct rte_eth_dev *dev,
- const struct mlx5_flow_meter *fm);
+ (struct rte_eth_dev *dev);
typedef int (*mlx5_flow_destroy_mtr_tbls_t)(struct rte_eth_dev *dev,
struct mlx5_meter_domains_infos *tbls);
typedef int (*mlx5_flow_create_policer_rules_t)
@@ -1252,7 +1257,7 @@ struct mlx5_flow_driver_ops {
mlx5_flow_query_t query;
mlx5_flow_create_mtr_tbls_t create_mtr_tbls;
mlx5_flow_destroy_mtr_tbls_t destroy_mtr_tbls;
- mlx5_flow_create_policer_rules_t create_policer_rules;
+ mlx5_flow_create_policer_rules_t prepare_policer_rules;
mlx5_flow_destroy_policer_rules_t destroy_policer_rules;
mlx5_flow_counter_alloc_t counter_alloc;
mlx5_flow_counter_free_t counter_free;
@@ -1452,11 +1457,10 @@ int mlx5_flow_validate_item_ecpri(const struct
rte_flow_item *item,
const struct rte_flow_item_ecpri *acc_mask,
struct rte_flow_error *error);
struct mlx5_meter_domains_infos *mlx5_flow_create_mtr_tbls
- (struct rte_eth_dev *dev,
- const struct mlx5_flow_meter *fm);
+ (struct rte_eth_dev *dev);
int mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev,
struct mlx5_meter_domains_infos *tbl);
-int mlx5_flow_create_policer_rules(struct rte_eth_dev *dev,
+int mlx5_flow_prepare_policer_rules(struct rte_eth_dev *dev,
struct mlx5_flow_meter *fm,
const struct rte_flow_attr *attr);
int mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev,
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 23e5849783..ce9857d3f7 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -10982,14 +10982,12 @@ flow_dv_translate(struct rte_eth_dev *dev,
const struct rte_flow_action_rss *rss;
const struct rte_flow_action *action = actions;
const uint8_t *rss_key;
- const struct rte_flow_action_meter *mtr;
struct mlx5_flow_tbl_resource *tbl;
struct mlx5_aso_age_action *age_act;
uint32_t port_id = 0;
struct mlx5_flow_dv_port_id_action_resource port_id_resource;
int action_type = actions->type;
const struct rte_flow_action *found_action = NULL;
- struct mlx5_flow_meter *fm = NULL;
uint32_t jump_group = 0;
if (!mlx5_flow_os_action_supported(action_type))
@@ -11414,33 +11412,13 @@ flow_dv_translate(struct rte_eth_dev *dev,
MLX5_FLOW_FATE_DEFAULT_MISS;
break;
case RTE_FLOW_ACTION_TYPE_METER:
- mtr = actions->conf;
- if (!flow->meter) {
- fm = mlx5_flow_meter_attach(priv, mtr->mtr_id,
- attr, error);
- if (!fm)
- return rte_flow_error_set(error,
- rte_errno,
- RTE_FLOW_ERROR_TYPE_ACTION,
- NULL,
- "meter not found "
- "or invalid parameters");
- flow->meter = fm->idx;
- }
+ if (!wks->fm)
+ return rte_flow_error_set(error, rte_errno,
+ RTE_FLOW_ERROR_TYPE_ACTION,
+ NULL, "Failed to get meter in flow.");
/* Set the meter action. */
- if (!fm) {
- fm = mlx5_ipool_get(priv->sh->ipool
- [MLX5_IPOOL_MTR], flow->meter);
- if (!fm)
- return rte_flow_error_set(error,
- rte_errno,
- RTE_FLOW_ERROR_TYPE_ACTION,
- NULL,
- "meter not found "
- "or invalid parameters");
- }
dev_flow->dv.actions[actions_n++] =
- fm->mfts->meter_action;
+ wks->fm->mfts->meter_action;
action_flags |= MLX5_FLOW_ACTION_METER;
break;
case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
@@ -12536,6 +12514,7 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct
rte_flow *flow)
{
struct mlx5_flow_handle *dev_handle;
struct mlx5_priv *priv = dev->data->dev_private;
+ struct mlx5_flow_meter *fm = NULL;
uint32_t srss = 0;
if (!flow)
@@ -12546,8 +12525,6 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct
rte_flow *flow)
flow->counter = 0;
}
if (flow->meter) {
- struct mlx5_flow_meter *fm;
-
fm = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR],
flow->meter);
if (fm)
@@ -12589,6 +12566,10 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct
rte_flow *flow)
flow_dv_fate_resource_release(dev, dev_handle);
else if (!srss)
srss = dev_handle->rix_srss;
+ if (fm && dev_handle->is_meter_flow_id &&
+ dev_handle->split_flow_id)
+ mlx5_ipool_free(fm->flow_ipool,
+ dev_handle->split_flow_id);
mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
tmp_idx);
}
@@ -13274,49 +13255,20 @@ flow_dv_destroy_mtr_tbl(struct rte_eth_dev *dev,
if (!mtd || !priv->config.dv_flow_en)
return 0;
- if (mtd->ingress.policer_rules[RTE_MTR_DROPPED])
- claim_zero(mlx5_flow_os_destroy_flow
- (mtd->ingress.policer_rules[RTE_MTR_DROPPED]));
- if (mtd->egress.policer_rules[RTE_MTR_DROPPED])
- claim_zero(mlx5_flow_os_destroy_flow
- (mtd->egress.policer_rules[RTE_MTR_DROPPED]));
- if (mtd->transfer.policer_rules[RTE_MTR_DROPPED])
- claim_zero(mlx5_flow_os_destroy_flow
- (mtd->transfer.policer_rules[RTE_MTR_DROPPED]));
- if (mtd->egress.color_matcher)
- claim_zero(mlx5_flow_os_destroy_flow_matcher
- (mtd->egress.color_matcher));
- if (mtd->egress.any_matcher)
- claim_zero(mlx5_flow_os_destroy_flow_matcher
- (mtd->egress.any_matcher));
if (mtd->egress.tbl)
flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->egress.tbl);
if (mtd->egress.sfx_tbl)
flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->egress.sfx_tbl);
- if (mtd->ingress.color_matcher)
- claim_zero(mlx5_flow_os_destroy_flow_matcher
- (mtd->ingress.color_matcher));
- if (mtd->ingress.any_matcher)
- claim_zero(mlx5_flow_os_destroy_flow_matcher
- (mtd->ingress.any_matcher));
if (mtd->ingress.tbl)
flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->ingress.tbl);
if (mtd->ingress.sfx_tbl)
flow_dv_tbl_resource_release(MLX5_SH(dev),
mtd->ingress.sfx_tbl);
- if (mtd->transfer.color_matcher)
- claim_zero(mlx5_flow_os_destroy_flow_matcher
- (mtd->transfer.color_matcher));
- if (mtd->transfer.any_matcher)
- claim_zero(mlx5_flow_os_destroy_flow_matcher
- (mtd->transfer.any_matcher));
if (mtd->transfer.tbl)
flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->transfer.tbl);
if (mtd->transfer.sfx_tbl)
flow_dv_tbl_resource_release(MLX5_SH(dev),
mtd->transfer.sfx_tbl);
- if (mtd->drop_actn)
- claim_zero(mlx5_flow_os_destroy_flow_action(mtd->drop_actn));
mlx5_free(mtd);
return 0;
}
@@ -13335,37 +13287,17 @@ flow_dv_destroy_mtr_tbl(struct rte_eth_dev *dev,
* Table attribute.
* @param[in] transfer
* Table attribute.
- * @param[in] color_reg_c_idx
- * Reg C index for color match.
*
* @return
- * 0 on success, -1 otherwise and rte_errno is set.
+ * 0 on success, -1 otherwise.
*/
static int
flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev,
struct mlx5_meter_domains_infos *mtb,
- uint8_t egress, uint8_t transfer,
- uint32_t color_reg_c_idx)
+ uint8_t egress, uint8_t transfer)
{
- struct mlx5_priv *priv = dev->data->dev_private;
- struct mlx5_dev_ctx_shared *sh = priv->sh;
- struct mlx5_flow_dv_match_params mask = {
- .size = sizeof(mask.buf),
- };
- struct mlx5_flow_dv_match_params value = {
- .size = sizeof(value.buf),
- };
- struct mlx5dv_flow_matcher_attr dv_attr = {
- .type = IBV_FLOW_ATTR_NORMAL,
- .priority = 0,
- .match_criteria_enable = 0,
- .match_mask = (void *)&mask,
- };
- void *actions[METER_ACTIONS];
- struct mlx5_meter_domain_info *dtb;
struct rte_flow_error error;
- int i = 0;
- int ret;
+ struct mlx5_meter_domain_info *dtb;
if (transfer)
dtb = &mtb->transfer;
@@ -13390,41 +13322,7 @@ flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev,
DRV_LOG(ERR, "Failed to create meter suffix table.");
return -1;
}
- /* Create matchers, Any and Color. */
- dv_attr.priority = 3;
- dv_attr.match_criteria_enable = 0;
- ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj,
- &dtb->any_matcher);
- if (ret) {
- DRV_LOG(ERR, "Failed to create meter"
- " policer default matcher.");
- goto error_exit;
- }
- dv_attr.priority = 0;
- dv_attr.match_criteria_enable =
- 1 << MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT;
- flow_dv_match_meta_reg(mask.buf, value.buf, color_reg_c_idx,
- rte_col_2_mlx5_col(RTE_COLORS), UINT8_MAX);
- ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj,
- &dtb->color_matcher);
- if (ret) {
- DRV_LOG(ERR, "Failed to create meter policer color matcher.");
- goto error_exit;
- }
- if (mtb->count_actns[RTE_MTR_DROPPED])
- actions[i++] = mtb->count_actns[RTE_MTR_DROPPED];
- actions[i++] = mtb->drop_actn;
- /* Default rule: lowest priority, match any, actions: drop. */
- ret = mlx5_flow_os_create_flow(dtb->any_matcher, (void *)&value, i,
- actions,
- &dtb->policer_rules[RTE_MTR_DROPPED]);
- if (ret) {
- DRV_LOG(ERR, "Failed to create meter policer drop rule.");
- goto error_exit;
- }
return 0;
-error_exit:
- return -1;
}
/**
@@ -13433,20 +13331,16 @@ flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev,
*
* @param[in] dev
* Pointer to Ethernet device.
- * @param[in] fm
- * Pointer to the flow meter.
*
* @return
* Pointer to table set on success, NULL otherwise and rte_errno is set.
*/
static struct mlx5_meter_domains_infos *
-flow_dv_create_mtr_tbl(struct rte_eth_dev *dev,
- const struct mlx5_flow_meter *fm)
+flow_dv_create_mtr_tbl(struct rte_eth_dev *dev)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_meter_domains_infos *mtb;
int ret;
- int i;
if (!priv->mtr_en) {
rte_errno = ENOTSUP;
@@ -13457,37 +13351,21 @@ flow_dv_create_mtr_tbl(struct rte_eth_dev *dev,
DRV_LOG(ERR, "Failed to allocate memory for meter.");
return NULL;
}
- /* Create meter count actions */
- for (i = 0; i <= RTE_MTR_DROPPED; i++) {
- struct mlx5_flow_counter *cnt;
- if (!fm->policer_stats.cnt[i])
- continue;
- cnt = flow_dv_counter_get_by_idx(dev,
- fm->policer_stats.cnt[i], NULL);
- mtb->count_actns[i] = cnt->action;
- }
- /* Create drop action. */
- ret = mlx5_flow_os_create_flow_action_drop(&mtb->drop_actn);
- if (ret) {
- DRV_LOG(ERR, "Failed to create drop action.");
- goto error_exit;
- }
/* Egress meter table. */
- ret = flow_dv_prepare_mtr_tables(dev, mtb, 1, 0, priv->mtr_color_reg);
+ ret = flow_dv_prepare_mtr_tables(dev, mtb, 1, 0);
if (ret) {
DRV_LOG(ERR, "Failed to prepare egress meter table.");
goto error_exit;
}
/* Ingress meter table. */
- ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 0, priv->mtr_color_reg);
+ ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 0);
if (ret) {
DRV_LOG(ERR, "Failed to prepare ingress meter table.");
goto error_exit;
}
/* FDB meter table. */
if (priv->config.dv_esw_en) {
- ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 1,
- priv->mtr_color_reg);
+ ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 1);
if (ret) {
DRV_LOG(ERR, "Failed to prepare fdb meter table.");
goto error_exit;
@@ -13499,24 +13377,153 @@ flow_dv_create_mtr_tbl(struct rte_eth_dev *dev,
return NULL;
}
+/**
+ * Destroy the meter table matchers.
+ * Lock free, (mutex should be acquired by caller).
+ *
+ * @param[in] dev
+ * Pointer to Ethernet device.
+ * @param[in,out] dtb
+ * Pointer to DV meter table.
+ *
+ * @return
+ * Always 0.
+ */
+static int
+flow_dv_destroy_mtr_matchers(struct rte_eth_dev *dev,
+ struct mlx5_meter_domain_info *dtb)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+ struct mlx5_flow_tbl_data_entry *tbl;
+
+ if (!priv->config.dv_flow_en)
+ return 0;
+ if (dtb->drop_matcher) {
+ tbl = container_of(dtb->drop_matcher->tbl, typeof(*tbl), tbl);
+ mlx5_cache_unregister(&tbl->matchers,
+ &dtb->drop_matcher->entry);
+ dtb->drop_matcher = NULL;
+ }
+ if (dtb->color_matcher) {
+ tbl = container_of(dtb->color_matcher->tbl, typeof(*tbl), tbl);
+ mlx5_cache_unregister(&tbl->matchers,
+ &dtb->color_matcher->entry);
+ dtb->color_matcher = NULL;
+ }
+ return 0;
+}
+
+/**
+ * Create the matchers for meter table.
+ *
+ * @param[in] dev
+ * Pointer to Ethernet device.
+ * @param[in] color_reg_c_idx
+ * Reg C index for color match.
+ * @param[in] mtr_id_reg_c_idx
+ * Reg C index for meter_id match.
+ * @param[in] mtr_id_mask
+ * Mask for meter_id match criteria.
+ * @param[in,out] dtb
+ * Pointer to DV meter table.
+ * @param[out] error
+ * Perform verbose error reporting if not NULL.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+flow_dv_prepare_mtr_matchers(struct rte_eth_dev *dev,
+ uint32_t color_reg_c_idx,
+ uint32_t mtr_id_reg_c_idx,
+ uint32_t mtr_id_mask,
+ struct mlx5_meter_domain_info *dtb,
+ struct rte_flow_error *error)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+ struct mlx5_flow_tbl_data_entry *tbl_data;
+ struct mlx5_cache_entry *entry;
+ struct mlx5_flow_dv_matcher matcher = {
+ .mask = {
+ .size = sizeof(matcher.mask.buf) -
+ MLX5_ST_SZ_BYTES(fte_match_set_misc4),
+ },
+ .tbl = dtb->tbl,
+ };
+ struct mlx5_flow_dv_match_params value = {
+ .size = sizeof(value.buf) -
+ MLX5_ST_SZ_BYTES(fte_match_set_misc4),
+ };
+ struct mlx5_flow_cb_ctx ctx = {
+ .error = error,
+ .data = &matcher,
+ };
+
+ tbl_data = container_of(dtb->tbl, struct mlx5_flow_tbl_data_entry, tbl);
+ if (!dtb->drop_matcher) {
+ /* Create matchers for Drop. */
+ flow_dv_match_meta_reg(matcher.mask.buf, value.buf,
+ mtr_id_reg_c_idx, 0, mtr_id_mask);
+ matcher.priority = MLX5_REG_BITS * 2 - priv->max_mtr_bits;
+ matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
+ matcher.mask.size);
+ entry = mlx5_cache_register(&tbl_data->matchers, &ctx);
+ if (!entry) {
+ DRV_LOG(ERR, "Failed to register meter drop matcher.");
+ return -1;
+ }
+ dtb->drop_matcher =
+ container_of(entry, struct mlx5_flow_dv_matcher, entry);
+ }
+ if (!dtb->color_matcher) {
+ /* Create matchers for Color + meter_id. */
+ if (priv->mtr_reg_share) {
+ flow_dv_match_meta_reg(matcher.mask.buf, value.buf,
+ color_reg_c_idx, 0,
+ (mtr_id_mask |
+ LS32_MASK(MLX5_MTR_COLOR_BITS)));
+ } else {
+ flow_dv_match_meta_reg(matcher.mask.buf, value.buf,
+ color_reg_c_idx, 0,
+ LS32_MASK(MLX5_MTR_COLOR_BITS));
+ flow_dv_match_meta_reg(matcher.mask.buf, value.buf,
+ mtr_id_reg_c_idx, 0, mtr_id_mask);
+ }
+ matcher.priority = MLX5_REG_BITS - priv->max_mtr_bits;
+ matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
+ matcher.mask.size);
+ entry = mlx5_cache_register(&tbl_data->matchers, &ctx);
+ if (!entry) {
+ DRV_LOG(ERR, "Failed to register meter color matcher.");
+ return -1;
+ }
+ dtb->color_matcher =
+ container_of(entry, struct mlx5_flow_dv_matcher, entry);
+ }
+ return 0;
+}
+
/**
* Destroy domain policer rule.
*
+ * @param[in] dev
+ * Pointer to Ethernet device.
* @param[in] dt
* Pointer to domain table.
*/
static void
-flow_dv_destroy_domain_policer_rule(struct mlx5_meter_domain_info *dt)
+flow_dv_destroy_domain_policer_rule(struct rte_eth_dev *dev,
+ struct mlx5_meter_domain_info *dt)
{
- int i;
-
- for (i = 0; i < RTE_MTR_DROPPED; i++) {
- if (dt->policer_rules[i]) {
- claim_zero(mlx5_flow_os_destroy_flow
- (dt->policer_rules[i]));
- dt->policer_rules[i] = NULL;
- }
+ if (dt->drop_rule) {
+ claim_zero(mlx5_flow_os_destroy_flow(dt->drop_rule));
+ dt->drop_rule = NULL;
+ }
+ if (dt->green_rule) {
+ claim_zero(mlx5_flow_os_destroy_flow(dt->green_rule));
+ dt->green_rule = NULL;
}
+ flow_dv_destroy_mtr_matchers(dev, dt);
if (dt->jump_actn) {
claim_zero(mlx5_flow_os_destroy_flow_action(dt->jump_actn));
dt->jump_actn = NULL;
@@ -13537,7 +13544,7 @@ flow_dv_destroy_domain_policer_rule(struct
mlx5_meter_domain_info *dt)
* Always 0.
*/
static int
-flow_dv_destroy_policer_rules(struct rte_eth_dev *dev __rte_unused,
+flow_dv_destroy_policer_rules(struct rte_eth_dev *dev,
const struct mlx5_flow_meter *fm,
const struct rte_flow_attr *attr)
{
@@ -13546,39 +13553,55 @@ flow_dv_destroy_policer_rules(struct rte_eth_dev *dev
__rte_unused,
if (!mtb)
return 0;
if (attr->egress)
- flow_dv_destroy_domain_policer_rule(&mtb->egress);
+ flow_dv_destroy_domain_policer_rule(dev, &mtb->egress);
if (attr->ingress)
- flow_dv_destroy_domain_policer_rule(&mtb->ingress);
+ flow_dv_destroy_domain_policer_rule(dev, &mtb->ingress);
if (attr->transfer)
- flow_dv_destroy_domain_policer_rule(&mtb->transfer);
+ flow_dv_destroy_domain_policer_rule(dev, &mtb->transfer);
return 0;
}
/**
* Create specify domain meter policer rule.
*
+ * @param[in] dev
+ * Pointer to Ethernet device.
* @param[in] fm
* Pointer to flow meter structure.
* @param[in] mtb
* Pointer to DV meter table set.
- * @param[in] mtr_reg_c
- * Color match REG_C.
+ * @param[out] drop_rule
+ * The address of pointer saving drop rule.
+ * @param[out] color_rule
+ * The address of pointer saving green rule.
*
* @return
* 0 on success, -1 otherwise.
*/
static int
-flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm,
+flow_dv_create_policer_forward_rule(struct rte_eth_dev *dev,
+ struct mlx5_flow_meter *fm,
struct mlx5_meter_domain_info *dtb,
- uint8_t mtr_reg_c)
+ void **drop_rule,
+ void **green_rule)
{
+ struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_flow_dv_match_params matcher = {
- .size = sizeof(matcher.buf),
+ .size = sizeof(matcher.buf) -
+ MLX5_ST_SZ_BYTES(fte_match_set_misc4),
};
struct mlx5_flow_dv_match_params value = {
- .size = sizeof(value.buf),
+ .size = sizeof(value.buf) -
+ MLX5_ST_SZ_BYTES(fte_match_set_misc4),
};
struct mlx5_meter_domains_infos *mtb = fm->mfts;
+ struct rte_flow_error error;
+ uint32_t color_reg_c = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR,
+ 0, &error);
+ uint32_t mtr_id_reg_c = mlx5_flow_get_reg_id(dev, MLX5_MTR_ID,
+ 0, &error);
+ uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0;
+ uint32_t mtr_id_mask = LS32_MASK(priv->max_mtr_bits) << mtr_id_offset;
void *actions[METER_ACTIONS];
int i;
int ret = 0;
@@ -13591,25 +13614,52 @@ flow_dv_create_policer_forward_rule(struct
mlx5_flow_meter *fm,
DRV_LOG(ERR, "Failed to create policer jump action.");
goto error;
}
- for (i = 0; i < RTE_MTR_DROPPED; i++) {
- int j = 0;
-
- flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_reg_c,
- rte_col_2_mlx5_col(i), UINT8_MAX);
- if (mtb->count_actns[i])
- actions[j++] = mtb->count_actns[i];
- if (fm->action[i] == MTR_POLICER_ACTION_DROP)
- actions[j++] = mtb->drop_actn;
- else
- actions[j++] = dtb->jump_actn;
- ret = mlx5_flow_os_create_flow(dtb->color_matcher,
- (void *)&value, j, actions,
- &dtb->policer_rules[i]);
+ /* Prepare matchers. */
+ if (!dtb->drop_matcher || !dtb->color_matcher) {
+ ret = flow_dv_prepare_mtr_matchers(dev, color_reg_c,
+ mtr_id_reg_c, mtr_id_mask,
+ dtb, &error);
if (ret) {
- DRV_LOG(ERR, "Failed to create policer rule.");
+ DRV_LOG(ERR, "Failed to setup matchers for mtr table.");
goto error;
}
}
+ /* Create Drop flow, matching meter_id only. */
+ i = 0;
+ flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_id_reg_c,
+ (fm->idx << mtr_id_offset), UINT32_MAX);
+ if (mtb->drop_count)
+ actions[i++] = mtb->drop_count;
+ actions[i++] = priv->sh->esw_drop_action;
+ ret = mlx5_flow_os_create_flow(dtb->drop_matcher->matcher_object,
+ (void *)&value, i, actions, drop_rule);
+ if (ret) {
+ DRV_LOG(ERR, "Failed to create meter policer drop rule.");
+ goto error;
+ }
+ /* Create flow matching Green color + meter_id. */
+ i = 0;
+ if (priv->mtr_reg_share) {
+ flow_dv_match_meta_reg(matcher.buf, value.buf, color_reg_c,
+ ((fm->idx << mtr_id_offset) |
+ rte_col_2_mlx5_col(RTE_COLOR_GREEN)),
+ UINT32_MAX);
+ } else {
+ flow_dv_match_meta_reg(matcher.buf, value.buf, color_reg_c,
+ rte_col_2_mlx5_col(RTE_COLOR_GREEN),
+ UINT32_MAX);
+ flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_id_reg_c,
+ fm->idx, UINT32_MAX);
+ }
+ if (mtb->green_count)
+ actions[i++] = mtb->green_count;
+ actions[i++] = dtb->jump_actn;
+ ret = mlx5_flow_os_create_flow(dtb->color_matcher->matcher_object,
+ (void *)&value, i, actions, green_rule);
+ if (ret) {
+ DRV_LOG(ERR, "Failed to create meter policer color rule.");
+ goto error;
+ }
return 0;
error:
rte_errno = errno;
@@ -13617,7 +13667,8 @@ flow_dv_create_policer_forward_rule(struct
mlx5_flow_meter *fm,
}
/**
- * Create policer rules.
+ * Prepare policer rules for all domains.
+ * If meter already initialized, this will replace all old rules with new ones.
*
* @param[in] dev
* Pointer to Ethernet device.
@@ -13630,41 +13681,107 @@ flow_dv_create_policer_forward_rule(struct
mlx5_flow_meter *fm,
* 0 on success, -1 otherwise.
*/
static int
-flow_dv_create_policer_rules(struct rte_eth_dev *dev,
- struct mlx5_flow_meter *fm,
- const struct rte_flow_attr *attr)
+flow_dv_prepare_policer_rules(struct rte_eth_dev *dev,
+ struct mlx5_flow_meter *fm,
+ const struct rte_flow_attr *attr)
{
- struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_meter_domains_infos *mtb = fm->mfts;
+ bool initialized = false;
+ struct mlx5_flow_counter *cnt;
+ void *egress_drop_rule = NULL;
+ void *egress_green_rule = NULL;
+ void *ingress_drop_rule = NULL;
+ void *ingress_green_rule = NULL;
+ void *transfer_drop_rule = NULL;
+ void *transfer_green_rule = NULL;
int ret;
+ /* Get the statistics counters for green/drop. */
+ if (fm->policer_stats.cnt[RTE_COLOR_GREEN]) {
+ cnt = flow_dv_counter_get_by_idx(dev,
+ fm->policer_stats.cnt[RTE_COLOR_GREEN],
+ NULL);
+ mtb->green_count = cnt->action;
+ } else {
+ mtb->green_count = NULL;
+ }
+ if (fm->policer_stats.cnt[RTE_MTR_DROPPED]) {
+ cnt = flow_dv_counter_get_by_idx(dev,
+ fm->policer_stats.cnt[RTE_MTR_DROPPED],
+ NULL);
+ mtb->drop_count = cnt->action;
+ } else {
+ mtb->drop_count = NULL;
+ }
+ /**
+ * If flow meter has been initilized, all policer rules
+ * are created. So can get if meter initialized by checking
+ * any policer rule.
+ */
+ if (mtb->egress.drop_rule)
+ initialized = true;
if (attr->egress) {
- ret = flow_dv_create_policer_forward_rule(fm, &mtb->egress,
- priv->mtr_color_reg);
+ ret = flow_dv_create_policer_forward_rule(dev,
+ fm, &mtb->egress,
+ &egress_drop_rule, &egress_green_rule);
if (ret) {
DRV_LOG(ERR, "Failed to create egress policer.");
goto error;
}
}
if (attr->ingress) {
- ret = flow_dv_create_policer_forward_rule(fm, &mtb->ingress,
- priv->mtr_color_reg);
+ ret = flow_dv_create_policer_forward_rule(dev,
+ fm, &mtb->ingress,
+ &ingress_drop_rule, &ingress_green_rule);
if (ret) {
DRV_LOG(ERR, "Failed to create ingress policer.");
goto error;
}
}
if (attr->transfer) {
- ret = flow_dv_create_policer_forward_rule(fm, &mtb->transfer,
- priv->mtr_color_reg);
+ ret = flow_dv_create_policer_forward_rule(dev,
+ fm, &mtb->transfer,
+ &transfer_drop_rule, &transfer_green_rule);
if (ret) {
DRV_LOG(ERR, "Failed to create transfer policer.");
goto error;
}
}
+ /* Replace old flows if existing. */
+ if (mtb->egress.drop_rule)
+ claim_zero(mlx5_flow_os_destroy_flow(mtb->egress.drop_rule));
+ if (mtb->egress.green_rule)
+ claim_zero(mlx5_flow_os_destroy_flow(mtb->egress.green_rule));
+ if (mtb->ingress.drop_rule)
+ claim_zero(mlx5_flow_os_destroy_flow(mtb->ingress.drop_rule));
+ if (mtb->ingress.green_rule)
+ claim_zero(mlx5_flow_os_destroy_flow(mtb->ingress.green_rule));
+ if (mtb->transfer.drop_rule)
+ claim_zero(mlx5_flow_os_destroy_flow(mtb->transfer.drop_rule));
+ if (mtb->transfer.green_rule)
+ claim_zero(mlx5_flow_os_destroy_flow(mtb->transfer.green_rule));
+ mtb->egress.drop_rule = egress_drop_rule;
+ mtb->egress.green_rule = egress_green_rule;
+ mtb->ingress.drop_rule = ingress_drop_rule;
+ mtb->ingress.green_rule = ingress_green_rule;
+ mtb->transfer.drop_rule = transfer_drop_rule;
+ mtb->transfer.green_rule = transfer_green_rule;
return 0;
error:
- flow_dv_destroy_policer_rules(dev, fm, attr);
+ if (egress_drop_rule)
+ claim_zero(mlx5_flow_os_destroy_flow(egress_drop_rule));
+ if (egress_green_rule)
+ claim_zero(mlx5_flow_os_destroy_flow(egress_green_rule));
+ if (ingress_drop_rule)
+ claim_zero(mlx5_flow_os_destroy_flow(ingress_drop_rule));
+ if (ingress_green_rule)
+ claim_zero(mlx5_flow_os_destroy_flow(ingress_green_rule));
+ if (transfer_drop_rule)
+ claim_zero(mlx5_flow_os_destroy_flow(transfer_drop_rule));
+ if (transfer_green_rule)
+ claim_zero(mlx5_flow_os_destroy_flow(transfer_green_rule));
+ if (!initialized)
+ flow_dv_destroy_policer_rules(dev, fm, attr);
return -1;
}
@@ -13960,7 +14077,7 @@ const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops
= {
.query = flow_dv_query,
.create_mtr_tbls = flow_dv_create_mtr_tbl,
.destroy_mtr_tbls = flow_dv_destroy_mtr_tbl,
- .create_policer_rules = flow_dv_create_policer_rules,
+ .prepare_policer_rules = flow_dv_prepare_policer_rules,
.destroy_policer_rules = flow_dv_destroy_policer_rules,
.counter_alloc = flow_dv_counter_allocate,
.counter_free = flow_dv_counter_free,
diff --git a/drivers/net/mlx5/mlx5_flow_meter.c
b/drivers/net/mlx5/mlx5_flow_meter.c
index dbc574b508..68f5c344cf 100644
--- a/drivers/net/mlx5/mlx5_flow_meter.c
+++ b/drivers/net/mlx5/mlx5_flow_meter.c
@@ -474,6 +474,12 @@ mlx5_flow_meter_validate(struct mlx5_priv *priv, uint32_t
meter_id,
MTR_POLICER_ACTION_COLOR_RED };
int i;
+ /* Meter must use global drop action. */
+ if (!priv->sh->esw_drop_action)
+ return -rte_mtr_error_set(error, ENOTSUP,
+ RTE_MTR_ERROR_TYPE_MTR_PARAMS,
+ NULL,
+ "No drop action ready for meter.");
/* Meter params must not be NULL. */
if (params == NULL)
return -rte_mtr_error_set(error, EINVAL,
@@ -630,9 +636,18 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t
meter_id,
.egress = 1,
.transfer = priv->config.dv_esw_en ? 1 : 0,
};
+ struct mlx5_indexed_pool_config flow_ipool_cfg = {
+ .size = 0,
+ .trunk_size = 64,
+ .need_lock = 1,
+ .type = "mlx5_flow_mtr_flow_id_pool",
+ };
int ret;
unsigned int i;
uint32_t idx = 0;
+ uint8_t mtr_id_bits;
+ uint8_t mtr_reg_bits = priv->mtr_reg_share ?
+ MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS;
if (!priv->mtr_en)
return -rte_mtr_error_set(error, ENOTSUP,
@@ -654,6 +669,13 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t
meter_id,
return -rte_mtr_error_set(error, ENOMEM,
RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
"Memory alloc failed for meter.");
+ mtr_id_bits = MLX5_REG_BITS - __builtin_clz(idx);
+ if ((mtr_id_bits + priv->max_mtr_flow_bits) > mtr_reg_bits) {
+ DRV_LOG(ERR, "Meter number exceeds max limit.");
+ goto error;
+ }
+ if (mtr_id_bits > priv->max_mtr_bits)
+ priv->max_mtr_bits = mtr_id_bits;
fm->idx = idx;
/* Fill the flow meter parameters. */
fm->meter_id = meter_id;
@@ -667,10 +689,10 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t
meter_id,
if (!fm->policer_stats.cnt[i])
goto error;
}
- fm->mfts = mlx5_flow_create_mtr_tbls(dev, fm);
+ fm->mfts = mlx5_flow_create_mtr_tbls(dev);
if (!fm->mfts)
goto error;
- ret = mlx5_flow_create_policer_rules(dev, fm, &attr);
+ ret = mlx5_flow_prepare_policer_rules(dev, fm, &attr);
if (ret)
goto error;
/* Add to the flow meter list. */
@@ -679,6 +701,9 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t
meter_id,
fm->shared = !!shared;
fm->policer_stats.stats_mask = params->stats_mask;
fm->profile->ref_cnt++;
+ fm->flow_ipool = mlx5_ipool_create(&flow_ipool_cfg);
+ if (!fm->flow_ipool)
+ goto error;
rte_spinlock_init(&fm->sl);
return 0;
error:
@@ -749,6 +774,8 @@ mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t
meter_id,
if (fm->policer_stats.cnt[i])
mlx5_counter_free(dev, fm->policer_stats.cnt[i]);
/* Free meter flow table */
+ if (fm->flow_ipool)
+ mlx5_ipool_destroy(fm->flow_ipool);
mlx5_flow_destroy_policer_rules(dev, fm, &attr);
mlx5_flow_destroy_mtr_tbls(dev, fm->mfts);
mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], fm->idx);
@@ -1153,30 +1180,24 @@ mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t
meter_id)
*
* @param [in] priv
* Pointer to mlx5 private data.
- * @param [in] meter_id
- * Flow meter id.
+ * @param[in] fm
+ * Pointer to flow meter.
* @param [in] attr
* Pointer to flow attributes.
* @param [out] error
* Pointer to error structure.
*
- * @return the flow meter pointer, NULL otherwise.
+ * @return
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
*/
-struct mlx5_flow_meter *
-mlx5_flow_meter_attach(struct mlx5_priv *priv, uint32_t meter_id,
+int
+mlx5_flow_meter_attach(struct mlx5_priv *priv,
+ struct mlx5_flow_meter *fm,
const struct rte_flow_attr *attr,
struct rte_flow_error *error)
{
- struct mlx5_flow_meter *fm;
int ret = 0;
- fm = mlx5_flow_meter_find(priv, meter_id);
- if (fm == NULL) {
- rte_flow_error_set(error, ENOENT,
- RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
- "Meter object id not valid");
- return fm;
- }
rte_spinlock_lock(&fm->sl);
if (fm->mfts->meter_action) {
if (fm->shared &&
@@ -1210,7 +1231,7 @@ mlx5_flow_meter_attach(struct mlx5_priv *priv, uint32_t
meter_id,
fm->mfts->meter_action ?
"Meter attr not match" :
"Meter action create failed");
- return ret ? NULL : fm;
+ return ret ? -rte_errno : 0;
}
/**
--
2.27.0
