From: Bing Zhao <bi...@nvidia.com>
The new mode 4 of devarg "dv_xmeta_en" is added for HWS only. In this
mode, the Rx / Tx metadata with 32b width copy between FDB and NIC is
supported. The mark is only supported in NIC and there is no copy
supported.
Signed-off-by: Bing Zhao <bi...@nvidia.com>
---
doc/guides/nics/mlx5.rst | 4 +
drivers/net/mlx5/linux/mlx5_os.c | 10 +-
drivers/net/mlx5/mlx5.c | 7 +-
drivers/net/mlx5/mlx5.h | 8 +-
drivers/net/mlx5/mlx5_flow.c | 8 +-
drivers/net/mlx5/mlx5_flow.h | 14 +
drivers/net/mlx5/mlx5_flow_dv.c | 43 +-
drivers/net/mlx5/mlx5_flow_hw.c | 864 ++++++++++++++++++++++++++++---
drivers/net/mlx5/mlx5_trigger.c | 3 +
9 files changed, 876 insertions(+), 85 deletions(-)
diff --git a/doc/guides/nics/mlx5.rst b/doc/guides/nics/mlx5.rst
index 7d2095f075..0c7bd042a4 100644
--- a/doc/guides/nics/mlx5.rst
+++ b/doc/guides/nics/mlx5.rst
@@ -980,6 +980,10 @@ for an additional list of options shared with other mlx5
drivers.
- 3, this engages tunnel offload mode. In E-Switch configuration, that
mode implicitly activates ``dv_xmeta_en=1``.
+ - 4, this mode only supported in HWS (``dv_flow_en=2``). The Rx / Tx
+ metadata with 32b width copy between FDB and NIC is supported. The
+ mark is only supported in NIC and there is no copy supported.
+
+------+-----------+-----------+-------------+-------------+
| Mode | ``MARK`` | ``META`` | ``META`` Tx | FDB/Through |
+======+===========+===========+=============+=============+
diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index d674b54624..c70cd84b8d 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -1554,6 +1554,15 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
#ifdef HAVE_MLX5_HWS_SUPPORT
if (priv->vport_meta_mask)
flow_hw_set_port_info(eth_dev);
+ if (priv->sh->config.dv_esw_en &&
+ priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
+ priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_META32_HWS)
{
+ DRV_LOG(ERR,
+ "metadata mode %u is not supported in HWS
eswitch mode",
+ priv->sh->config.dv_xmeta_en);
+ err = ENOTSUP;
+ goto error;
+ }
/* Only HWS requires this information. */
flow_hw_init_tags_set(eth_dev);
if (priv->sh->config.dv_esw_en &&
@@ -1569,7 +1578,6 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
goto error;
#endif
}
- /* Port representor shares the same max priority with pf port. */
if (!priv->sh->flow_priority_check_flag) {
/* Supported Verbs flow priority number detection. */
err = mlx5_flow_discover_priorities(eth_dev);
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 470b9c2d0f..9cd4892858 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -1218,7 +1218,8 @@ mlx5_dev_args_check_handler(const char *key, const char
*val, void *opaque)
if (tmp != MLX5_XMETA_MODE_LEGACY &&
tmp != MLX5_XMETA_MODE_META16 &&
tmp != MLX5_XMETA_MODE_META32 &&
- tmp != MLX5_XMETA_MODE_MISS_INFO) {
+ tmp != MLX5_XMETA_MODE_MISS_INFO &&
+ tmp != MLX5_XMETA_MODE_META32_HWS) {
DRV_LOG(ERR, "Invalid extensive metadata parameter.");
rte_errno = EINVAL;
return -rte_errno;
@@ -2849,6 +2850,10 @@ mlx5_set_metadata_mask(struct rte_eth_dev *dev)
meta = UINT32_MAX;
mark = (reg_c0 >> rte_bsf32(reg_c0)) & MLX5_FLOW_MARK_MASK;
break;
+ case MLX5_XMETA_MODE_META32_HWS:
+ meta = UINT32_MAX;
+ mark = MLX5_FLOW_MARK_MASK;
+ break;
default:
meta = 0;
mark = 0;
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 69a0a60030..6e7216efab 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -303,8 +303,8 @@ struct mlx5_sh_config {
uint32_t reclaim_mode:2; /* Memory reclaim mode. */
uint32_t dv_esw_en:1; /* Enable E-Switch DV flow. */
/* Enable DV flow. 1 means SW steering, 2 means HW steering. */
- unsigned int dv_flow_en:2;
- uint32_t dv_xmeta_en:2; /* Enable extensive flow metadata. */
+ uint32_t dv_flow_en:2; /* Enable DV flow. */
+ uint32_t dv_xmeta_en:3; /* Enable extensive flow metadata. */
uint32_t dv_miss_info:1; /* Restore packet after partial hw miss. */
uint32_t l3_vxlan_en:1; /* Enable L3 VXLAN flow creation. */
uint32_t vf_nl_en:1; /* Enable Netlink requests in VF mode. */
@@ -317,7 +317,6 @@ struct mlx5_sh_config {
uint32_t fdb_def_rule:1; /* Create FDB default jump rule */
};
-
/* Structure for VF VLAN workaround. */
struct mlx5_vf_vlan {
uint32_t tag:12;
@@ -1284,12 +1283,12 @@ struct mlx5_dev_ctx_shared {
struct mlx5_lb_ctx self_lb; /* QP to enable self loopback for Devx. */
unsigned int flow_max_priority;
enum modify_reg flow_mreg_c[MLX5_MREG_C_NUM];
+ /* Availability of mreg_c's. */
void *devx_channel_lwm;
struct rte_intr_handle *intr_handle_lwm;
pthread_mutex_t lwm_config_lock;
uint32_t host_shaper_rate:8;
uint32_t lwm_triggered:1;
- /* Availability of mreg_c's. */
struct mlx5_dev_shared_port port[]; /* per device port data array. */
};
@@ -1515,6 +1514,7 @@ struct mlx5_priv {
struct rte_flow_template_table *hw_esw_sq_miss_root_tbl;
struct rte_flow_template_table *hw_esw_sq_miss_tbl;
struct rte_flow_template_table *hw_esw_zero_tbl;
+ struct rte_flow_template_table *hw_tx_meta_cpy_tbl;
struct mlx5_indexed_pool *flows[MLX5_FLOW_TYPE_MAXI];
/* RTE Flow rules. */
uint32_t ctrl_flows; /* Control flow rules. */
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 60f76f5a43..3b8e97ccd0 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -1109,6 +1109,8 @@ mlx5_flow_get_reg_id(struct rte_eth_dev *dev,
return REG_C_0;
case MLX5_XMETA_MODE_META32:
return REG_C_1;
+ case MLX5_XMETA_MODE_META32_HWS:
+ return REG_C_1;
}
break;
case MLX5_METADATA_TX:
@@ -1121,11 +1123,14 @@ mlx5_flow_get_reg_id(struct rte_eth_dev *dev,
return REG_C_0;
case MLX5_XMETA_MODE_META32:
return REG_C_1;
+ case MLX5_XMETA_MODE_META32_HWS:
+ return REG_C_1;
}
break;
case MLX5_FLOW_MARK:
switch (config->dv_xmeta_en) {
case MLX5_XMETA_MODE_LEGACY:
+ case MLX5_XMETA_MODE_META32_HWS:
return REG_NON;
case MLX5_XMETA_MODE_META16:
return REG_C_1;
@@ -4444,7 +4449,8 @@ static bool flow_check_modify_action_type(struct
rte_eth_dev *dev,
return true;
case RTE_FLOW_ACTION_TYPE_FLAG:
case RTE_FLOW_ACTION_TYPE_MARK:
- if (priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY)
+ if (priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
+ priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_META32_HWS)
return true;
else
return false;
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index 25b44ccca2..b0af13886a 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -48,6 +48,12 @@ enum mlx5_rte_flow_action_type {
MLX5_RTE_FLOW_ACTION_TYPE_RSS,
};
+/* Private (internal) Field IDs for MODIFY_FIELD action. */
+enum mlx5_rte_flow_field_id {
+ MLX5_RTE_FLOW_FIELD_END = INT_MIN,
+ MLX5_RTE_FLOW_FIELD_META_REG,
+};
+
#define MLX5_INDIRECT_ACTION_TYPE_OFFSET 30
enum {
@@ -1178,6 +1184,7 @@ struct rte_flow_actions_template {
struct rte_flow_action *masks; /* Cached action masks.*/
uint16_t mhdr_off; /* Offset of DR modify header action. */
uint32_t refcnt; /* Reference counter. */
+ uint16_t rx_cpy_pos; /* Action position of Rx metadata to be copied. */
};
/* Jump action struct. */
@@ -1254,6 +1261,11 @@ struct mlx5_flow_group {
#define MLX5_HW_TBL_MAX_ITEM_TEMPLATE 2
#define MLX5_HW_TBL_MAX_ACTION_TEMPLATE 32
+struct mlx5_flow_template_table_cfg {
+ struct rte_flow_template_table_attr attr; /* Table attributes passed
through flow API. */
+ bool external; /* True if created by flow API, false if table is
internal to PMD. */
+};
+
struct rte_flow_template_table {
LIST_ENTRY(rte_flow_template_table) next;
struct mlx5_flow_group *grp; /* The group rte_flow_template_table uses.
*/
@@ -1263,6 +1275,7 @@ struct rte_flow_template_table {
/* Action templates bind to the table. */
struct mlx5_hw_action_template ats[MLX5_HW_TBL_MAX_ACTION_TEMPLATE];
struct mlx5_indexed_pool *flow; /* The table's flow ipool. */
+ struct mlx5_flow_template_table_cfg cfg;
uint32_t type; /* Flow table type RX/TX/FDB. */
uint8_t nb_item_templates; /* Item template number. */
uint8_t nb_action_templates; /* Action template number. */
@@ -2370,4 +2383,5 @@ int mlx5_flow_hw_esw_create_mgr_sq_miss_flow(struct
rte_eth_dev *dev);
int mlx5_flow_hw_esw_create_sq_miss_flow(struct rte_eth_dev *dev,
uint32_t txq);
int mlx5_flow_hw_esw_create_default_jump_flow(struct rte_eth_dev *dev);
+int mlx5_flow_hw_create_tx_default_mreg_copy_flow(struct rte_eth_dev *dev);
#endif /* RTE_PMD_MLX5_FLOW_H_ */
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 1ee26be975..a0bcaa5c53 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -1758,7 +1758,8 @@ mlx5_flow_field_id_to_modify_info
int reg;
if (priv->sh->config.dv_flow_en == 2)
- reg = REG_C_1;
+ reg = flow_hw_get_reg_id(RTE_FLOW_ITEM_TYPE_TAG,
+ data->level);
else
reg = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG,
data->level, error);
@@ -1837,6 +1838,24 @@ mlx5_flow_field_id_to_modify_info
else
info[idx].offset = off_be;
break;
+ case MLX5_RTE_FLOW_FIELD_META_REG:
+ {
+ uint32_t meta_mask = priv->sh->dv_meta_mask;
+ uint32_t meta_count = __builtin_popcount(meta_mask);
+ uint32_t reg = data->level;
+
+ RTE_SET_USED(meta_count);
+ MLX5_ASSERT(data->offset + width <= meta_count);
+ MLX5_ASSERT(reg != REG_NON);
+ MLX5_ASSERT(reg < RTE_DIM(reg_to_field));
+ info[idx] = (struct field_modify_info){4, 0,
reg_to_field[reg]};
+ if (mask)
+ mask[idx] = flow_modify_info_mask_32_masked
+ (width, data->offset, meta_mask);
+ else
+ info[idx].offset = data->offset;
+ }
+ break;
case RTE_FLOW_FIELD_POINTER:
case RTE_FLOW_FIELD_VALUE:
default:
@@ -9794,7 +9813,19 @@ flow_dv_translate_item_meta(struct rte_eth_dev *dev,
mask = meta_m->data;
if (key_type == MLX5_SET_MATCHER_HS_M)
mask = value;
- reg = flow_dv_get_metadata_reg(dev, attr, NULL);
+ /*
+ * In the current implementation, REG_B cannot be used to match.
+ * Force to use REG_C_1 in HWS root table as other tables.
+ * This map may change.
+ * NIC: modify - REG_B to be present in SW
+ * match - REG_C_1 when copied from FDB, different from SWS
+ * FDB: modify - REG_C_1 in Xmeta mode, REG_NON in legacy mode
+ * match - REG_C_1 in FDB
+ */
+ if (!!(key_type & MLX5_SET_MATCHER_SW))
+ reg = flow_dv_get_metadata_reg(dev, attr, NULL);
+ else
+ reg = flow_hw_get_reg_id(RTE_FLOW_ITEM_TYPE_META, 0);
if (reg < 0)
return;
MLX5_ASSERT(reg != REG_NON);
@@ -9894,7 +9925,10 @@ flow_dv_translate_item_tag(struct rte_eth_dev *dev, void
*key,
/* When set mask, the index should be from spec. */
index = tag_vv ? tag_vv->index : tag_v->index;
/* Get the metadata register index for the tag. */
- reg = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, index, NULL);
+ if (!!(key_type & MLX5_SET_MATCHER_SW))
+ reg = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, index, NULL);
+ else
+ reg = flow_hw_get_reg_id(RTE_FLOW_ITEM_TYPE_TAG, index);
MLX5_ASSERT(reg > 0);
flow_dv_match_meta_reg(key, reg, tag_v->data, tag_m->data);
}
@@ -13412,7 +13446,8 @@ flow_dv_translate_items_sws(struct rte_eth_dev *dev,
*/
if (!(wks.item_flags & MLX5_FLOW_ITEM_PORT_ID) &&
!(wks.item_flags & MLX5_FLOW_ITEM_REPRESENTED_PORT) &&
priv->sh->esw_mode &&
- !(attr->egress && !attr->transfer)) {
+ !(attr->egress && !attr->transfer) &&
+ attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP) {
if (flow_dv_translate_item_port_id_all(dev, match_mask,
match_value, NULL, attr))
return -rte_errno;
diff --git a/drivers/net/mlx5/mlx5_flow_hw.c b/drivers/net/mlx5/mlx5_flow_hw.c
index 991e4c9b7b..319c8d1a89 100644
--- a/drivers/net/mlx5/mlx5_flow_hw.c
+++ b/drivers/net/mlx5/mlx5_flow_hw.c
@@ -20,13 +20,27 @@
/* Default queue to flush the flows. */
#define MLX5_DEFAULT_FLUSH_QUEUE 0
-/* Maximum number of rules in control flow tables */
+/* Maximum number of rules in control flow tables. */
#define MLX5_HW_CTRL_FLOW_NB_RULES (4096)
-/* Flow group for SQ miss default flows/ */
-#define MLX5_HW_SQ_MISS_GROUP (UINT32_MAX)
+/* Lowest flow group usable by an application. */
+#define MLX5_HW_LOWEST_USABLE_GROUP (1)
+
+/* Maximum group index usable by user applications for transfer flows. */
+#define MLX5_HW_MAX_TRANSFER_GROUP (UINT32_MAX - 1)
+
+/* Lowest priority for HW root table. */
+#define MLX5_HW_LOWEST_PRIO_ROOT 15
+
+/* Lowest priority for HW non-root table. */
+#define MLX5_HW_LOWEST_PRIO_NON_ROOT (UINT32_MAX)
static int flow_hw_flush_all_ctrl_flows(struct rte_eth_dev *dev);
+static int flow_hw_translate_group(struct rte_eth_dev *dev,
+ const struct mlx5_flow_template_table_cfg
*cfg,
+ uint32_t group,
+ uint32_t *table_group,
+ struct rte_flow_error *error);
const struct mlx5_flow_driver_ops mlx5_flow_hw_drv_ops;
@@ -213,12 +227,12 @@ flow_hw_rss_item_flags_get(const struct rte_flow_item
items[])
*/
static struct mlx5_hw_jump_action *
flow_hw_jump_action_register(struct rte_eth_dev *dev,
- const struct rte_flow_attr *attr,
+ const struct mlx5_flow_template_table_cfg *cfg,
uint32_t dest_group,
struct rte_flow_error *error)
{
struct mlx5_priv *priv = dev->data->dev_private;
- struct rte_flow_attr jattr = *attr;
+ struct rte_flow_attr jattr = cfg->attr.flow_attr;
struct mlx5_flow_group *grp;
struct mlx5_flow_cb_ctx ctx = {
.dev = dev,
@@ -226,9 +240,13 @@ flow_hw_jump_action_register(struct rte_eth_dev *dev,
.data = &jattr,
};
struct mlx5_list_entry *ge;
+ uint32_t target_group;
- jattr.group = dest_group;
- ge = mlx5_hlist_register(priv->sh->flow_tbls, dest_group, &ctx);
+ target_group = dest_group;
+ if (flow_hw_translate_group(dev, cfg, dest_group, &target_group, error))
+ return NULL;
+ jattr.group = target_group;
+ ge = mlx5_hlist_register(priv->sh->flow_tbls, target_group, &ctx);
if (!ge)
return NULL;
grp = container_of(ge, struct mlx5_flow_group, entry);
@@ -760,7 +778,8 @@ flow_hw_modify_field_compile(struct rte_eth_dev *dev,
(void *)(uintptr_t)conf->src.pvalue :
(void *)(uintptr_t)&conf->src.value;
if (conf->dst.field == RTE_FLOW_FIELD_META ||
- conf->dst.field == RTE_FLOW_FIELD_TAG) {
+ conf->dst.field == RTE_FLOW_FIELD_TAG ||
+ conf->dst.field == (enum
rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG) {
value = *(const unaligned_uint32_t *)item.spec;
value = rte_cpu_to_be_32(value);
item.spec = &value;
@@ -860,6 +879,9 @@ flow_hw_represented_port_compile(struct rte_eth_dev *dev,
if (m && !!m->port_id) {
struct mlx5_priv *port_priv;
+ if (!v)
+ return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ACTION,
+ action, "port index was not
provided");
port_priv = mlx5_port_to_eswitch_info(v->port_id, false);
if (port_priv == NULL)
return rte_flow_error_set
@@ -903,8 +925,8 @@ flow_hw_represented_port_compile(struct rte_eth_dev *dev,
*
* @param[in] dev
* Pointer to the rte_eth_dev structure.
- * @param[in] table_attr
- * Pointer to the table attributes.
+ * @param[in] cfg
+ * Pointer to the table configuration.
* @param[in] item_templates
* Item template array to be binded to the table.
* @param[in/out] acts
@@ -919,12 +941,13 @@ flow_hw_represented_port_compile(struct rte_eth_dev *dev,
*/
static int
flow_hw_actions_translate(struct rte_eth_dev *dev,
- const struct rte_flow_template_table_attr *table_attr,
+ const struct mlx5_flow_template_table_cfg *cfg,
struct mlx5_hw_actions *acts,
struct rte_flow_actions_template *at,
struct rte_flow_error *error)
{
struct mlx5_priv *priv = dev->data->dev_private;
+ const struct rte_flow_template_table_attr *table_attr = &cfg->attr;
const struct rte_flow_attr *attr = &table_attr->flow_attr;
struct rte_flow_action *actions = at->actions;
struct rte_flow_action *action_start = actions;
@@ -991,7 +1014,7 @@ flow_hw_actions_translate(struct rte_eth_dev *dev,
((const struct rte_flow_action_jump *)
actions->conf)->group;
acts->jump = flow_hw_jump_action_register
- (dev, attr, jump_group, error);
+ (dev, cfg, jump_group, error);
if (!acts->jump)
goto err;
acts->rule_acts[i].action = (!!attr->group) ?
@@ -1101,6 +1124,16 @@ flow_hw_actions_translate(struct rte_eth_dev *dev,
error);
if (err)
goto err;
+ /*
+ * Adjust the action source position for the following.
+ * ... / MODIFY_FIELD: rx_cpy_pos / (QUEUE|RSS) / ...
+ * The next action will be Q/RSS, there will not be
+ * another adjustment and the real source position of
+ * the following actions will be decreased by 1.
+ * No change of the total actions in the new template.
+ */
+ if ((actions - action_start) == at->rx_cpy_pos)
+ action_start += 1;
break;
case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT:
if (flow_hw_represented_port_compile
@@ -1365,7 +1398,8 @@ flow_hw_modify_field_construct(struct mlx5_hw_q_job *job,
else
rte_memcpy(values, mhdr_action->src.pvalue, sizeof(values));
if (mhdr_action->dst.field == RTE_FLOW_FIELD_META ||
- mhdr_action->dst.field == RTE_FLOW_FIELD_TAG) {
+ mhdr_action->dst.field == RTE_FLOW_FIELD_TAG ||
+ mhdr_action->dst.field == (enum
rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG) {
value_p = (unaligned_uint32_t *)values;
*value_p = rte_cpu_to_be_32(*value_p);
} else if (mhdr_action->dst.field == RTE_FLOW_FIELD_GTP_PSC_QFI) {
@@ -1513,7 +1547,7 @@ flow_hw_actions_construct(struct rte_eth_dev *dev,
jump_group = ((const struct rte_flow_action_jump *)
action->conf)->group;
jump = flow_hw_jump_action_register
- (dev, &attr, jump_group, NULL);
+ (dev, &table->cfg, jump_group, NULL);
if (!jump)
return -1;
rule_acts[act_data->action_dst].action =
@@ -1710,7 +1744,13 @@ flow_hw_async_flow_create(struct rte_eth_dev *dev,
job->user_data = user_data;
rule_attr.user_data = job;
hw_acts = &table->ats[action_template_index].acts;
- /* Construct the flow actions based on the input actions.*/
+ /*
+ * Construct the flow actions based on the input actions.
+ * The implicitly appended action is always fixed, like metadata
+ * copy action from FDB to NIC Rx.
+ * No need to copy and contrust a new "actions" list based on the
+ * user's input, in order to save the cost.
+ */
if (flow_hw_actions_construct(dev, job, hw_acts, pattern_template_index,
actions, rule_acts, &acts_num)) {
rte_errno = EINVAL;
@@ -1981,6 +2021,8 @@ flow_hw_q_flow_flush(struct rte_eth_dev *dev,
/* Flush flow per-table from MLX5_DEFAULT_FLUSH_QUEUE. */
hw_q = &priv->hw_q[MLX5_DEFAULT_FLUSH_QUEUE];
LIST_FOREACH(tbl, &priv->flow_hw_tbl, next) {
+ if (!tbl->cfg.external)
+ continue;
MLX5_IPOOL_FOREACH(tbl->flow, fidx, flow) {
if (flow_hw_async_flow_destroy(dev,
MLX5_DEFAULT_FLUSH_QUEUE,
@@ -2018,8 +2060,8 @@ flow_hw_q_flow_flush(struct rte_eth_dev *dev,
*
* @param[in] dev
* Pointer to the rte_eth_dev structure.
- * @param[in] attr
- * Pointer to the table attributes.
+ * @param[in] table_cfg
+ * Pointer to the table configuration.
* @param[in] item_templates
* Item template array to be binded to the table.
* @param[in] nb_item_templates
@@ -2036,7 +2078,7 @@ flow_hw_q_flow_flush(struct rte_eth_dev *dev,
*/
static struct rte_flow_template_table *
flow_hw_table_create(struct rte_eth_dev *dev,
- const struct rte_flow_template_table_attr *attr,
+ const struct mlx5_flow_template_table_cfg *table_cfg,
struct rte_flow_pattern_template *item_templates[],
uint8_t nb_item_templates,
struct rte_flow_actions_template *action_templates[],
@@ -2048,6 +2090,7 @@ flow_hw_table_create(struct rte_eth_dev *dev,
struct rte_flow_template_table *tbl = NULL;
struct mlx5_flow_group *grp;
struct mlx5dr_match_template *mt[MLX5_HW_TBL_MAX_ITEM_TEMPLATE];
+ const struct rte_flow_template_table_attr *attr = &table_cfg->attr;
struct rte_flow_attr flow_attr = attr->flow_attr;
struct mlx5_flow_cb_ctx ctx = {
.dev = dev,
@@ -2088,6 +2131,7 @@ flow_hw_table_create(struct rte_eth_dev *dev,
tbl = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*tbl), 0, rte_socket_id());
if (!tbl)
goto error;
+ tbl->cfg = *table_cfg;
/* Allocate flow indexed pool. */
tbl->flow = mlx5_ipool_create(&cfg);
if (!tbl->flow)
@@ -2131,7 +2175,7 @@ flow_hw_table_create(struct rte_eth_dev *dev,
goto at_error;
}
LIST_INIT(&tbl->ats[i].acts.act_list);
- err = flow_hw_actions_translate(dev, attr,
+ err = flow_hw_actions_translate(dev, &tbl->cfg,
&tbl->ats[i].acts,
action_templates[i], error);
if (err) {
@@ -2174,6 +2218,96 @@ flow_hw_table_create(struct rte_eth_dev *dev,
return NULL;
}
+/**
+ * Translates group index specified by the user in @p attr to internal
+ * group index.
+ *
+ * Translation is done by incrementing group index, so group n becomes n + 1.
+ *
+ * @param[in] dev
+ * Pointer to Ethernet device.
+ * @param[in] cfg
+ * Pointer to the template table configuration.
+ * @param[in] group
+ * Currently used group index (table group or jump destination).
+ * @param[out] table_group
+ * Pointer to output group index.
+ * @param[out] error
+ * Pointer to error structure.
+ *
+ * @return
+ * 0 on success. Otherwise, returns negative error code, rte_errno is set
+ * and error structure is filled.
+ */
+static int
+flow_hw_translate_group(struct rte_eth_dev *dev,
+ const struct mlx5_flow_template_table_cfg *cfg,
+ uint32_t group,
+ uint32_t *table_group,
+ struct rte_flow_error *error)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+ const struct rte_flow_attr *flow_attr = &cfg->attr.flow_attr;
+
+ if (priv->sh->config.dv_esw_en && cfg->external && flow_attr->transfer)
{
+ if (group > MLX5_HW_MAX_TRANSFER_GROUP)
+ return rte_flow_error_set(error, EINVAL,
+
RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
+ NULL,
+ "group index not supported");
+ *table_group = group + 1;
+ } else {
+ *table_group = group;
+ }
+ return 0;
+}
+
+/**
+ * Create flow table.
+ *
+ * This function is a wrapper over @ref flow_hw_table_create(), which
translates parameters
+ * provided by user to proper internal values.
+ *
+ * @param[in] dev
+ * Pointer to Ethernet device.
+ * @param[in] attr
+ * Pointer to the table attributes.
+ * @param[in] item_templates
+ * Item template array to be binded to the table.
+ * @param[in] nb_item_templates
+ * Number of item templates.
+ * @param[in] action_templates
+ * Action template array to be binded to the table.
+ * @param[in] nb_action_templates
+ * Number of action templates.
+ * @param[out] error
+ * Pointer to error structure.
+ *
+ * @return
+ * Table on success, Otherwise, returns negative error code, rte_errno is set
+ * and error structure is filled.
+ */
+static struct rte_flow_template_table *
+flow_hw_template_table_create(struct rte_eth_dev *dev,
+ const struct rte_flow_template_table_attr *attr,
+ struct rte_flow_pattern_template
*item_templates[],
+ uint8_t nb_item_templates,
+ struct rte_flow_actions_template
*action_templates[],
+ uint8_t nb_action_templates,
+ struct rte_flow_error *error)
+{
+ struct mlx5_flow_template_table_cfg cfg = {
+ .attr = *attr,
+ .external = true,
+ };
+ uint32_t group = attr->flow_attr.group;
+
+ if (flow_hw_translate_group(dev, &cfg, group,
&cfg.attr.flow_attr.group, error))
+ return NULL;
+ return flow_hw_table_create(dev, &cfg, item_templates,
nb_item_templates,
+ action_templates, nb_action_templates,
error);
+}
+
/**
* Destroy flow table.
*
@@ -2309,10 +2443,13 @@ flow_hw_validate_action_represented_port(struct
rte_eth_dev *dev,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
"cannot use represented_port actions"
" without an E-Switch");
- if (mask_conf->port_id) {
+ if (mask_conf && mask_conf->port_id) {
struct mlx5_priv *port_priv;
struct mlx5_priv *dev_priv;
+ if (!action_conf)
+ return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ACTION,
+ action, "port index was not
provided");
port_priv = mlx5_port_to_eswitch_info(action_conf->port_id,
false);
if (!port_priv)
return rte_flow_error_set(error, rte_errno,
@@ -2337,20 +2474,77 @@ flow_hw_validate_action_represented_port(struct
rte_eth_dev *dev,
return 0;
}
+static inline int
+flow_hw_action_meta_copy_insert(const struct rte_flow_action actions[],
+ const struct rte_flow_action masks[],
+ const struct rte_flow_action *ins_actions,
+ const struct rte_flow_action *ins_masks,
+ struct rte_flow_action *new_actions,
+ struct rte_flow_action *new_masks,
+ uint16_t *ins_pos)
+{
+ uint16_t idx, total = 0;
+ bool ins = false;
+ bool act_end = false;
+
+ MLX5_ASSERT(actions && masks);
+ MLX5_ASSERT(new_actions && new_masks);
+ MLX5_ASSERT(ins_actions && ins_masks);
+ for (idx = 0; !act_end; idx++) {
+ if (idx >= MLX5_HW_MAX_ACTS)
+ return -1;
+ if (actions[idx].type == RTE_FLOW_ACTION_TYPE_RSS ||
+ actions[idx].type == RTE_FLOW_ACTION_TYPE_QUEUE) {
+ ins = true;
+ *ins_pos = idx;
+ }
+ if (actions[idx].type == RTE_FLOW_ACTION_TYPE_END)
+ act_end = true;
+ }
+ if (!ins)
+ return 0;
+ else if (idx == MLX5_HW_MAX_ACTS)
+ return -1; /* No more space. */
+ total = idx;
+ /* Before the position, no change for the actions. */
+ for (idx = 0; idx < *ins_pos; idx++) {
+ new_actions[idx] = actions[idx];
+ new_masks[idx] = masks[idx];
+ }
+ /* Insert the new action and mask to the position. */
+ new_actions[idx] = *ins_actions;
+ new_masks[idx] = *ins_masks;
+ /* Remaining content is right shifted by one position. */
+ for (; idx < total; idx++) {
+ new_actions[idx + 1] = actions[idx];
+ new_masks[idx + 1] = masks[idx];
+ }
+ return 0;
+}
+
static int
flow_hw_action_validate(struct rte_eth_dev *dev,
+ const struct rte_flow_actions_template_attr *attr,
const struct rte_flow_action actions[],
const struct rte_flow_action masks[],
struct rte_flow_error *error)
{
- int i;
+ struct mlx5_priv *priv = dev->data->dev_private;
+ uint16_t i;
bool actions_end = false;
int ret;
+ /* FDB actions are only valid to proxy port. */
+ if (attr->transfer && (!priv->sh->config.dv_esw_en || !priv->master))
+ return rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+ NULL,
+ "transfer actions are only valid to
proxy port");
for (i = 0; !actions_end; ++i) {
const struct rte_flow_action *action = &actions[i];
const struct rte_flow_action *mask = &masks[i];
+ MLX5_ASSERT(i < MLX5_HW_MAX_ACTS);
if (action->type != mask->type)
return rte_flow_error_set(error, ENOTSUP,
RTE_FLOW_ERROR_TYPE_ACTION,
@@ -2447,21 +2641,77 @@ flow_hw_actions_template_create(struct rte_eth_dev *dev,
{
struct mlx5_priv *priv = dev->data->dev_private;
int len, act_len, mask_len, i;
- struct rte_flow_actions_template *at;
+ struct rte_flow_actions_template *at = NULL;
+ uint16_t pos = MLX5_HW_MAX_ACTS;
+ struct rte_flow_action tmp_action[MLX5_HW_MAX_ACTS];
+ struct rte_flow_action tmp_mask[MLX5_HW_MAX_ACTS];
+ const struct rte_flow_action *ra;
+ const struct rte_flow_action *rm;
+ const struct rte_flow_action_modify_field rx_mreg = {
+ .operation = RTE_FLOW_MODIFY_SET,
+ .dst = {
+ .field = (enum
rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
+ .level = REG_B,
+ },
+ .src = {
+ .field = (enum
rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
+ .level = REG_C_1,
+ },
+ .width = 32,
+ };
+ const struct rte_flow_action_modify_field rx_mreg_mask = {
+ .operation = RTE_FLOW_MODIFY_SET,
+ .dst = {
+ .field = (enum
rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
+ .level = UINT32_MAX,
+ .offset = UINT32_MAX,
+ },
+ .src = {
+ .field = (enum
rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
+ .level = UINT32_MAX,
+ .offset = UINT32_MAX,
+ },
+ .width = UINT32_MAX,
+ };
+ const struct rte_flow_action rx_cpy = {
+ .type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD,
+ .conf = &rx_mreg,
+ };
+ const struct rte_flow_action rx_cpy_mask = {
+ .type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD,
+ .conf = &rx_mreg_mask,
+ };
- if (flow_hw_action_validate(dev, actions, masks, error))
+ if (flow_hw_action_validate(dev, attr, actions, masks, error))
return NULL;
- act_len = rte_flow_conv(RTE_FLOW_CONV_OP_ACTIONS,
- NULL, 0, actions, error);
+ if (priv->sh->config.dv_xmeta_en == MLX5_XMETA_MODE_META32_HWS &&
+ priv->sh->config.dv_esw_en) {
+ if (flow_hw_action_meta_copy_insert(actions, masks, &rx_cpy,
&rx_cpy_mask,
+ tmp_action, tmp_mask,
&pos)) {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ACTION, NULL,
+ "Failed to concatenate new
action/mask");
+ return NULL;
+ }
+ }
+ /* Application should make sure only one Q/RSS exist in one rule. */
+ if (pos == MLX5_HW_MAX_ACTS) {
+ ra = actions;
+ rm = masks;
+ } else {
+ ra = tmp_action;
+ rm = tmp_mask;
+ }
+ act_len = rte_flow_conv(RTE_FLOW_CONV_OP_ACTIONS, NULL, 0, ra, error);
if (act_len <= 0)
return NULL;
len = RTE_ALIGN(act_len, 16);
- mask_len = rte_flow_conv(RTE_FLOW_CONV_OP_ACTIONS,
- NULL, 0, masks, error);
+ mask_len = rte_flow_conv(RTE_FLOW_CONV_OP_ACTIONS, NULL, 0, rm, error);
if (mask_len <= 0)
return NULL;
len += RTE_ALIGN(mask_len, 16);
- at = mlx5_malloc(MLX5_MEM_ZERO, len + sizeof(*at), 64, rte_socket_id());
+ at = mlx5_malloc(MLX5_MEM_ZERO, len + sizeof(*at),
+ RTE_CACHE_LINE_SIZE, rte_socket_id());
if (!at) {
rte_flow_error_set(error, ENOMEM,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
@@ -2469,18 +2719,20 @@ flow_hw_actions_template_create(struct rte_eth_dev *dev,
"cannot allocate action template");
return NULL;
}
+ /* Actions part is in the first half. */
at->attr = *attr;
at->actions = (struct rte_flow_action *)(at + 1);
- act_len = rte_flow_conv(RTE_FLOW_CONV_OP_ACTIONS, at->actions, len,
- actions, error);
+ act_len = rte_flow_conv(RTE_FLOW_CONV_OP_ACTIONS, at->actions,
+ len, ra, error);
if (act_len <= 0)
goto error;
- at->masks = (struct rte_flow_action *)
- (((uint8_t *)at->actions) + act_len);
+ /* Masks part is in the second half. */
+ at->masks = (struct rte_flow_action *)(((uint8_t *)at->actions) +
act_len);
mask_len = rte_flow_conv(RTE_FLOW_CONV_OP_ACTIONS, at->masks,
- len - act_len, masks, error);
+ len - act_len, rm, error);
if (mask_len <= 0)
goto error;
+ at->rx_cpy_pos = pos;
/*
* mlx5 PMD hacks indirect action index directly to the action conf.
* The rte_flow_conv() function copies the content from conf pointer.
@@ -2497,7 +2749,8 @@ flow_hw_actions_template_create(struct rte_eth_dev *dev,
LIST_INSERT_HEAD(&priv->flow_hw_at, at, next);
return at;
error:
- mlx5_free(at);
+ if (at)
+ mlx5_free(at);
return NULL;
}
@@ -2572,6 +2825,80 @@ flow_hw_copy_prepend_port_item(const struct
rte_flow_item *items,
return copied_items;
}
+static int
+flow_hw_pattern_validate(struct rte_eth_dev *dev,
+ const struct rte_flow_pattern_template_attr *attr,
+ const struct rte_flow_item items[],
+ struct rte_flow_error *error)
+{
+ int i;
+ bool items_end = false;
+ RTE_SET_USED(dev);
+ RTE_SET_USED(attr);
+
+ for (i = 0; !items_end; i++) {
+ int type = items[i].type;
+
+ switch (type) {
+ case RTE_FLOW_ITEM_TYPE_TAG:
+ {
+ int reg;
+ const struct rte_flow_item_tag *tag =
+ (const struct rte_flow_item_tag *)items[i].spec;
+
+ reg = flow_hw_get_reg_id(RTE_FLOW_ITEM_TYPE_TAG,
tag->index);
+ if (reg == REG_NON)
+ return rte_flow_error_set(error, EINVAL,
+
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+ NULL,
+ "Unsupported tag
index");
+ break;
+ }
+ case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
+ {
+ const struct rte_flow_item_tag *tag =
+ (const struct rte_flow_item_tag *)items[i].spec;
+ struct mlx5_priv *priv = dev->data->dev_private;
+ uint8_t regcs =
(uint8_t)priv->sh->cdev->config.hca_attr.set_reg_c;
+
+ if (!((1 << (tag->index - REG_C_0)) & regcs))
+ return rte_flow_error_set(error, EINVAL,
+
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+ NULL,
+ "Unsupported internal
tag index");
+ }
+ case RTE_FLOW_ITEM_TYPE_VOID:
+ case RTE_FLOW_ITEM_TYPE_ETH:
+ case RTE_FLOW_ITEM_TYPE_VLAN:
+ case RTE_FLOW_ITEM_TYPE_IPV4:
+ case RTE_FLOW_ITEM_TYPE_IPV6:
+ case RTE_FLOW_ITEM_TYPE_UDP:
+ case RTE_FLOW_ITEM_TYPE_TCP:
+ case RTE_FLOW_ITEM_TYPE_GTP:
+ case RTE_FLOW_ITEM_TYPE_GTP_PSC:
+ case RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT:
+ case RTE_FLOW_ITEM_TYPE_VXLAN:
+ case MLX5_RTE_FLOW_ITEM_TYPE_SQ:
+ case RTE_FLOW_ITEM_TYPE_META:
+ case RTE_FLOW_ITEM_TYPE_GRE:
+ case RTE_FLOW_ITEM_TYPE_GRE_KEY:
+ case RTE_FLOW_ITEM_TYPE_GRE_OPTION:
+ case RTE_FLOW_ITEM_TYPE_ICMP:
+ case RTE_FLOW_ITEM_TYPE_ICMP6:
+ break;
+ case RTE_FLOW_ITEM_TYPE_END:
+ items_end = true;
+ break;
+ default:
+ return rte_flow_error_set(error, EINVAL,
+
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+ NULL,
+ "Unsupported item type");
+ }
+ }
+ return 0;
+}
+
/**
* Create flow item template.
*
@@ -2598,6 +2925,8 @@ flow_hw_pattern_template_create(struct rte_eth_dev *dev,
struct rte_flow_item *copied_items = NULL;
const struct rte_flow_item *tmpl_items;
+ if (flow_hw_pattern_validate(dev, attr, items, error))
+ return NULL;
if (priv->sh->config.dv_esw_en && attr->ingress) {
/*
* Disallow pattern template with ingress and egress/transfer
@@ -3032,6 +3361,17 @@ flow_hw_free_vport_actions(struct mlx5_priv *priv)
priv->hw_vport = NULL;
}
+static uint32_t
+flow_hw_usable_lsb_vport_mask(struct mlx5_priv *priv)
+{
+ uint32_t usable_mask = ~priv->vport_meta_mask;
+
+ if (usable_mask)
+ return (1 << rte_bsf32(usable_mask));
+ else
+ return 0;
+}
+
/**
* Creates a flow pattern template used to match on E-Switch Manager.
* This template is used to set up a table for SQ miss default flow.
@@ -3070,7 +3410,10 @@ flow_hw_create_ctrl_esw_mgr_pattern_template(struct
rte_eth_dev *dev)
}
/**
- * Creates a flow pattern template used to match on a TX queue.
+ * Creates a flow pattern template used to match REG_C_0 and a TX queue.
+ * Matching on REG_C_0 is set up to match on least significant bit usable
+ * by user-space, which is set when packet was originated from E-Switch
Manager.
+ *
* This template is used to set up a table for SQ miss default flow.
*
* @param dev
@@ -3080,16 +3423,30 @@ flow_hw_create_ctrl_esw_mgr_pattern_template(struct
rte_eth_dev *dev)
* Pointer to flow pattern template on success, NULL otherwise.
*/
static struct rte_flow_pattern_template *
-flow_hw_create_ctrl_sq_pattern_template(struct rte_eth_dev *dev)
+flow_hw_create_ctrl_regc_sq_pattern_template(struct rte_eth_dev *dev)
{
+ struct mlx5_priv *priv = dev->data->dev_private;
+ uint32_t marker_bit = flow_hw_usable_lsb_vport_mask(priv);
struct rte_flow_pattern_template_attr attr = {
.relaxed_matching = 0,
.transfer = 1,
};
+ struct rte_flow_item_tag reg_c0_spec = {
+ .index = (uint8_t)REG_C_0,
+ };
+ struct rte_flow_item_tag reg_c0_mask = {
+ .index = 0xff,
+ };
struct mlx5_rte_flow_item_sq queue_mask = {
.queue = UINT32_MAX,
};
struct rte_flow_item items[] = {
+ {
+ .type = (enum rte_flow_item_type)
+ MLX5_RTE_FLOW_ITEM_TYPE_TAG,
+ .spec = ®_c0_spec,
+ .mask = ®_c0_mask,
+ },
{
.type = (enum rte_flow_item_type)
MLX5_RTE_FLOW_ITEM_TYPE_SQ,
@@ -3100,6 +3457,12 @@ flow_hw_create_ctrl_sq_pattern_template(struct
rte_eth_dev *dev)
},
};
+ if (!marker_bit) {
+ DRV_LOG(ERR, "Unable to set up pattern template for SQ miss
table");
+ return NULL;
+ }
+ reg_c0_spec.data = marker_bit;
+ reg_c0_mask.data = marker_bit;
return flow_hw_pattern_template_create(dev, &attr, items, NULL);
}
@@ -3137,6 +3500,132 @@ flow_hw_create_ctrl_port_pattern_template(struct
rte_eth_dev *dev)
return flow_hw_pattern_template_create(dev, &attr, items, NULL);
}
+/*
+ * Creating a flow pattern template with all ETH packets matching.
+ * This template is used to set up a table for default Tx copy (Tx metadata
+ * to REG_C_1) flow rule usage.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ *
+ * @return
+ * Pointer to flow pattern template on success, NULL otherwise.
+ */
+static struct rte_flow_pattern_template *
+flow_hw_create_tx_default_mreg_copy_pattern_template(struct rte_eth_dev *dev)
+{
+ struct rte_flow_pattern_template_attr tx_pa_attr = {
+ .relaxed_matching = 0,
+ .egress = 1,
+ };
+ struct rte_flow_item_eth promisc = {
+ .dst.addr_bytes = "\x00\x00\x00\x00\x00\x00",
+ .src.addr_bytes = "\x00\x00\x00\x00\x00\x00",
+ .type = 0,
+ };
+ struct rte_flow_item eth_all[] = {
+ [0] = {
+ .type = RTE_FLOW_ITEM_TYPE_ETH,
+ .spec = &promisc,
+ .mask = &promisc,
+ },
+ [1] = {
+ .type = RTE_FLOW_ITEM_TYPE_END,
+ },
+ };
+ struct rte_flow_error drop_err;
+
+ RTE_SET_USED(drop_err);
+ return flow_hw_pattern_template_create(dev, &tx_pa_attr, eth_all,
&drop_err);
+}
+
+/**
+ * Creates a flow actions template with modify field action and masked jump
action.
+ * Modify field action sets the least significant bit of REG_C_0 (usable by
user-space)
+ * to 1, meaning that packet was originated from E-Switch Manager. Jump action
+ * transfers steering to group 1.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ *
+ * @return
+ * Pointer to flow actions template on success, NULL otherwise.
+ */
+static struct rte_flow_actions_template *
+flow_hw_create_ctrl_regc_jump_actions_template(struct rte_eth_dev *dev)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+ uint32_t marker_bit = flow_hw_usable_lsb_vport_mask(priv);
+ uint32_t marker_bit_mask = UINT32_MAX;
+ struct rte_flow_actions_template_attr attr = {
+ .transfer = 1,
+ };
+ struct rte_flow_action_modify_field set_reg_v = {
+ .operation = RTE_FLOW_MODIFY_SET,
+ .dst = {
+ .field = (enum
rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
+ .level = REG_C_0,
+ },
+ .src = {
+ .field = RTE_FLOW_FIELD_VALUE,
+ },
+ .width = 1,
+ };
+ struct rte_flow_action_modify_field set_reg_m = {
+ .operation = RTE_FLOW_MODIFY_SET,
+ .dst = {
+ .field = (enum
rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
+ .level = UINT32_MAX,
+ .offset = UINT32_MAX,
+ },
+ .src = {
+ .field = RTE_FLOW_FIELD_VALUE,
+ },
+ .width = UINT32_MAX,
+ };
+ struct rte_flow_action_jump jump_v = {
+ .group = MLX5_HW_LOWEST_USABLE_GROUP,
+ };
+ struct rte_flow_action_jump jump_m = {
+ .group = UINT32_MAX,
+ };
+ struct rte_flow_action actions_v[] = {
+ {
+ .type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD,
+ .conf = &set_reg_v,
+ },
+ {
+ .type = RTE_FLOW_ACTION_TYPE_JUMP,
+ .conf = &jump_v,
+ },
+ {
+ .type = RTE_FLOW_ACTION_TYPE_END,
+ }
+ };
+ struct rte_flow_action actions_m[] = {
+ {
+ .type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD,
+ .conf = &set_reg_m,
+ },
+ {
+ .type = RTE_FLOW_ACTION_TYPE_JUMP,
+ .conf = &jump_m,
+ },
+ {
+ .type = RTE_FLOW_ACTION_TYPE_END,
+ }
+ };
+
+ if (!marker_bit) {
+ DRV_LOG(ERR, "Unable to set up actions template for SQ miss
table");
+ return NULL;
+ }
+ set_reg_v.dst.offset = rte_bsf32(marker_bit);
+ rte_memcpy(set_reg_v.src.value, &marker_bit, sizeof(marker_bit));
+ rte_memcpy(set_reg_m.src.value, &marker_bit_mask,
sizeof(marker_bit_mask));
+ return flow_hw_actions_template_create(dev, &attr, actions_v,
actions_m, NULL);
+}
+
/**
* Creates a flow actions template with an unmasked JUMP action. Flows
* based on this template will perform a jump to some group. This template
@@ -3231,6 +3720,73 @@ flow_hw_create_ctrl_port_actions_template(struct
rte_eth_dev *dev)
NULL);
}
+/*
+ * Creating an actions template to use header modify action for register
+ * copying. This template is used to set up a table for copy flow.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ *
+ * @return
+ * Pointer to flow actions template on success, NULL otherwise.
+ */
+static struct rte_flow_actions_template *
+flow_hw_create_tx_default_mreg_copy_actions_template(struct rte_eth_dev *dev)
+{
+ struct rte_flow_actions_template_attr tx_act_attr = {
+ .egress = 1,
+ };
+ const struct rte_flow_action_modify_field mreg_action = {
+ .operation = RTE_FLOW_MODIFY_SET,
+ .dst = {
+ .field = (enum
rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
+ .level = REG_C_1,
+ },
+ .src = {
+ .field = (enum
rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
+ .level = REG_A,
+ },
+ .width = 32,
+ };
+ const struct rte_flow_action_modify_field mreg_mask = {
+ .operation = RTE_FLOW_MODIFY_SET,
+ .dst = {
+ .field = (enum
rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
+ .level = UINT32_MAX,
+ .offset = UINT32_MAX,
+ },
+ .src = {
+ .field = (enum
rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
+ .level = UINT32_MAX,
+ .offset = UINT32_MAX,
+ },
+ .width = UINT32_MAX,
+ };
+ const struct rte_flow_action copy_reg_action[] = {
+ [0] = {
+ .type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD,
+ .conf = &mreg_action,
+ },
+ [1] = {
+ .type = RTE_FLOW_ACTION_TYPE_END,
+ },
+ };
+ const struct rte_flow_action copy_reg_mask[] = {
+ [0] = {
+ .type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD,
+ .conf = &mreg_mask,
+ },
+ [1] = {
+ .type = RTE_FLOW_ACTION_TYPE_END,
+ },
+ };
+ struct rte_flow_error drop_err;
+
+ RTE_SET_USED(drop_err);
+ return flow_hw_actions_template_create(dev, &tx_act_attr,
copy_reg_action,
+ copy_reg_mask, &drop_err);
+}
+
/**
* Creates a control flow table used to transfer traffic from E-Switch Manager
* and TX queues from group 0 to group 1.
@@ -3260,8 +3816,12 @@ flow_hw_create_ctrl_sq_miss_root_table(struct
rte_eth_dev *dev,
},
.nb_flows = MLX5_HW_CTRL_FLOW_NB_RULES,
};
+ struct mlx5_flow_template_table_cfg cfg = {
+ .attr = attr,
+ .external = false,
+ };
- return flow_hw_table_create(dev, &attr, &it, 1, &at, 1, NULL);
+ return flow_hw_table_create(dev, &cfg, &it, 1, &at, 1, NULL);
}
@@ -3286,16 +3846,56 @@ flow_hw_create_ctrl_sq_miss_table(struct rte_eth_dev
*dev,
{
struct rte_flow_template_table_attr attr = {
.flow_attr = {
- .group = MLX5_HW_SQ_MISS_GROUP,
- .priority = 0,
+ .group = 1,
+ .priority = MLX5_HW_LOWEST_PRIO_NON_ROOT,
.ingress = 0,
.egress = 0,
.transfer = 1,
},
.nb_flows = MLX5_HW_CTRL_FLOW_NB_RULES,
};
+ struct mlx5_flow_template_table_cfg cfg = {
+ .attr = attr,
+ .external = false,
+ };
+
+ return flow_hw_table_create(dev, &cfg, &it, 1, &at, 1, NULL);
+}
+
+/*
+ * Creating the default Tx metadata copy table on NIC Tx group 0.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param pt
+ * Pointer to flow pattern template.
+ * @param at
+ * Pointer to flow actions template.
+ *
+ * @return
+ * Pointer to flow table on success, NULL otherwise.
+ */
+static struct rte_flow_template_table*
+flow_hw_create_tx_default_mreg_copy_table(struct rte_eth_dev *dev,
+ struct rte_flow_pattern_template *pt,
+ struct rte_flow_actions_template *at)
+{
+ struct rte_flow_template_table_attr tx_tbl_attr = {
+ .flow_attr = {
+ .group = 0, /* Root */
+ .priority = MLX5_HW_LOWEST_PRIO_ROOT,
+ .egress = 1,
+ },
+ .nb_flows = 1, /* One default flow rule for all. */
+ };
+ struct mlx5_flow_template_table_cfg tx_tbl_cfg = {
+ .attr = tx_tbl_attr,
+ .external = false,
+ };
+ struct rte_flow_error drop_err;
- return flow_hw_table_create(dev, &attr, &it, 1, &at, 1, NULL);
+ RTE_SET_USED(drop_err);
+ return flow_hw_table_create(dev, &tx_tbl_cfg, &pt, 1, &at, 1,
&drop_err);
}
/**
@@ -3320,15 +3920,19 @@ flow_hw_create_ctrl_jump_table(struct rte_eth_dev *dev,
struct rte_flow_template_table_attr attr = {
.flow_attr = {
.group = 0,
- .priority = 15, /* TODO: Flow priority discovery. */
+ .priority = MLX5_HW_LOWEST_PRIO_ROOT,
.ingress = 0,
.egress = 0,
.transfer = 1,
},
.nb_flows = MLX5_HW_CTRL_FLOW_NB_RULES,
};
+ struct mlx5_flow_template_table_cfg cfg = {
+ .attr = attr,
+ .external = false,
+ };
- return flow_hw_table_create(dev, &attr, &it, 1, &at, 1, NULL);
+ return flow_hw_table_create(dev, &cfg, &it, 1, &at, 1, NULL);
}
/**
@@ -3346,11 +3950,14 @@ flow_hw_create_ctrl_tables(struct rte_eth_dev *dev)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct rte_flow_pattern_template *esw_mgr_items_tmpl = NULL;
- struct rte_flow_pattern_template *sq_items_tmpl = NULL;
+ struct rte_flow_pattern_template *regc_sq_items_tmpl = NULL;
struct rte_flow_pattern_template *port_items_tmpl = NULL;
- struct rte_flow_actions_template *jump_sq_actions_tmpl = NULL;
+ struct rte_flow_pattern_template *tx_meta_items_tmpl = NULL;
+ struct rte_flow_actions_template *regc_jump_actions_tmpl = NULL;
struct rte_flow_actions_template *port_actions_tmpl = NULL;
struct rte_flow_actions_template *jump_one_actions_tmpl = NULL;
+ struct rte_flow_actions_template *tx_meta_actions_tmpl = NULL;
+ uint32_t xmeta = priv->sh->config.dv_xmeta_en;
/* Item templates */
esw_mgr_items_tmpl = flow_hw_create_ctrl_esw_mgr_pattern_template(dev);
@@ -3359,8 +3966,8 @@ flow_hw_create_ctrl_tables(struct rte_eth_dev *dev)
" template for control flows", dev->data->port_id);
goto error;
}
- sq_items_tmpl = flow_hw_create_ctrl_sq_pattern_template(dev);
- if (!sq_items_tmpl) {
+ regc_sq_items_tmpl = flow_hw_create_ctrl_regc_sq_pattern_template(dev);
+ if (!regc_sq_items_tmpl) {
DRV_LOG(ERR, "port %u failed to create SQ item template for"
" control flows", dev->data->port_id);
goto error;
@@ -3371,11 +3978,18 @@ flow_hw_create_ctrl_tables(struct rte_eth_dev *dev)
" control flows", dev->data->port_id);
goto error;
}
+ if (xmeta == MLX5_XMETA_MODE_META32_HWS) {
+ tx_meta_items_tmpl =
flow_hw_create_tx_default_mreg_copy_pattern_template(dev);
+ if (!tx_meta_items_tmpl) {
+ DRV_LOG(ERR, "port %u failed to Tx metadata copy
pattern"
+ " template for control flows",
dev->data->port_id);
+ goto error;
+ }
+ }
/* Action templates */
- jump_sq_actions_tmpl = flow_hw_create_ctrl_jump_actions_template(dev,
-
MLX5_HW_SQ_MISS_GROUP);
- if (!jump_sq_actions_tmpl) {
- DRV_LOG(ERR, "port %u failed to create jump action template"
+ regc_jump_actions_tmpl =
flow_hw_create_ctrl_regc_jump_actions_template(dev);
+ if (!regc_jump_actions_tmpl) {
+ DRV_LOG(ERR, "port %u failed to create REG_C set and jump
action template"
" for control flows", dev->data->port_id);
goto error;
}
@@ -3385,23 +3999,32 @@ flow_hw_create_ctrl_tables(struct rte_eth_dev *dev)
" for control flows", dev->data->port_id);
goto error;
}
- jump_one_actions_tmpl = flow_hw_create_ctrl_jump_actions_template(dev,
1);
+ jump_one_actions_tmpl = flow_hw_create_ctrl_jump_actions_template
+ (dev, MLX5_HW_LOWEST_USABLE_GROUP);
if (!jump_one_actions_tmpl) {
DRV_LOG(ERR, "port %u failed to create jump action template"
" for control flows", dev->data->port_id);
goto error;
}
+ if (xmeta == MLX5_XMETA_MODE_META32_HWS) {
+ tx_meta_actions_tmpl =
flow_hw_create_tx_default_mreg_copy_actions_template(dev);
+ if (!tx_meta_actions_tmpl) {
+ DRV_LOG(ERR, "port %u failed to Tx metadata copy
actions"
+ " template for control flows",
dev->data->port_id);
+ goto error;
+ }
+ }
/* Tables */
MLX5_ASSERT(priv->hw_esw_sq_miss_root_tbl == NULL);
priv->hw_esw_sq_miss_root_tbl = flow_hw_create_ctrl_sq_miss_root_table
- (dev, esw_mgr_items_tmpl, jump_sq_actions_tmpl);
+ (dev, esw_mgr_items_tmpl, regc_jump_actions_tmpl);
if (!priv->hw_esw_sq_miss_root_tbl) {
DRV_LOG(ERR, "port %u failed to create table for default sq
miss (root table)"
" for control flows", dev->data->port_id);
goto error;
}
MLX5_ASSERT(priv->hw_esw_sq_miss_tbl == NULL);
- priv->hw_esw_sq_miss_tbl = flow_hw_create_ctrl_sq_miss_table(dev,
sq_items_tmpl,
+ priv->hw_esw_sq_miss_tbl = flow_hw_create_ctrl_sq_miss_table(dev,
regc_sq_items_tmpl,
port_actions_tmpl);
if (!priv->hw_esw_sq_miss_tbl) {
DRV_LOG(ERR, "port %u failed to create table for default sq
miss (non-root table)"
@@ -3416,6 +4039,16 @@ flow_hw_create_ctrl_tables(struct rte_eth_dev *dev)
" for control flows", dev->data->port_id);
goto error;
}
+ if (xmeta == MLX5_XMETA_MODE_META32_HWS) {
+ MLX5_ASSERT(priv->hw_tx_meta_cpy_tbl == NULL);
+ priv->hw_tx_meta_cpy_tbl =
flow_hw_create_tx_default_mreg_copy_table(dev,
+ tx_meta_items_tmpl,
tx_meta_actions_tmpl);
+ if (!priv->hw_tx_meta_cpy_tbl) {
+ DRV_LOG(ERR, "port %u failed to create table for
default"
+ " Tx metadata copy flow rule",
dev->data->port_id);
+ goto error;
+ }
+ }
return 0;
error:
if (priv->hw_esw_zero_tbl) {
@@ -3430,16 +4063,20 @@ flow_hw_create_ctrl_tables(struct rte_eth_dev *dev)
flow_hw_table_destroy(dev, priv->hw_esw_sq_miss_root_tbl, NULL);
priv->hw_esw_sq_miss_root_tbl = NULL;
}
+ if (xmeta == MLX5_XMETA_MODE_META32_HWS && tx_meta_actions_tmpl)
+ flow_hw_actions_template_destroy(dev, tx_meta_actions_tmpl,
NULL);
if (jump_one_actions_tmpl)
flow_hw_actions_template_destroy(dev, jump_one_actions_tmpl,
NULL);
if (port_actions_tmpl)
flow_hw_actions_template_destroy(dev, port_actions_tmpl, NULL);
- if (jump_sq_actions_tmpl)
- flow_hw_actions_template_destroy(dev, jump_sq_actions_tmpl,
NULL);
+ if (regc_jump_actions_tmpl)
+ flow_hw_actions_template_destroy(dev, regc_jump_actions_tmpl,
NULL);
+ if (xmeta == MLX5_XMETA_MODE_META32_HWS && tx_meta_items_tmpl)
+ flow_hw_pattern_template_destroy(dev, tx_meta_items_tmpl, NULL);
if (port_items_tmpl)
flow_hw_pattern_template_destroy(dev, port_items_tmpl, NULL);
- if (sq_items_tmpl)
- flow_hw_pattern_template_destroy(dev, sq_items_tmpl, NULL);
+ if (regc_sq_items_tmpl)
+ flow_hw_pattern_template_destroy(dev, regc_sq_items_tmpl, NULL);
if (esw_mgr_items_tmpl)
flow_hw_pattern_template_destroy(dev, esw_mgr_items_tmpl, NULL);
return -EINVAL;
@@ -3491,7 +4128,7 @@ flow_hw_configure(struct rte_eth_dev *dev,
struct rte_flow_queue_attr **_queue_attr = NULL;
struct rte_flow_queue_attr ctrl_queue_attr = {0};
bool is_proxy = !!(priv->sh->config.dv_esw_en && priv->master);
- int ret;
+ int ret = 0;
if (!port_attr || !nb_queue || !queue_attr) {
rte_errno = EINVAL;
@@ -3642,6 +4279,9 @@ flow_hw_configure(struct rte_eth_dev *dev,
}
if (_queue_attr)
mlx5_free(_queue_attr);
+ /* Do not overwrite the internal errno information. */
+ if (ret)
+ return ret;
return rte_flow_error_set(error, rte_errno,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
"fail to configure port");
@@ -3751,17 +4391,17 @@ void flow_hw_init_tags_set(struct rte_eth_dev *dev)
return;
unset |= 1 << (priv->mtr_color_reg - REG_C_0);
unset |= 1 << (REG_C_6 - REG_C_0);
- if (meta_mode == MLX5_XMETA_MODE_META32_HWS) {
- unset |= 1 << (REG_C_1 - REG_C_0);
+ if (priv->sh->config.dv_esw_en)
unset |= 1 << (REG_C_0 - REG_C_0);
- }
+ if (meta_mode == MLX5_XMETA_MODE_META32_HWS)
+ unset |= 1 << (REG_C_1 - REG_C_0);
masks &= ~unset;
if (mlx5_flow_hw_avl_tags_init_cnt) {
for (i = 0; i < MLX5_FLOW_HW_TAGS_MAX; i++) {
if (mlx5_flow_hw_avl_tags[i] != REG_NON && !!((1 << i)
& masks)) {
copy[mlx5_flow_hw_avl_tags[i] - REG_C_0] =
mlx5_flow_hw_avl_tags[i];
- copy_masks |= (1 << i);
+ copy_masks |= (1 << (mlx5_flow_hw_avl_tags[i] -
REG_C_0));
}
}
if (copy_masks != masks) {
@@ -3903,7 +4543,6 @@ flow_hw_action_handle_destroy(struct rte_eth_dev *dev,
uint32_t queue,
return flow_dv_action_destroy(dev, handle, error);
}
-
const struct mlx5_flow_driver_ops mlx5_flow_hw_drv_ops = {
.info_get = flow_hw_info_get,
.configure = flow_hw_configure,
@@ -3911,7 +4550,7 @@ const struct mlx5_flow_driver_ops mlx5_flow_hw_drv_ops = {
.pattern_template_destroy = flow_hw_pattern_template_destroy,
.actions_template_create = flow_hw_actions_template_create,
.actions_template_destroy = flow_hw_actions_template_destroy,
- .template_table_create = flow_hw_table_create,
+ .template_table_create = flow_hw_template_table_create,
.template_table_destroy = flow_hw_table_destroy,
.async_flow_create = flow_hw_async_flow_create,
.async_flow_destroy = flow_hw_async_flow_destroy,
@@ -3927,13 +4566,6 @@ const struct mlx5_flow_driver_ops mlx5_flow_hw_drv_ops =
{
.action_query = flow_dv_action_query,
};
-static uint32_t
-flow_hw_get_ctrl_queue(struct mlx5_priv *priv)
-{
- MLX5_ASSERT(priv->nb_queue > 0);
- return priv->nb_queue - 1;
-}
-
/**
* Creates a control flow using flow template API on @p proxy_dev device,
* on behalf of @p owner_dev device.
@@ -3971,7 +4603,7 @@ flow_hw_create_ctrl_flow(struct rte_eth_dev *owner_dev,
uint8_t action_template_idx)
{
struct mlx5_priv *priv = proxy_dev->data->dev_private;
- uint32_t queue = flow_hw_get_ctrl_queue(priv);
+ uint32_t queue = priv->nb_queue - 1;
struct rte_flow_op_attr op_attr = {
.postpone = 0,
};
@@ -4046,7 +4678,7 @@ static int
flow_hw_destroy_ctrl_flow(struct rte_eth_dev *dev, struct rte_flow *flow)
{
struct mlx5_priv *priv = dev->data->dev_private;
- uint32_t queue = flow_hw_get_ctrl_queue(priv);
+ uint32_t queue = priv->nb_queue - 1;
struct rte_flow_op_attr op_attr = {
.postpone = 0,
};
@@ -4183,10 +4815,24 @@ mlx5_flow_hw_esw_create_mgr_sq_miss_flow(struct
rte_eth_dev *dev)
.type = RTE_FLOW_ITEM_TYPE_END,
},
};
+ struct rte_flow_action_modify_field modify_field = {
+ .operation = RTE_FLOW_MODIFY_SET,
+ .dst = {
+ .field = (enum
rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
+ },
+ .src = {
+ .field = RTE_FLOW_FIELD_VALUE,
+ },
+ .width = 1,
+ };
struct rte_flow_action_jump jump = {
- .group = MLX5_HW_SQ_MISS_GROUP,
+ .group = 1,
};
struct rte_flow_action actions[] = {
+ {
+ .type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD,
+ .conf = &modify_field,
+ },
{
.type = RTE_FLOW_ACTION_TYPE_JUMP,
.conf = &jump,
@@ -4209,6 +4855,12 @@ int
mlx5_flow_hw_esw_create_sq_miss_flow(struct rte_eth_dev *dev, uint32_t txq)
{
uint16_t port_id = dev->data->port_id;
+ struct rte_flow_item_tag reg_c0_spec = {
+ .index = (uint8_t)REG_C_0,
+ };
+ struct rte_flow_item_tag reg_c0_mask = {
+ .index = 0xff,
+ };
struct mlx5_rte_flow_item_sq queue_spec = {
.queue = txq,
};
@@ -4216,6 +4868,12 @@ mlx5_flow_hw_esw_create_sq_miss_flow(struct rte_eth_dev
*dev, uint32_t txq)
.queue = UINT32_MAX,
};
struct rte_flow_item items[] = {
+ {
+ .type = (enum rte_flow_item_type)
+ MLX5_RTE_FLOW_ITEM_TYPE_TAG,
+ .spec = ®_c0_spec,
+ .mask = ®_c0_mask,
+ },
{
.type = (enum rte_flow_item_type)
MLX5_RTE_FLOW_ITEM_TYPE_SQ,
@@ -4241,6 +4899,7 @@ mlx5_flow_hw_esw_create_sq_miss_flow(struct rte_eth_dev
*dev, uint32_t txq)
struct rte_eth_dev *proxy_dev;
struct mlx5_priv *proxy_priv;
uint16_t proxy_port_id = dev->data->port_id;
+ uint32_t marker_bit;
int ret;
RTE_SET_USED(txq);
@@ -4261,6 +4920,14 @@ mlx5_flow_hw_esw_create_sq_miss_flow(struct rte_eth_dev
*dev, uint32_t txq)
rte_errno = ENOMEM;
return -rte_errno;
}
+ marker_bit = flow_hw_usable_lsb_vport_mask(proxy_priv);
+ if (!marker_bit) {
+ DRV_LOG(ERR, "Unable to set up control flow in SQ miss table");
+ rte_errno = EINVAL;
+ return -rte_errno;
+ }
+ reg_c0_spec.data = marker_bit;
+ reg_c0_mask.data = marker_bit;
return flow_hw_create_ctrl_flow(dev, proxy_dev,
proxy_priv->hw_esw_sq_miss_tbl,
items, 0, actions, 0);
@@ -4320,4 +4987,53 @@ mlx5_flow_hw_esw_create_default_jump_flow(struct
rte_eth_dev *dev)
items, 0, actions, 0);
}
+int
+mlx5_flow_hw_create_tx_default_mreg_copy_flow(struct rte_eth_dev *dev)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+ struct rte_flow_item_eth promisc = {
+ .dst.addr_bytes = "\x00\x00\x00\x00\x00\x00",
+ .src.addr_bytes = "\x00\x00\x00\x00\x00\x00",
+ .type = 0,
+ };
+ struct rte_flow_item eth_all[] = {
+ [0] = {
+ .type = RTE_FLOW_ITEM_TYPE_ETH,
+ .spec = &promisc,
+ .mask = &promisc,
+ },
+ [1] = {
+ .type = RTE_FLOW_ITEM_TYPE_END,
+ },
+ };
+ struct rte_flow_action_modify_field mreg_action = {
+ .operation = RTE_FLOW_MODIFY_SET,
+ .dst = {
+ .field = (enum
rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
+ .level = REG_C_1,
+ },
+ .src = {
+ .field = (enum
rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
+ .level = REG_A,
+ },
+ .width = 32,
+ };
+ struct rte_flow_action copy_reg_action[] = {
+ [0] = {
+ .type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD,
+ .conf = &mreg_action,
+ },
+ [1] = {
+ .type = RTE_FLOW_ACTION_TYPE_END,
+ },
+ };
+
+ MLX5_ASSERT(priv->master);
+ if (!priv->dr_ctx || !priv->hw_tx_meta_cpy_tbl)
+ return 0;
+ return flow_hw_create_ctrl_flow(dev, dev,
+ priv->hw_tx_meta_cpy_tbl,
+ eth_all, 0, copy_reg_action, 0);
+}
+
#endif
diff --git a/drivers/net/mlx5/mlx5_trigger.c b/drivers/net/mlx5/mlx5_trigger.c
index f59d314ff4..cccec08d70 100644
--- a/drivers/net/mlx5/mlx5_trigger.c
+++ b/drivers/net/mlx5/mlx5_trigger.c
@@ -1292,6 +1292,9 @@ mlx5_traffic_enable_hws(struct rte_eth_dev *dev)
if (priv->sh->config.dv_esw_en && priv->master) {
if (mlx5_flow_hw_esw_create_mgr_sq_miss_flow(dev))
goto error;
+ if (priv->sh->config.dv_xmeta_en == MLX5_XMETA_MODE_META32_HWS)
+ if (mlx5_flow_hw_create_tx_default_mreg_copy_flow(dev))
+ goto error;
}
for (i = 0; i < priv->txqs_n; ++i) {
struct mlx5_txq_ctrl *txq = mlx5_txq_get(dev, i);
--
2.25.1
