[v3 00/18] net/mlx5: Add HW steering low level support
Mellanox ConnetX devices supports packet matching, packet modification and
redirection. These functionalities are also referred to as flow-steering.
To configure a steering rule, the rule is written to the device owned
memory, this memory is accessed and cached by the device when processing
a packet.
The highlight of this patchset is supporting HW Steering (HWS) which
is the new technology supported in new ConnectX devices, HWS allows
configuring steering rules directly to the HW using special HW queues
with minimal CPU effort.
This patchset is the internal low layer implementation for HWS used by
the mlx5 PMD. The mlx5dr (direct rule) is layer that bridges between the
PMD and the HW by configuring the HW offloads based on the PMD logic
v2:
Fix check patch and cosmetic changes
v3:
-Fix unsupported items
-Fix compilation with mlx5dv dependency
Alex Vesker (9):
net/mlx5: Add additional glue functions for HWS
net/mlx5/hws: Add HWS send layer
net/mlx5/hws: Add HWS definer layer
net/mlx5/hws: Add HWS context object
net/mlx5/hws: Add HWS table object
net/mlx5/hws: Add HWS matcher object
net/mlx5/hws: Add HWS rule object
net/mlx5/hws: Add HWS action object
net/mlx5/hws: Enable HWS
Bing Zhao (2):
common/mlx5: query set capability of registers
net/mlx5: provide the available tag registers
Dariusz Sosnowski (1):
net/mlx5: add port to metadata conversion
Erez Shitrit (2):
net/mlx5/hws: Add HWS command layer
net/mlx5/hws: Add HWS pool and buddy
Hamdan Igbaria (1):
net/mlx5/hws: Add HWS debug layer
Suanming Mou (3):
net/mlx5: split flow item translation
net/mlx5: split flow item matcher and value translation
net/mlx5: add hardware steering item translation function
drivers/common/mlx5/linux/meson.build|2 +
drivers/common/mlx5/linux/mlx5_glue.c| 121 +-
drivers/common/mlx5/linux/mlx5_glue.h| 17 +
drivers/common/mlx5/mlx5_devx_cmds.c | 30 +
drivers/common/mlx5/mlx5_devx_cmds.h |2 +
drivers/common/mlx5/mlx5_prm.h | 652 -
drivers/net/mlx5/hws/meson.build | 18 +
drivers/net/mlx5/{mlx5_dr.h => hws/mlx5dr.h} | 210 +-
drivers/net/mlx5/hws/mlx5dr_action.c | 2221 +++
drivers/net/mlx5/hws/mlx5dr_action.h | 253 ++
drivers/net/mlx5/hws/mlx5dr_buddy.c | 201 ++
drivers/net/mlx5/hws/mlx5dr_buddy.h | 22 +
drivers/net/mlx5/hws/mlx5dr_cmd.c| 948 +++
drivers/net/mlx5/hws/mlx5dr_cmd.h| 230 ++
drivers/net/mlx5/hws/mlx5dr_context.c| 223 ++
drivers/net/mlx5/hws/mlx5dr_context.h| 40 +
drivers/net/mlx5/hws/mlx5dr_debug.c | 462 +++
drivers/net/mlx5/hws/mlx5dr_debug.h | 28 +
drivers/net/mlx5/hws/mlx5dr_definer.c| 1968 +
drivers/net/mlx5/hws/mlx5dr_definer.h| 585
drivers/net/mlx5/hws/mlx5dr_internal.h | 93 +
drivers/net/mlx5/hws/mlx5dr_matcher.c| 922 ++
drivers/net/mlx5/hws/mlx5dr_matcher.h| 76 +
drivers/net/mlx5/hws/mlx5dr_pat_arg.c| 511
drivers/net/mlx5/hws/mlx5dr_pat_arg.h| 83 +
drivers/net/mlx5/hws/mlx5dr_pool.c | 672 +
drivers/net/mlx5/hws/mlx5dr_pool.h | 152 +
drivers/net/mlx5/hws/mlx5dr_rule.c | 528
drivers/net/mlx5/hws/mlx5dr_rule.h | 50 +
drivers/net/mlx5/hws/mlx5dr_send.c | 844 ++
drivers/net/mlx5/hws/mlx5dr_send.h | 275 ++
drivers/net/mlx5/hws/mlx5dr_table.c | 248 ++
drivers/net/mlx5/hws/mlx5dr_table.h | 44 +
drivers/net/mlx5/linux/mlx5_os.c | 12 +-
drivers/net/mlx5/meson.build |5 +-
drivers/net/mlx5/mlx5.c |3 +
drivers/net/mlx5/mlx5.h |3 +-
drivers/net/mlx5/mlx5_defs.h |2 +
drivers/net/mlx5/mlx5_dr.c | 383 ---
drivers/net/mlx5/mlx5_flow.c | 27 +-
drivers/net/mlx5/mlx5_flow.h | 174 +-
drivers/net/mlx5/mlx5_flow_dv.c | 2631 +-
drivers/net/mlx5/mlx5_flow_hw.c | 115 +-
43 files changed, 14365 insertions(+), 1721 deletions(-)
create mode 100644 drivers/net/mlx5/hws/meson.build
rename drivers/net/mlx5/{mlx5_dr.h => hws/mlx5dr.h} (66%)
create mode 100644 drivers/net/mlx5/hws/mlx5dr_action.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_action.h
create mode 100644 drivers/net/mlx5/hws/mlx5dr_buddy.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_buddy.h
create mode 100644 drivers/net/mlx5/hws/mlx5dr_cmd.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_cmd.h
create mode 100644 drivers/net/mlx5/hws/mlx5dr_context.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_context.h
create mode 100644 drivers/net/mlx5/hws/mlx5dr_debug.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_debug.h
create mode 100644 drivers/net/mlx5/hws/mlx5dr_definer.c
create mode
[v3 01/18] net/mlx5: split flow item translation
From: Suanming Mou
In order to share the item translation code with hardware steering
mode, this commit splits flow item translation code to a dedicate
function.
Signed-off-by: Suanming Mou
---
drivers/net/mlx5/mlx5_flow_dv.c | 1915 ---
1 file changed, 979 insertions(+), 936 deletions(-)
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 91f287af5c..70a3279e2f 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -13029,8 +13029,7 @@ flow_dv_translate_create_conntrack(struct rte_eth_dev
*dev,
}
/**
- * Fill the flow with DV spec, lock free
- * (mutex should be acquired by caller).
+ * Translate the flow item to matcher.
*
* @param[in] dev
* Pointer to rte_eth_dev structure.
@@ -13040,8 +13039,8 @@ flow_dv_translate_create_conntrack(struct rte_eth_dev
*dev,
* Pointer to the flow attributes.
* @param[in] items
* Pointer to the list of items.
- * @param[in] actions
- * Pointer to the list of actions.
+ * @param[in] matcher
+ * Pointer to the flow matcher.
* @param[out] error
* Pointer to the error structure.
*
@@ -13049,1041 +13048,1086 @@ flow_dv_translate_create_conntrack(struct
rte_eth_dev *dev,
* 0 on success, a negative errno value otherwise and rte_errno is set.
*/
static int
-flow_dv_translate(struct rte_eth_dev *dev,
- struct mlx5_flow *dev_flow,
- const struct rte_flow_attr *attr,
- const struct rte_flow_item items[],
- const struct rte_flow_action actions[],
- struct rte_flow_error *error)
+flow_dv_translate_items(struct rte_eth_dev *dev,
+ struct mlx5_flow *dev_flow,
+ const struct rte_flow_attr *attr,
+ const struct rte_flow_item items[],
+ struct mlx5_flow_dv_matcher *matcher,
+ struct rte_flow_error *error)
{
struct mlx5_priv *priv = dev->data->dev_private;
- struct mlx5_sh_config *dev_conf = &priv->sh->config;
struct rte_flow *flow = dev_flow->flow;
struct mlx5_flow_handle *handle = dev_flow->handle;
struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
- struct mlx5_flow_rss_desc *rss_desc;
+ struct mlx5_flow_rss_desc *rss_desc = &wks->rss_desc;
uint64_t item_flags = 0;
uint64_t last_item = 0;
- uint64_t action_flags = 0;
- struct mlx5_flow_dv_matcher matcher = {
- .mask = {
- .size = sizeof(matcher.mask.buf),
- },
- };
- int actions_n = 0;
- bool actions_end = false;
- union {
- struct mlx5_flow_dv_modify_hdr_resource res;
- uint8_t len[sizeof(struct mlx5_flow_dv_modify_hdr_resource) +
- sizeof(struct mlx5_modification_cmd) *
- (MLX5_MAX_MODIFY_NUM + 1)];
- } mhdr_dummy;
- struct mlx5_flow_dv_modify_hdr_resource *mhdr_res = &mhdr_dummy.res;
- const struct rte_flow_action_count *count = NULL;
- const struct rte_flow_action_age *non_shared_age = NULL;
- union flow_dv_attr flow_attr = { .attr = 0 };
- uint32_t tag_be;
- union mlx5_flow_tbl_key tbl_key;
- uint32_t modify_action_position = UINT32_MAX;
- void *match_mask = matcher.mask.buf;
+ void *match_mask = matcher->mask.buf;
void *match_value = dev_flow->dv.value.buf;
uint8_t next_protocol = 0xff;
- struct rte_vlan_hdr vlan = { 0 };
- struct mlx5_flow_dv_dest_array_resource mdest_res;
- struct mlx5_flow_dv_sample_resource sample_res;
- void *sample_actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS] = {0};
- const struct rte_flow_action_sample *sample = NULL;
- struct mlx5_flow_sub_actions_list *sample_act;
- uint32_t sample_act_pos = UINT32_MAX;
- uint32_t age_act_pos = UINT32_MAX;
- uint32_t num_of_dest = 0;
- int tmp_actions_n = 0;
- uint32_t table;
- int ret = 0;
- const struct mlx5_flow_tunnel *tunnel = NULL;
- struct flow_grp_info grp_info = {
- .external = !!dev_flow->external,
- .transfer = !!attr->transfer,
- .fdb_def_rule = !!priv->fdb_def_rule,
- .skip_scale = dev_flow->skip_scale &
- (1 << MLX5_SCALE_FLOW_GROUP_BIT),
- .std_tbl_fix = true,
- };
+ uint16_t priority = 0;
const struct rte_flow_item *integrity_items[2] = {NULL, NULL};
const struct rte_flow_item *tunnel_item = NULL;
const struct rte_flow_item *gre_item = NULL;
+ int ret = 0;
- if (!wks)
- return rte_flow_error_set(error, ENOMEM,
- RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
- NULL,
- "failed to
[v3 04/18] net/mlx5: add port to metadata conversion
From: Dariusz Sosnowski
This patch initial version of functions used to:
- convert between ethdev port_id and internal tag/mask value,
- convert between IB context and internal tag/mask value.
Signed-off-by: Dariusz Sosnowski
---
drivers/net/mlx5/linux/mlx5_os.c | 10 +-
drivers/net/mlx5/mlx5.c | 1 +
drivers/net/mlx5/mlx5_flow.c | 6
drivers/net/mlx5/mlx5_flow.h | 52
drivers/net/mlx5/mlx5_flow_hw.c | 29 ++
5 files changed, 97 insertions(+), 1 deletion(-)
diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index 60677eb8d7..98c6374547 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -1541,8 +1541,16 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
if (!priv->hrxqs)
goto error;
rte_rwlock_init(&priv->ind_tbls_lock);
- if (priv->sh->config.dv_flow_en == 2)
+ if (priv->sh->config.dv_flow_en == 2) {
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+ if (priv->vport_meta_mask)
+ flow_hw_set_port_info(eth_dev);
return eth_dev;
+#else
+ DRV_LOG(ERR, "DV support is missing for HWS.");
+ goto error;
+#endif
+ }
/* Port representor shares the same max priority with pf port. */
if (!priv->sh->flow_priority_check_flag) {
/* Supported Verbs flow priority number detection. */
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 752b60d769..1d10932619 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -1944,6 +1944,7 @@ mlx5_dev_close(struct rte_eth_dev *dev)
mlx5_flex_item_port_cleanup(dev);
#if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
flow_hw_resource_release(dev);
+ flow_hw_clear_port_info(dev);
#endif
if (priv->rxq_privs != NULL) {
/* XXX race condition if mlx5_rx_burst() is still running. */
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 81bed6f6a3..bdb0613d4a 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -33,6 +33,12 @@
#include "mlx5_common_os.h"
#include "rte_pmd_mlx5.h"
+/*
+ * Shared array for quick translation between port_id and vport mask/values
+ * used for HWS rules.
+ */
+struct flow_hw_port_info mlx5_flow_hw_port_infos[RTE_MAX_ETHPORTS];
+
struct tunnel_default_miss_ctx {
uint16_t *queue;
__extension__
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index 288e09d5ba..17102623c1 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -1323,6 +1323,58 @@ struct mlx5_flow_split_info {
uint64_t prefix_layers; /**< Prefix subflow layers. */
};
+struct flow_hw_port_info {
+ uint32_t regc_mask;
+ uint32_t regc_value;
+ uint32_t is_wire:1;
+};
+
+extern struct flow_hw_port_info mlx5_flow_hw_port_infos[RTE_MAX_ETHPORTS];
+
+/*
+ * Get metadata match tag and mask for given rte_eth_dev port.
+ * Used in HWS rule creation.
+ */
+static __rte_always_inline const struct flow_hw_port_info *
+flow_hw_conv_port_id(const uint16_t port_id)
+{
+ struct flow_hw_port_info *port_info;
+
+ if (port_id >= RTE_MAX_ETHPORTS)
+ return NULL;
+ port_info = &mlx5_flow_hw_port_infos[port_id];
+ return !!port_info->regc_mask ? port_info : NULL;
+}
+
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+/*
+ * Get metadata match tag and mask for the uplink port represented
+ * by given IB context. Used in HWS context creation.
+ */
+static __rte_always_inline const struct flow_hw_port_info *
+flow_hw_get_wire_port(struct ibv_context *ibctx)
+{
+ struct ibv_device *ibdev = ibctx->device;
+ uint16_t port_id;
+
+ MLX5_ETH_FOREACH_DEV(port_id, NULL) {
+ const struct mlx5_priv *priv =
+ rte_eth_devices[port_id].data->dev_private;
+
+ if (priv && priv->master) {
+ struct ibv_context *port_ibctx = priv->sh->cdev->ctx;
+
+ if (port_ibctx->device == ibdev)
+ return flow_hw_conv_port_id(port_id);
+ }
+ }
+ return NULL;
+}
+#endif
+
+void flow_hw_set_port_info(struct rte_eth_dev *dev);
+void flow_hw_clear_port_info(struct rte_eth_dev *dev);
+
typedef int (*mlx5_flow_validate_t)(struct rte_eth_dev *dev,
const struct rte_flow_attr *attr,
const struct rte_flow_item items[],
diff --git a/drivers/net/mlx5/mlx5_flow_hw.c b/drivers/net/mlx5/mlx5_flow_hw.c
index 12498794a5..fe809a83b9 100644
--- a/drivers/net/mlx5/mlx5_flow_hw.c
+++ b/drivers/net/mlx5/mlx5_flow_hw.c
@@ -2208,6 +2208,35 @@ flow_hw_resource_release(struct rte_eth_dev *dev)
priv->nb_queue = 0;
}
+/* Sets vport tag and mask, for given port, used in HWS ru
[v3 02/18] net/mlx5: split flow item matcher and value translation
From: Suanming Mou
As hardware steering mode translates flow matcher and value in two
different stages, split the flow item matcher and value translation
to help reuse the code.
Signed-off-by: Suanming Mou
---
drivers/net/mlx5/mlx5_flow.h| 32 +
drivers/net/mlx5/mlx5_flow_dv.c | 2314 +++
2 files changed, 1185 insertions(+), 1161 deletions(-)
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index 0fa1735b1a..2ebb8496f2 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -1264,6 +1264,38 @@ struct mlx5_flow_workspace {
uint32_t skip_matcher_reg:1;
/* Indicates if need to skip matcher register in translate. */
uint32_t mark:1; /* Indicates if flow contains mark action. */
+ uint32_t vport_meta_tag; /* Used for vport index match. */
+};
+
+/* Matcher translate type. */
+enum MLX5_SET_MATCHER {
+ MLX5_SET_MATCHER_SW_V = 1 << 0,
+ MLX5_SET_MATCHER_SW_M = 1 << 1,
+ MLX5_SET_MATCHER_HS_V = 1 << 2,
+ MLX5_SET_MATCHER_HS_M = 1 << 3,
+};
+
+#define MLX5_SET_MATCHER_SW (MLX5_SET_MATCHER_SW_V | MLX5_SET_MATCHER_SW_M)
+#define MLX5_SET_MATCHER_HS (MLX5_SET_MATCHER_HS_V | MLX5_SET_MATCHER_HS_M)
+#define MLX5_SET_MATCHER_V (MLX5_SET_MATCHER_SW_V | MLX5_SET_MATCHER_HS_V)
+#define MLX5_SET_MATCHER_M (MLX5_SET_MATCHER_SW_M | MLX5_SET_MATCHER_HS_M)
+
+/* Flow matcher workspace intermediate data. */
+struct mlx5_dv_matcher_workspace {
+ uint8_t priority; /* Flow priority. */
+ uint64_t last_item; /* Last item in pattern. */
+ uint64_t item_flags; /* Flow item pattern flags. */
+ uint64_t action_flags; /* Flow action flags. */
+ bool external; /* External flow or not. */
+ uint32_t vlan_tag:12; /* Flow item VLAN tag. */
+ uint8_t next_protocol; /* Tunnel next protocol */
+ uint32_t geneve_tlv_option; /* Flow item Geneve TLV option. */
+ uint32_t group; /* Flow group. */
+ uint16_t udp_dport; /* Flow item UDP port. */
+ const struct rte_flow_attr *attr; /* Flow attribute. */
+ struct mlx5_flow_rss_desc *rss_desc; /* RSS descriptor. */
+ const struct rte_flow_item *tunnel_item; /* Flow tunnel item. */
+ const struct rte_flow_item *gre_item; /* Flow GRE item. */
};
struct mlx5_flow_split_info {
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 70a3279e2f..0589cafc30 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -63,6 +63,25 @@
#define MLX5DV_FLOW_VLAN_PCP_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK)
#define MLX5DV_FLOW_VLAN_VID_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_VID_MASK)
+#define MLX5_ITEM_VALID(item, key_type) \
+ (((MLX5_SET_MATCHER_SW & (key_type)) && !((item)->spec)) || \
+((MLX5_SET_MATCHER_HS_V == (key_type)) && !((item)->spec)) || \
+((MLX5_SET_MATCHER_HS_M == (key_type)) && !((item)->mask)))
+
+#define MLX5_ITEM_UPDATE(item, key_type, v, m, gm) \
+ do { \
+ if ((key_type) == MLX5_SET_MATCHER_SW_V) { \
+ v = (item)->spec; \
+ m = (item)->mask ? (item)->mask : (gm); \
+ } else if ((key_type) == MLX5_SET_MATCHER_HS_V) { \
+ v = (item)->spec; \
+ m = (v); \
+ } else { \
+ v = (item)->mask ? (item)->mask : (gm); \
+ m = (v); \
+ } \
+ } while (0)
+
union flow_dv_attr {
struct {
uint32_t valid:1;
@@ -8323,70 +8342,61 @@ flow_dv_check_valid_spec(void *match_mask, void
*match_value)
static inline void
flow_dv_set_match_ip_version(uint32_t group,
void *headers_v,
-void *headers_m,
+uint32_t key_type,
uint8_t ip_version)
{
- if (group == 0)
- MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
+ if (group == 0 && (key_type & MLX5_SET_MATCHER_M))
+ MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, 0xf);
else
- MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version,
+ MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version,
ip_version);
- MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, ip_version);
MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype, 0);
- MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype, 0);
}
/**
- * Add Ethernet item to matcher and to the value.
+ * Add Ethernet item to the value.
*
- * @param[in, out] matcher
- * Flow matcher.
* @param[in, out] key
* Flow matcher value.
* @param[in] item
* Flow pattern to translate.
* @param[in] inner
* Item is inner pattern.
+ * @param[in] grpup
+ * Flow matcher group.
+ * @param[in] key_type
+ * Set flow matcher mask or value.
*/
static voi
[v3 05/18] common/mlx5: query set capability of registers
From: Bing Zhao
In the flow table capabilities, new fields are added to query the
capability to set, add, copy to a REG_C_x.
The set capability are queried and saved for the future usage.
Signed-off-by: Bing Zhao
---
drivers/common/mlx5/mlx5_devx_cmds.c | 30 +++
drivers/common/mlx5/mlx5_devx_cmds.h | 2 ++
drivers/common/mlx5/mlx5_prm.h | 45 +---
3 files changed, 73 insertions(+), 4 deletions(-)
diff --git a/drivers/common/mlx5/mlx5_devx_cmds.c
b/drivers/common/mlx5/mlx5_devx_cmds.c
index 76f0b6724f..9c185366d0 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.c
+++ b/drivers/common/mlx5/mlx5_devx_cmds.c
@@ -1064,6 +1064,24 @@ mlx5_devx_cmd_query_hca_attr(void *ctx,
attr->modify_outer_ip_ecn = MLX5_GET
(flow_table_nic_cap, hcattr,
ft_header_modify_nic_receive.outer_ip_ecn);
+ attr->set_reg_c = 0xff;
+ if (attr->nic_flow_table) {
+#define GET_RX_REG_X_BITS \
+ MLX5_GET(flow_table_nic_cap, hcattr, \
+ft_header_modify_nic_receive.metadata_reg_c_x)
+#define GET_TX_REG_X_BITS \
+ MLX5_GET(flow_table_nic_cap, hcattr, \
+ft_header_modify_nic_transmit.metadata_reg_c_x)
+
+ uint32_t tx_reg, rx_reg;
+
+ tx_reg = GET_TX_REG_X_BITS;
+ rx_reg = GET_RX_REG_X_BITS;
+ attr->set_reg_c &= (rx_reg & tx_reg);
+
+#undef GET_RX_REG_X_BITS
+#undef GET_TX_REG_X_BITS
+ }
attr->pkt_integrity_match = mlx5_devx_query_pkt_integrity_match(hcattr);
attr->inner_ipv4_ihl = MLX5_GET
(flow_table_nic_cap, hcattr,
@@ -1163,6 +1181,18 @@ mlx5_devx_cmd_query_hca_attr(void *ctx,
attr->esw_mgr_vport_id =
MLX5_GET(esw_cap, hcattr, esw_manager_vport_number);
}
+ if (attr->eswitch_manager) {
+ uint32_t esw_reg;
+
+ hcattr = mlx5_devx_get_hca_cap(ctx, in, out, &rc,
+ MLX5_GET_HCA_CAP_OP_MOD_ESW_FLOW_TABLE |
+ MLX5_HCA_CAP_OPMOD_GET_CUR);
+ if (!hcattr)
+ return rc;
+ esw_reg = MLX5_GET(flow_table_esw_cap, hcattr,
+ ft_header_modify_esw_fdb.metadata_reg_c_x);
+ attr->set_reg_c &= esw_reg;
+ }
return 0;
error:
rc = (rc > 0) ? -rc : rc;
diff --git a/drivers/common/mlx5/mlx5_devx_cmds.h
b/drivers/common/mlx5/mlx5_devx_cmds.h
index cceaf3411d..a10aa3331b 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.h
+++ b/drivers/common/mlx5/mlx5_devx_cmds.h
@@ -263,6 +263,8 @@ struct mlx5_hca_attr {
uint32_t crypto_wrapped_import_method:1;
uint16_t esw_mgr_vport_id; /* E-Switch Mgr vport ID . */
uint16_t max_wqe_sz_sq;
+ uint32_t set_reg_c:8;
+ uint32_t nic_flow_table:1;
uint32_t modify_outer_ip_ecn:1;
};
diff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h
index 9c1c93f916..ca4763f53d 100644
--- a/drivers/common/mlx5/mlx5_prm.h
+++ b/drivers/common/mlx5/mlx5_prm.h
@@ -1295,6 +1295,7 @@ enum {
MLX5_GET_HCA_CAP_OP_MOD_QOS_CAP = 0xc << 1,
MLX5_GET_HCA_CAP_OP_MOD_ROCE = 0x4 << 1,
MLX5_GET_HCA_CAP_OP_MOD_NIC_FLOW_TABLE = 0x7 << 1,
+ MLX5_GET_HCA_CAP_OP_MOD_ESW_FLOW_TABLE = 0x8 << 1,
MLX5_SET_HCA_CAP_OP_MOD_ESW = 0x9 << 1,
MLX5_GET_HCA_CAP_OP_MOD_VDPA_EMULATION = 0x13 << 1,
MLX5_GET_HCA_CAP_OP_MOD_CRYPTO = 0x1A << 1,
@@ -1892,6 +1893,7 @@ struct mlx5_ifc_roce_caps_bits {
};
struct mlx5_ifc_ft_fields_support_bits {
+ /* set_action_field_support */
u8 outer_dmac[0x1];
u8 outer_smac[0x1];
u8 outer_ether_type[0x1];
@@ -1919,7 +1921,7 @@ struct mlx5_ifc_ft_fields_support_bits {
u8 outer_gre_key[0x1];
u8 outer_vxlan_vni[0x1];
u8 reserved_at_1a[0x5];
- u8 source_eswitch_port[0x1];
+ u8 source_eswitch_port[0x1]; /* end of DW0 */
u8 inner_dmac[0x1];
u8 inner_smac[0x1];
u8 inner_ether_type[0x1];
@@ -1943,8 +1945,33 @@ struct mlx5_ifc_ft_fields_support_bits {
u8 inner_tcp_sport[0x1];
u8 inner_tcp_dport[0x1];
u8 inner_tcp_flags[0x1];
- u8 reserved_at_37[0x9];
- u8 reserved_at_40[0x40];
+ u8 reserved_at_37[0x9]; /* end of DW1 */
+ u8 reserved_at_40[0x20]; /* end of DW2 */
+ u8 reserved_at_60[0x18];
+ union {
+ struct {
+ u8 metadata_reg_c_7[0x1];
+ u8 metadata_reg_c_6[0x1];
+ u8 metadata_reg_c_5[0x1];
+ u8 metadata_reg_c_4[0x1];
+ u8 metadata_reg_c_3[0x1];
+ u8 metadata_reg_c_2[0x1];
+ u8 metadata_reg_c_1[0x1];
+ u8 metadata_reg_c_0[0x1];
+ };
+ u8 metadata_reg_c_x[0
[v3 03/18] net/mlx5: add hardware steering item translation function
From: Suanming Mou
As hardware steering root table flows still work under FW steering
mode. This commit provides shared item tranlsation code for hardware
steering root table flows.
Signed-off-by: Suanming Mou
---
drivers/net/mlx5/mlx5_flow.c| 10 +--
drivers/net/mlx5/mlx5_flow.h| 52 -
drivers/net/mlx5/mlx5_flow_dv.c | 134
3 files changed, 155 insertions(+), 41 deletions(-)
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index e4744b0a67..81bed6f6a3 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -7108,7 +7108,7 @@ mlx5_flow_create_devx_sq_miss_flow(struct rte_eth_dev
*dev, uint32_t txq)
struct rte_flow_item_port_id port_spec = {
.id = MLX5_PORT_ESW_MGR,
};
- struct mlx5_rte_flow_item_tx_queue txq_spec = {
+ struct mlx5_rte_flow_item_sq txq_spec = {
.queue = txq,
};
struct rte_flow_item pattern[] = {
@@ -7118,7 +7118,7 @@ mlx5_flow_create_devx_sq_miss_flow(struct rte_eth_dev
*dev, uint32_t txq)
},
{
.type = (enum rte_flow_item_type)
- MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE,
+ MLX5_RTE_FLOW_ITEM_TYPE_SQ,
.spec = &txq_spec,
},
{
@@ -7504,16 +7504,16 @@ mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev,
.egress = 1,
.priority = 0,
};
- struct mlx5_rte_flow_item_tx_queue queue_spec = {
+ struct mlx5_rte_flow_item_sq queue_spec = {
.queue = queue,
};
- struct mlx5_rte_flow_item_tx_queue queue_mask = {
+ struct mlx5_rte_flow_item_sq queue_mask = {
.queue = UINT32_MAX,
};
struct rte_flow_item items[] = {
{
.type = (enum rte_flow_item_type)
- MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE,
+ MLX5_RTE_FLOW_ITEM_TYPE_SQ,
.spec = &queue_spec,
.last = NULL,
.mask = &queue_mask,
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index 2ebb8496f2..288e09d5ba 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -28,7 +28,7 @@
enum mlx5_rte_flow_item_type {
MLX5_RTE_FLOW_ITEM_TYPE_END = INT_MIN,
MLX5_RTE_FLOW_ITEM_TYPE_TAG,
- MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE,
+ MLX5_RTE_FLOW_ITEM_TYPE_SQ,
MLX5_RTE_FLOW_ITEM_TYPE_VLAN,
MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL,
};
@@ -95,7 +95,7 @@ struct mlx5_flow_action_copy_mreg {
};
/* Matches on source queue. */
-struct mlx5_rte_flow_item_tx_queue {
+struct mlx5_rte_flow_item_sq {
uint32_t queue;
};
@@ -159,7 +159,7 @@ enum mlx5_feature_name {
#define MLX5_FLOW_LAYER_GENEVE (1u << 26)
/* Queue items. */
-#define MLX5_FLOW_ITEM_TX_QUEUE (1u << 27)
+#define MLX5_FLOW_ITEM_SQ (1u << 27)
/* Pattern tunnel Layer bits (continued). */
#define MLX5_FLOW_LAYER_GTP (1u << 28)
@@ -196,6 +196,9 @@ enum mlx5_feature_name {
#define MLX5_FLOW_ITEM_PORT_REPRESENTOR (UINT64_C(1) << 41)
#define MLX5_FLOW_ITEM_REPRESENTED_PORT (UINT64_C(1) << 42)
+/* Meter color item */
+#define MLX5_FLOW_ITEM_METER_COLOR (UINT64_C(1) << 44)
+
/* Outer Masks. */
#define MLX5_FLOW_LAYER_OUTER_L3 \
(MLX5_FLOW_LAYER_OUTER_L3_IPV4 | MLX5_FLOW_LAYER_OUTER_L3_IPV6)
@@ -1006,6 +1009,18 @@ flow_items_to_tunnel(const struct rte_flow_item items[])
return items[0].spec;
}
+/* HW steering flow attributes. */
+struct mlx5_flow_attr {
+ uint32_t port_id; /* Port index. */
+ uint32_t group; /* Flow group. */
+ uint32_t priority; /* Original Priority. */
+ /* rss level, used by priority adjustment. */
+ uint32_t rss_level;
+ /* Action flags, used by priority adjustment. */
+ uint32_t act_flags;
+ uint32_t tbl_type; /* Flow table type. */
+};
+
/* Flow structure. */
struct rte_flow {
uint32_t dev_handles;
@@ -1766,6 +1781,32 @@ mlx5_translate_tunnel_etypes(uint64_t pattern_flags)
int flow_hw_q_flow_flush(struct rte_eth_dev *dev,
struct rte_flow_error *error);
+
+/*
+ * Convert rte_mtr_color to mlx5 color.
+ *
+ * @param[in] rcol
+ * rte_mtr_color.
+ *
+ * @return
+ * mlx5 color.
+ */
+static inline int
+rte_col_2_mlx5_col(enum rte_color rcol)
+{
+ switch (rcol) {
+ case RTE_COLOR_GREEN:
+ return MLX5_FLOW_COLOR_GREEN;
+ case RTE_COLOR_YELLOW:
+ return MLX5_FLOW_COLOR_YELLOW;
+ case RTE_COLOR_RED:
+ return MLX5_FLOW_COLOR_RED;
+ default:
+ break;
+ }
+ return MLX5_FLOW_COLOR_UNDEFINED;
+}
+
int mlx5_flow_group_to_table(struct rte_eth_dev *dev,
const struct mlx5_f
[v3 07/18] net/mlx5: Add additional glue functions for HWS
Add missing glue support for HWS mlx5dr layer. The new glue
functions are needed for mlx5dv create matcher and action,
which are used as the kernel root table as well as for
capabilities query like device name and ports info.
Signed-off-by: Alex Vesker
---
drivers/common/mlx5/linux/mlx5_glue.c | 121 --
drivers/common/mlx5/linux/mlx5_glue.h | 17
2 files changed, 131 insertions(+), 7 deletions(-)
diff --git a/drivers/common/mlx5/linux/mlx5_glue.c
b/drivers/common/mlx5/linux/mlx5_glue.c
index 450dd6a06a..943d4bf833 100644
--- a/drivers/common/mlx5/linux/mlx5_glue.c
+++ b/drivers/common/mlx5/linux/mlx5_glue.c
@@ -111,6 +111,12 @@ mlx5_glue_query_device_ex(struct ibv_context *context,
return ibv_query_device_ex(context, input, attr);
}
+static const char *
+mlx5_glue_get_device_name(struct ibv_device *device)
+{
+return ibv_get_device_name(device);
+}
+
static int
mlx5_glue_query_rt_values_ex(struct ibv_context *context,
struct ibv_values_ex *values)
@@ -620,6 +626,20 @@ mlx5_glue_dv_create_qp(struct ibv_context *context,
#endif
}
+static void *
+__mlx5_glue_dv_create_flow_matcher(struct ibv_context *context,
+ struct mlx5dv_flow_matcher_attr *matcher_attr)
+{
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+ return mlx5dv_create_flow_matcher(context, matcher_attr);
+#else
+ (void)context;
+ (void)matcher_attr;
+ errno = ENOTSUP;
+ return NULL;
+#endif
+}
+
static void *
mlx5_glue_dv_create_flow_matcher(struct ibv_context *context,
struct mlx5dv_flow_matcher_attr *matcher_attr,
@@ -633,7 +653,7 @@ mlx5_glue_dv_create_flow_matcher(struct ibv_context
*context,
matcher_attr->match_mask);
#else
(void)tbl;
- return mlx5dv_create_flow_matcher(context, matcher_attr);
+ return __mlx5_glue_dv_create_flow_matcher(context, matcher_attr);
#endif
#else
(void)context;
@@ -644,6 +664,26 @@ mlx5_glue_dv_create_flow_matcher(struct ibv_context
*context,
#endif
}
+static void *
+__mlx5_glue_dv_create_flow(void *matcher,
+ void *match_value,
+ size_t num_actions,
+ void *actions)
+{
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+ return mlx5dv_create_flow(matcher,
+ match_value,
+ num_actions,
+ (struct mlx5dv_flow_action_attr *)actions);
+#else
+ (void)matcher;
+ (void)match_value;
+ (void)num_actions;
+ (void)actions;
+ return NULL;
+#endif
+}
+
static void *
mlx5_glue_dv_create_flow(void *matcher,
void *match_value,
@@ -663,8 +703,8 @@ mlx5_glue_dv_create_flow(void *matcher,
for (i = 0; i < num_actions; i++)
actions_attr[i] =
*((struct mlx5dv_flow_action_attr *)(actions[i]));
- return mlx5dv_create_flow(matcher, match_value,
- num_actions, actions_attr);
+ return __mlx5_glue_dv_create_flow(matcher, match_value,
+ num_actions, actions_attr);
#endif
#else
(void)matcher;
@@ -735,6 +775,26 @@ mlx5_glue_dv_create_flow_action_dest_devx_tir(void *tir)
#endif
}
+static void *
+__mlx5_glue_dv_create_flow_action_modify_header
+ (struct ibv_context *ctx,
+size_t actions_sz,
+uint64_t actions[],
+enum mlx5dv_flow_table_type ft_type)
+{
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+ return mlx5dv_create_flow_action_modify_header
+ (ctx, actions_sz, actions, ft_type);
+#else
+ (void)ctx;
+ (void)ft_type;
+ (void)actions_sz;
+ (void)actions;
+ errno = ENOTSUP;
+ return NULL;
+#endif
+}
+
static void *
mlx5_glue_dv_create_flow_action_modify_header
(struct ibv_context *ctx,
@@ -758,7 +818,7 @@ mlx5_glue_dv_create_flow_action_modify_header
if (!action)
return NULL;
action->type = MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION;
- action->action = mlx5dv_create_flow_action_modify_header
+ action->action = __mlx5_glue_dv_create_flow_action_modify_header
(ctx, actions_sz, actions, ft_type);
return action;
#endif
@@ -774,6 +834,27 @@ mlx5_glue_dv_create_flow_action_modify_header
#endif
}
+static void *
+__mlx5_glue_dv_create_flow_action_packet_reformat
+ (struct ibv_context *ctx,
+size_t data_sz, void *data,
+enum mlx5dv_flow_action_packet_reformat_type reformat_type,
+enum mlx5dv_flow_table_type ft_type)
+{
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+ return mlx5dv_create_flow_action_
[v3 06/18] net/mlx5: provide the available tag registers
From: Bing Zhao
The available tags that can be used by the application are fixed
after startup.
A global array is used to store the information and transfer the TAG
item directly from the ID to the REG_C_x.
Signed-off-by: Bing Zhao
---
drivers/net/mlx5/linux/mlx5_os.c | 2 +
drivers/net/mlx5/mlx5.c | 2 +
drivers/net/mlx5/mlx5.h | 1 +
drivers/net/mlx5/mlx5_defs.h | 2 +
drivers/net/mlx5/mlx5_flow.c | 11 +
drivers/net/mlx5/mlx5_flow.h | 27
drivers/net/mlx5/mlx5_flow_hw.c | 76
7 files changed, 121 insertions(+)
diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index 98c6374547..aed55e6a62 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -1545,6 +1545,8 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
#ifdef HAVE_IBV_FLOW_DV_SUPPORT
if (priv->vport_meta_mask)
flow_hw_set_port_info(eth_dev);
+ /* Only HWS requires this information. */
+ flow_hw_init_tags_set(eth_dev);
return eth_dev;
#else
DRV_LOG(ERR, "DV support is missing for HWS.");
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 1d10932619..b39ef1ecbe 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -1945,6 +1945,8 @@ mlx5_dev_close(struct rte_eth_dev *dev)
#if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
flow_hw_resource_release(dev);
flow_hw_clear_port_info(dev);
+ if (priv->sh->config.dv_flow_en == 2)
+ flow_hw_clear_tags_set(dev);
#endif
if (priv->rxq_privs != NULL) {
/* XXX race condition if mlx5_rx_burst() is still running. */
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 3c9e6bad53..741be2df98 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -1200,6 +1200,7 @@ struct mlx5_dev_ctx_shared {
uint32_t drop_action_check_flag:1; /* Check Flag for drop action. */
uint32_t flow_priority_check_flag:1; /* Check Flag for flow priority. */
uint32_t metadata_regc_check_flag:1; /* Check Flag for metadata REGC. */
+ uint32_t hws_tags:1; /* Check if tags info for HWS initialized. */
uint32_t max_port; /* Maximal IB device port index. */
struct mlx5_bond_info bond; /* Bonding information. */
struct mlx5_common_device *cdev; /* Backend mlx5 device. */
diff --git a/drivers/net/mlx5/mlx5_defs.h b/drivers/net/mlx5/mlx5_defs.h
index 018d3f0f0c..585afb0a98 100644
--- a/drivers/net/mlx5/mlx5_defs.h
+++ b/drivers/net/mlx5/mlx5_defs.h
@@ -139,6 +139,8 @@
#define MLX5_XMETA_MODE_META32 2
/* Provide info on patrial hw miss. Implies MLX5_XMETA_MODE_META16 */
#define MLX5_XMETA_MODE_MISS_INFO 3
+/* Only valid in HWS, 32bits extended META without MARK support in FDB. */
+#define MLX5_XMETA_MODE_META32_HWS 4
/* Tx accurate scheduling on timestamps parameters. */
#define MLX5_TXPP_WAIT_INIT_TS 1000ul /* How long to wait timestamp. */
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index bdb0613d4a..84bc471392 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -39,6 +39,17 @@
*/
struct flow_hw_port_info mlx5_flow_hw_port_infos[RTE_MAX_ETHPORTS];
+/*
+ * A global structure to save the available REG_C_x for tags usage.
+ * The Meter color REG (ASO) and the last available one will be reserved
+ * for PMD internal usage.
+ * Since there is no "port" concept in the driver, it is assumed that the
+ * available tags set will be the minimum intersection.
+ * 3 - in FDB mode / 5 - in legacy mode
+ */
+uint32_t mlx5_flow_hw_avl_tags_init_cnt;
+enum modify_reg mlx5_flow_hw_avl_tags[MLX5_FLOW_HW_TAGS_MAX] = {REG_NON};
+
struct tunnel_default_miss_ctx {
uint16_t *queue;
__extension__
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index 17102623c1..2002f6ef4b 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -1331,6 +1331,10 @@ struct flow_hw_port_info {
extern struct flow_hw_port_info mlx5_flow_hw_port_infos[RTE_MAX_ETHPORTS];
+#define MLX5_FLOW_HW_TAGS_MAX 8
+extern uint32_t mlx5_flow_hw_avl_tags_init_cnt;
+extern enum modify_reg mlx5_flow_hw_avl_tags[];
+
/*
* Get metadata match tag and mask for given rte_eth_dev port.
* Used in HWS rule creation.
@@ -1372,9 +1376,32 @@ flow_hw_get_wire_port(struct ibv_context *ibctx)
}
#endif
+/*
+ * Convert metadata or tag to the actual register.
+ * META: Can only be used to match in the FDB in this stage, fixed C_1.
+ * TAG: C_x expect meter color reg and the reserved ones.
+ * TODO: Per port / device, FDB or NIC for Meta matching.
+ */
+static __rte_always_inline int
+flow_hw_get_reg_id(enum rte_flow_item_type type, uint32_t id)
+{
+ switch (type) {
+ case RTE_FLOW_ITEM_TYPE_META:
+ return REG_
[v3 09/18] net/mlx5/hws: Add HWS pool and buddy
From: Erez Shitrit
HWS needs to manage different types of device memory in
an efficient and quick way. For this, memory pools are
being used.
Signed-off-by: Erez Shitrit
Signed-off-by: Alex Vesker
---
drivers/net/mlx5/hws/mlx5dr_buddy.c | 201 +
drivers/net/mlx5/hws/mlx5dr_buddy.h | 22 +
drivers/net/mlx5/hws/mlx5dr_pool.c | 672
drivers/net/mlx5/hws/mlx5dr_pool.h | 152 +++
4 files changed, 1047 insertions(+)
create mode 100644 drivers/net/mlx5/hws/mlx5dr_buddy.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_buddy.h
create mode 100644 drivers/net/mlx5/hws/mlx5dr_pool.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_pool.h
diff --git a/drivers/net/mlx5/hws/mlx5dr_buddy.c
b/drivers/net/mlx5/hws/mlx5dr_buddy.c
new file mode 100644
index 00..9dba95f0b1
--- /dev/null
+++ b/drivers/net/mlx5/hws/mlx5dr_buddy.c
@@ -0,0 +1,201 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2022 NVIDIA Corporation & Affiliates
+ */
+
+#include
+#include
+#include "mlx5dr_internal.h"
+#include "mlx5dr_buddy.h"
+
+static struct rte_bitmap *bitmap_alloc0(int s)
+{
+ struct rte_bitmap *bitmap;
+ uint32_t bmp_size;
+ void *mem;
+
+ bmp_size = rte_bitmap_get_memory_footprint(s);
+ mem = rte_zmalloc("create_bmap", bmp_size, RTE_CACHE_LINE_SIZE);
+ if (!mem) {
+ DR_LOG(ERR, "No mem for bitmap");
+ rte_errno = ENOMEM;
+ return NULL;
+ }
+
+ bitmap = rte_bitmap_init(s, mem, bmp_size);
+ if (!bitmap) {
+ DR_LOG(ERR, "%s Failed to initialize bitmap", __func__);
+ rte_errno = EINVAL;
+ goto err_mem_alloc;
+ }
+
+ return bitmap;
+
+err_mem_alloc:
+ rte_free(mem);
+ return NULL;
+}
+
+static void bitmap_set_bit(struct rte_bitmap *bmp, uint32_t pos)
+{
+ rte_bitmap_set(bmp, pos);
+}
+
+static void bitmap_clear_bit(struct rte_bitmap *bmp, uint32_t pos)
+{
+ rte_bitmap_clear(bmp, pos);
+}
+
+static bool bitmap_test_bit(struct rte_bitmap *bmp, unsigned long n)
+{
+ return !!rte_bitmap_get(bmp, n);
+}
+
+static unsigned long bitmap_ffs(struct rte_bitmap *bmap,
+ unsigned long n, unsigned long m)
+{
+ uint64_t out_slab = 0;
+ uint32_t pos = 0; /* Compilation warn */
+
+ __rte_bitmap_scan_init(bmap);
+ if (!rte_bitmap_scan(bmap, &pos, &out_slab)) {
+ DR_LOG(ERR, "Failed to get slab from bitmap.");
+ return m;
+ }
+ pos = pos + __builtin_ctzll(out_slab);
+
+ if (pos < n) {
+ DR_LOG(ERR, "Unexpected bit (%d < %"PRIx64") from bitmap", pos,
n);
+ return m;
+ }
+ return pos;
+}
+
+static unsigned long mlx5dr_buddy_find_first_bit(struct rte_bitmap *addr,
+uint32_t size)
+{
+ return bitmap_ffs(addr, 0, size);
+}
+
+static int mlx5dr_buddy_init(struct mlx5dr_buddy_mem *buddy, uint32_t
max_order)
+{
+ int i, s;
+
+ buddy->max_order = max_order;
+
+ buddy->bits = simple_calloc(buddy->max_order + 1, sizeof(long *));
+ if (!buddy->bits) {
+ rte_errno = ENOMEM;
+ return -1;
+ }
+
+ buddy->num_free = simple_calloc(buddy->max_order + 1,
sizeof(*buddy->num_free));
+ if (!buddy->num_free) {
+ rte_errno = ENOMEM;
+ goto err_out_free_bits;
+ }
+
+ for (i = 0; i <= (int)buddy->max_order; ++i) {
+ s = 1 << (buddy->max_order - i);
+ buddy->bits[i] = bitmap_alloc0(s);
+ if (!buddy->bits[i])
+ goto err_out_free_num_free;
+ }
+
+ bitmap_set_bit(buddy->bits[buddy->max_order], 0);
+
+ buddy->num_free[buddy->max_order] = 1;
+
+ return 0;
+
+err_out_free_num_free:
+ for (i = 0; i <= (int)buddy->max_order; ++i)
+ rte_free(buddy->bits[i]);
+
+ simple_free(buddy->num_free);
+
+err_out_free_bits:
+ simple_free(buddy->bits);
+ return -1;
+}
+
+struct mlx5dr_buddy_mem *mlx5dr_buddy_create(uint32_t max_order)
+{
+ struct mlx5dr_buddy_mem *buddy;
+
+ buddy = simple_calloc(1, sizeof(*buddy));
+ if (!buddy) {
+ rte_errno = ENOMEM;
+ return NULL;
+ }
+
+ if (mlx5dr_buddy_init(buddy, max_order))
+ goto free_buddy;
+
+ return buddy;
+
+free_buddy:
+ simple_free(buddy);
+ return NULL;
+}
+
+void mlx5dr_buddy_cleanup(struct mlx5dr_buddy_mem *buddy)
+{
+ int i;
+
+ for (i = 0; i <= (int)buddy->max_order; ++i) {
+ rte_free(buddy->bits[i]);
+ }
+
+ simple_free(buddy->num_free);
+ simple_free(bud
[v3 10/18] net/mlx5/hws: Add HWS send layer
HWS configures flows to the HW using a QP, each WQE has
the details of the flow we want to offload. The send layer
allocates the resources needed to send the request to the HW
as well as managing the queues, getting completions and
handling failures.
Signed-off-by: Mark Bloch
Signed-off-by: Alex Vesker
---
drivers/net/mlx5/hws/mlx5dr_send.c | 844 +
drivers/net/mlx5/hws/mlx5dr_send.h | 275 ++
2 files changed, 1119 insertions(+)
create mode 100644 drivers/net/mlx5/hws/mlx5dr_send.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_send.h
diff --git a/drivers/net/mlx5/hws/mlx5dr_send.c
b/drivers/net/mlx5/hws/mlx5dr_send.c
new file mode 100644
index 00..26904a9040
--- /dev/null
+++ b/drivers/net/mlx5/hws/mlx5dr_send.c
@@ -0,0 +1,844 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2022 NVIDIA Corporation & Affiliates
+ */
+
+#include "mlx5dr_internal.h"
+
+struct mlx5dr_send_ring_dep_wqe *
+mlx5dr_send_add_new_dep_wqe(struct mlx5dr_send_engine *queue)
+{
+ struct mlx5dr_send_ring_sq *send_sq = &queue->send_ring->send_sq;
+ unsigned int idx = send_sq->head_dep_idx++ & (queue->num_entries - 1);
+
+ memset(&send_sq->dep_wqe[idx].wqe_data.tag, 0, MLX5DR_MATCH_TAG_SZ);
+
+ return &send_sq->dep_wqe[idx];
+}
+
+void mlx5dr_send_abort_new_dep_wqe(struct mlx5dr_send_engine *queue)
+{
+ queue->send_ring->send_sq.head_dep_idx--;
+}
+
+void mlx5dr_send_all_dep_wqe(struct mlx5dr_send_engine *queue)
+{
+ struct mlx5dr_send_ring_sq *send_sq = &queue->send_ring->send_sq;
+ struct mlx5dr_send_ste_attr ste_attr = {0};
+ struct mlx5dr_send_ring_dep_wqe *dep_wqe;
+
+ ste_attr.send_attr.opmod = MLX5DR_WQE_GTA_OPMOD_STE;
+ ste_attr.send_attr.opcode = MLX5DR_WQE_OPCODE_TBL_ACCESS;
+ ste_attr.send_attr.len = MLX5DR_WQE_SZ_GTA_CTRL +
MLX5DR_WQE_SZ_GTA_DATA;
+ ste_attr.gta_opcode = MLX5DR_WQE_GTA_OP_ACTIVATE;
+
+ /* Fence first from previous depend WQEs */
+ ste_attr.send_attr.fence = 1;
+
+ while (send_sq->head_dep_idx != send_sq->tail_dep_idx) {
+ dep_wqe = &send_sq->dep_wqe[send_sq->tail_dep_idx++ &
(queue->num_entries - 1)];
+
+ /* Notify HW on the last WQE */
+ ste_attr.send_attr.notify_hw = (send_sq->tail_dep_idx ==
send_sq->head_dep_idx);
+ ste_attr.send_attr.user_data = dep_wqe->user_data;
+ ste_attr.send_attr.rule = dep_wqe->rule;
+
+ ste_attr.rtc_0 = dep_wqe->rtc_0;
+ ste_attr.rtc_1 = dep_wqe->rtc_1;
+ ste_attr.retry_rtc_0 = dep_wqe->retry_rtc_0;
+ ste_attr.retry_rtc_1 = dep_wqe->retry_rtc_1;
+ ste_attr.used_id_rtc_0 = &dep_wqe->rule->rtc_0;
+ ste_attr.used_id_rtc_1 = &dep_wqe->rule->rtc_1;
+ ste_attr.wqe_ctrl = &dep_wqe->wqe_ctrl;
+ ste_attr.wqe_data = &dep_wqe->wqe_data;
+
+ mlx5dr_send_ste(queue, &ste_attr);
+
+ /* Fencing is done only on the first WQE */
+ ste_attr.send_attr.fence = 0;
+ }
+}
+
+struct mlx5dr_send_engine_post_ctrl
+mlx5dr_send_engine_post_start(struct mlx5dr_send_engine *queue)
+{
+ struct mlx5dr_send_engine_post_ctrl ctrl;
+
+ ctrl.queue = queue;
+ /* Currently only one send ring is supported */
+ ctrl.send_ring = &queue->send_ring[0];
+ ctrl.num_wqebbs = 0;
+
+ return ctrl;
+}
+
+void mlx5dr_send_engine_post_req_wqe(struct mlx5dr_send_engine_post_ctrl *ctrl,
+char **buf, size_t *len)
+{
+ struct mlx5dr_send_ring_sq *send_sq = &ctrl->send_ring->send_sq;
+ unsigned int idx;
+
+ idx = (send_sq->cur_post + ctrl->num_wqebbs) & send_sq->buf_mask;
+
+ *buf = send_sq->buf + (idx << MLX5_SEND_WQE_SHIFT);
+ *len = MLX5_SEND_WQE_BB;
+
+ if (!ctrl->num_wqebbs) {
+ *buf += sizeof(struct mlx5dr_wqe_ctrl_seg);
+ *len -= sizeof(struct mlx5dr_wqe_ctrl_seg);
+ }
+
+ ctrl->num_wqebbs++;
+}
+
+static void mlx5dr_send_engine_post_ring(struct mlx5dr_send_ring_sq *sq,
+struct mlx5dv_devx_uar *uar,
+struct mlx5dr_wqe_ctrl_seg *wqe_ctrl)
+{
+ rte_compiler_barrier();
+ sq->db[MLX5_SND_DBR] = rte_cpu_to_be_32(sq->cur_post);
+
+ rte_wmb();
+ mlx5dr_uar_write64_relaxed(*((uint64_t *)wqe_ctrl), uar->reg_addr);
+ rte_wmb();
+}
+
+static void
+mlx5dr_send_wqe_set_tag(struct mlx5dr_wqe_gta_data_seg_ste *wqe_data,
+ struct mlx5dr_rule_match_tag *tag,
+ bool is_jumbo)
+{
+ if (is_jumbo) {
+ /* Clear previous possibly dirty control */
+
[v3 08/18] net/mlx5/hws: Add HWS command layer
From: Erez Shitrit
The command layer is used to communicate with the FW,
query capabilities and allocate FW resources needed for HWS.
Signed-off-by: Erez Shitrit
Signed-off-by: Alex Vesker
---
drivers/common/mlx5/mlx5_prm.h| 607 ++-
drivers/net/mlx5/hws/mlx5dr_cmd.c | 948 ++
drivers/net/mlx5/hws/mlx5dr_cmd.h | 230
3 files changed, 1775 insertions(+), 10 deletions(-)
create mode 100644 drivers/net/mlx5/hws/mlx5dr_cmd.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_cmd.h
diff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h
index ca4763f53d..371942ae50 100644
--- a/drivers/common/mlx5/mlx5_prm.h
+++ b/drivers/common/mlx5/mlx5_prm.h
@@ -289,6 +289,8 @@
/* The alignment needed for CQ buffer. */
#define MLX5_CQE_BUF_ALIGNMENT rte_mem_page_size()
+#define MAX_ACTIONS_DATA_IN_HEADER_MODIFY 512
+
/* Completion mode. */
enum mlx5_completion_mode {
MLX5_COMP_ONLY_ERR = 0x0,
@@ -677,6 +679,10 @@ enum {
MLX5_MODIFICATION_TYPE_SET = 0x1,
MLX5_MODIFICATION_TYPE_ADD = 0x2,
MLX5_MODIFICATION_TYPE_COPY = 0x3,
+ MLX5_MODIFICATION_TYPE_INSERT = 0x4,
+ MLX5_MODIFICATION_TYPE_REMOVE = 0x5,
+ MLX5_MODIFICATION_TYPE_NOP = 0x6,
+ MLX5_MODIFICATION_TYPE_REMOVE_WORDS = 0x7,
};
/* The field of packet to be modified. */
@@ -,6 +1117,10 @@ enum {
MLX5_CMD_OP_QUERY_TIS = 0x915,
MLX5_CMD_OP_CREATE_RQT = 0x916,
MLX5_CMD_OP_MODIFY_RQT = 0x917,
+ MLX5_CMD_OP_CREATE_FLOW_TABLE = 0x930,
+ MLX5_CMD_OP_CREATE_FLOW_GROUP = 0x933,
+ MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY = 0x936,
+ MLX5_CMD_OP_MODIFY_FLOW_TABLE = 0x93c,
MLX5_CMD_OP_ALLOC_FLOW_COUNTER = 0x939,
MLX5_CMD_OP_QUERY_FLOW_COUNTER = 0x93b,
MLX5_CMD_OP_CREATE_GENERAL_OBJECT = 0xa00,
@@ -1299,6 +1309,7 @@ enum {
MLX5_SET_HCA_CAP_OP_MOD_ESW = 0x9 << 1,
MLX5_GET_HCA_CAP_OP_MOD_VDPA_EMULATION = 0x13 << 1,
MLX5_GET_HCA_CAP_OP_MOD_CRYPTO = 0x1A << 1,
+ MLX5_GET_HCA_CAP_OP_MOD_WQE_BASED_FLOW_TABLE = 0x1B << 1,
MLX5_GET_HCA_CAP_OP_MOD_PARSE_GRAPH_NODE_CAP = 0x1C << 1,
MLX5_GET_HCA_CAP_OP_MOD_GENERAL_DEVICE_2 = 0x20 << 1,
};
@@ -1317,6 +1328,14 @@ enum {
(1ULL << MLX5_GENERAL_OBJ_TYPE_GENEVE_TLV_OPT)
#define MLX5_GENERAL_OBJ_TYPES_CAP_CONN_TRACK_OFFLOAD \
(1ULL << MLX5_GENERAL_OBJ_TYPE_CONN_TRACK_OFFLOAD)
+#define MLX5_GENERAL_OBJ_TYPES_CAP_RTC \
+ (1ULL << MLX5_GENERAL_OBJ_TYPE_RTC)
+#define MLX5_GENERAL_OBJ_TYPES_CAP_STC \
+ (1ULL << MLX5_GENERAL_OBJ_TYPE_STC)
+#define MLX5_GENERAL_OBJ_TYPES_CAP_STE \
+ (1ULL << MLX5_GENERAL_OBJ_TYPE_STE)
+#define MLX5_GENERAL_OBJ_TYPES_CAP_DEFINER \
+ (1ULL << MLX5_GENERAL_OBJ_TYPE_DEFINER)
#define MLX5_GENERAL_OBJ_TYPES_CAP_DEK \
(1ULL << MLX5_GENERAL_OBJ_TYPE_DEK)
#define MLX5_GENERAL_OBJ_TYPES_CAP_IMPORT_KEK \
@@ -1373,6 +1392,11 @@ enum {
#define MLX5_HCA_FLEX_VXLAN_GPE_ENABLED (1UL << 7)
#define MLX5_HCA_FLEX_ICMP_ENABLED (1UL << 8)
#define MLX5_HCA_FLEX_ICMPV6_ENABLED (1UL << 9)
+#define MLX5_HCA_FLEX_GTPU_ENABLED (1UL << 11)
+#define MLX5_HCA_FLEX_GTPU_DW_2_ENABLED (1UL << 16)
+#define MLX5_HCA_FLEX_GTPU_FIRST_EXT_DW_0_ENABLED (1UL << 17)
+#define MLX5_HCA_FLEX_GTPU_DW_0_ENABLED (1UL << 18)
+#define MLX5_HCA_FLEX_GTPU_TEID_ENABLED (1UL << 19)
/* The device steering logic format. */
#define MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 0x0
@@ -1505,7 +1529,8 @@ struct mlx5_ifc_cmd_hca_cap_bits {
u8 wol_u[0x1];
u8 wol_p[0x1];
u8 stat_rate_support[0x10];
- u8 reserved_at_1f0[0xc];
+ u8 reserved_at_1ef[0xb];
+ u8 wqe_based_flow_table_update_cap[0x1];
u8 cqe_version[0x4];
u8 compact_address_vector[0x1];
u8 striding_rq[0x1];
@@ -1681,7 +1706,9 @@ struct mlx5_ifc_cmd_hca_cap_bits {
u8 cqe_compression[0x1];
u8 cqe_compression_timeout[0x10];
u8 cqe_compression_max_num[0x10];
- u8 reserved_at_5e0[0x10];
+ u8 reserved_at_5e0[0x8];
+ u8 flex_parser_id_gtpu_dw_0[0x4];
+ u8 reserved_at_5ec[0x4];
u8 tag_matching[0x1];
u8 rndv_offload_rc[0x1];
u8 rndv_offload_dc[0x1];
@@ -1691,17 +1718,38 @@ struct mlx5_ifc_cmd_hca_cap_bits {
u8 affiliate_nic_vport_criteria[0x8];
u8 native_port_num[0x8];
u8 num_vhca_ports[0x8];
- u8 reserved_at_618[0x6];
+ u8 flex_parser_id_gtpu_teid[0x4];
+ u8 reserved_at_61c[0x2];
u8 sw_owner_id[0x1];
u8 reserved_at_61f[0x6C];
u8 wait_on_data[0x1];
u8 wait_on_time[0x1];
- u8 reserved_at_68d[0xBB];
+ u8 reserved_at_68d[0x37];
+ u8 flex_parser_id_geneve_opt_0[0x
[v3 11/18] net/mlx5/hws: Add HWS definer layer
Definers are HW objects that are used for matching, rte items
are translated to definers, each definer holds the fields and
bit-masks used for HW flow matching. The definer layer is used
for finding the most efficient definer for each set of items.
In addition to definer creation we also calculate the field
copy (fc) array used for efficient items to WQE conversion.
Signed-off-by: Mark Bloch
Signed-off-by: Alex Vesker
---
drivers/net/mlx5/hws/mlx5dr_definer.c | 1968 +
drivers/net/mlx5/hws/mlx5dr_definer.h | 585
2 files changed, 2553 insertions(+)
create mode 100644 drivers/net/mlx5/hws/mlx5dr_definer.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_definer.h
diff --git a/drivers/net/mlx5/hws/mlx5dr_definer.c
b/drivers/net/mlx5/hws/mlx5dr_definer.c
new file mode 100644
index 00..6b98eb8c96
--- /dev/null
+++ b/drivers/net/mlx5/hws/mlx5dr_definer.c
@@ -0,0 +1,1968 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2022 NVIDIA Corporation & Affiliates
+ */
+
+#include "mlx5dr_internal.h"
+
+#define GTP_PDU_SC 0x85
+#define BAD_PORT 0xBAD
+#define ETH_TYPE_IPV4_VXLAN0x0800
+#define ETH_TYPE_IPV6_VXLAN0x86DD
+#define ETH_VXLAN_DEFAULT_PORT 4789
+
+#define STE_NO_VLAN0x0
+#define STE_SVLAN 0x1
+#define STE_CVLAN 0x2
+#define STE_IPV4 0x1
+#define STE_IPV6 0x2
+#define STE_TCP0x1
+#define STE_UDP0x2
+#define STE_ICMP 0x3
+
+/* Setter function based on bit offset and mask, for 32bit DW*/
+#define _DR_SET_32(p, v, byte_off, bit_off, mask) \
+ do { \
+ u32 _v = v; \
+ *((rte_be32_t *)(p) + ((byte_off) / 4)) = \
+ rte_cpu_to_be_32((rte_be_to_cpu_32(*((u32 *)(p) + \
+ ((byte_off) / 4))) & \
+ (~((mask) << (bit_off | \
+(((_v) & (mask)) << \
+ (bit_off))); \
+ } while (0)
+
+/* Setter function based on bit offset and mask */
+#define DR_SET(p, v, byte_off, bit_off, mask) \
+ do { \
+ if (unlikely((bit_off) < 0)) { \
+ u32 _bit_off = -1 * (bit_off); \
+ u32 second_dw_mask = (mask) & ((1 << _bit_off) - 1); \
+ _DR_SET_32(p, (v) >> _bit_off, byte_off, 0, (mask) >>
_bit_off); \
+ _DR_SET_32(p, (v) & second_dw_mask, (byte_off) +
DW_SIZE, \
+ (bit_off) % BITS_IN_DW, second_dw_mask); \
+ } else { \
+ _DR_SET_32(p, v, byte_off, (bit_off), (mask)); \
+ } \
+ } while (0)
+
+/* Setter function based on byte offset to directly set FULL BE32 value */
+#define DR_SET_BE32(p, v, byte_off, bit_off, mask) \
+ (*((rte_be32_t *)((uint8_t *)(p) + (byte_off))) = (v))
+
+/* Setter function based on byte offset to directly set FULL BE32 value from
ptr */
+#define DR_SET_BE32P(p, v_ptr, byte_off, bit_off, mask) \
+ memcpy((uint8_t *)(p) + (byte_off), v_ptr, 4)
+
+/* Setter function based on byte offset to directly set FULL BE16 value */
+#define DR_SET_BE16(p, v, byte_off, bit_off, mask) \
+ (*((rte_be16_t *)((uint8_t *)(p) + (byte_off))) = (v))
+
+/* Setter function based on byte offset to directly set FULL BE16 value from
ptr */
+#define DR_SET_BE16P(p, v_ptr, byte_off, bit_off, mask) \
+ memcpy((uint8_t *)(p) + (byte_off), v_ptr, 2)
+
+#define DR_CALC_FNAME(field, inner) \
+ ((inner) ? MLX5DR_DEFINER_FNAME_##field##_I : \
+ MLX5DR_DEFINER_FNAME_##field##_O)
+
+#define DR_CALC_SET_HDR(fc, hdr, field) \
+ do { \
+ (fc)->bit_mask = __mlx5_mask(definer_hl, hdr.field); \
+ (fc)->bit_off = __mlx5_dw_bit_off(definer_hl, hdr.field); \
+ (fc)->byte_off = MLX5_BYTE_OFF(definer_hl, hdr.field); \
+ } while (0)
+
+/* Helper to calculate data used by DR_SET */
+#define DR_CALC_SET(fc, hdr, field, is_inner) \
+ do { \
+ if (is_inner) { \
+ DR_CALC_SET_HDR(fc, hdr##_inner, field); \
+ } else { \
+ DR_CALC_SET_HDR(fc, hdr##_outer, field); \
+ } \
+ } while (0)
+
+ #define DR_GET(typ, p, fld) \
+ ((rte_be_to_cpu_32(*((const rte_be32_t *)(p) + \
+ __mlx5_dw_off(typ, fld))) >> __mlx5_dw_bit_off(typ, fld)) & \
+ __mlx5_mask(typ, fld))
+
+struct mlx5dr_definer_sel_ctrl {
+ uint8_t allowed_full_dw; /* Full DW selectors cover all offsets */
+ uint8_t allowed_lim_dw; /* Limited DW selectors cover offset < 64 */
+ uint8_t allowed_bytes; /* Bytes selectors, up to offset 255 */
+ uint8_t used_full_dw;
+ uint8_t used_lim_dw;
+ uint8_t used_bytes;
+ uint8_t full_dw_selector[DW_SELECTORS];
[v3 13/18] net/mlx5/hws: Add HWS table object
HWS table resides under the context object, each context can
have multiple tables with different steering types RX/TX/FDB.
The table is not only a logical object but it is also represented
in the HW, packets can be steered to the table and from there
to other tables.
Signed-off-by: Erez Shitrit
Signed-off-by: Alex Vesker
---
drivers/net/mlx5/hws/mlx5dr_table.c | 248
drivers/net/mlx5/hws/mlx5dr_table.h | 44 +
2 files changed, 292 insertions(+)
create mode 100644 drivers/net/mlx5/hws/mlx5dr_table.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_table.h
diff --git a/drivers/net/mlx5/hws/mlx5dr_table.c
b/drivers/net/mlx5/hws/mlx5dr_table.c
new file mode 100644
index 00..d3f77e4780
--- /dev/null
+++ b/drivers/net/mlx5/hws/mlx5dr_table.c
@@ -0,0 +1,248 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2022 NVIDIA Corporation & Affiliates
+ */
+
+#include "mlx5dr_internal.h"
+
+static void mlx5dr_table_init_next_ft_attr(struct mlx5dr_table *tbl,
+ struct mlx5dr_cmd_ft_create_attr
*ft_attr)
+{
+ ft_attr->type = tbl->fw_ft_type;
+ if (tbl->type == MLX5DR_TABLE_TYPE_FDB)
+ ft_attr->level = tbl->ctx->caps->fdb_ft.max_level - 1;
+ else
+ ft_attr->level = tbl->ctx->caps->nic_ft.max_level - 1;
+ ft_attr->rtc_valid = true;
+}
+
+/* Call this under ctx->ctrl_lock */
+static int
+mlx5dr_table_up_default_fdb_miss_tbl(struct mlx5dr_table *tbl)
+{
+ struct mlx5dr_cmd_ft_create_attr ft_attr = {0};
+ struct mlx5dr_cmd_forward_tbl *default_miss;
+ struct mlx5dr_context *ctx = tbl->ctx;
+ uint8_t tbl_type = tbl->type;
+ uint32_t vport;
+
+ if (tbl->type != MLX5DR_TABLE_TYPE_FDB)
+ return 0;
+
+ if (ctx->common_res[tbl_type].default_miss) {
+ ctx->common_res[tbl_type].default_miss->refcount++;
+ return 0;
+ }
+
+ ft_attr.type = tbl->fw_ft_type;
+ ft_attr.level = tbl->ctx->caps->fdb_ft.max_level; /* The last level */
+ ft_attr.rtc_valid = false;
+
+ assert(ctx->caps->eswitch_manager);
+ vport = ctx->caps->eswitch_manager_vport_number;
+
+ default_miss = mlx5dr_cmd_miss_ft_create(ctx->ibv_ctx, &ft_attr, vport);
+ if (!default_miss) {
+ DR_LOG(ERR, "Failed to default miss table type: 0x%x",
tbl_type);
+ return rte_errno;
+ }
+
+ ctx->common_res[tbl_type].default_miss = default_miss;
+ ctx->common_res[tbl_type].default_miss->refcount++;
+ return 0;
+}
+
+/* Called under pthread_spin_lock(&ctx->ctrl_lock) */
+static void mlx5dr_table_down_default_fdb_miss_tbl(struct mlx5dr_table *tbl)
+{
+ struct mlx5dr_cmd_forward_tbl *default_miss;
+ struct mlx5dr_context *ctx = tbl->ctx;
+ uint8_t tbl_type = tbl->type;
+
+ if (tbl->type != MLX5DR_TABLE_TYPE_FDB)
+ return;
+
+ default_miss = ctx->common_res[tbl_type].default_miss;
+ if (--default_miss->refcount)
+ return;
+
+ mlx5dr_cmd_miss_ft_destroy(default_miss);
+
+ simple_free(default_miss);
+ ctx->common_res[tbl_type].default_miss = NULL;
+}
+
+static int
+mlx5dr_table_connect_to_default_miss_tbl(struct mlx5dr_table *tbl,
+struct mlx5dr_devx_obj *ft)
+{
+ struct mlx5dr_cmd_ft_modify_attr ft_attr = {0};
+ int ret;
+
+ assert(tbl->type == MLX5DR_TABLE_TYPE_FDB);
+
+ mlx5dr_cmd_set_attr_connect_miss_tbl(tbl->ctx,
+tbl->fw_ft_type,
+tbl->type,
+&ft_attr);
+
+ /* Connect to next */
+ ret = mlx5dr_cmd_flow_table_modify(ft, &ft_attr);
+ if (ret) {
+ DR_LOG(ERR, "Failed to connect FT to default FDB FT");
+ return errno;
+ }
+
+ return 0;
+}
+
+struct mlx5dr_devx_obj *
+mlx5dr_table_create_default_ft(struct mlx5dr_table *tbl)
+{
+ struct mlx5dr_cmd_ft_create_attr ft_attr = {0};
+ struct mlx5dr_devx_obj *ft_obj;
+ int ret;
+
+ mlx5dr_table_init_next_ft_attr(tbl, &ft_attr);
+
+ ft_obj = mlx5dr_cmd_flow_table_create(tbl->ctx->ibv_ctx, &ft_attr);
+ if (ft_obj && tbl->type == MLX5DR_TABLE_TYPE_FDB) {
+ /* Take/create ref over the default miss */
+ ret = mlx5dr_table_up_default_fdb_miss_tbl(tbl);
+ if (ret) {
+ DR_LOG(ERR, "Failed to get default fdb miss");
+ goto free_ft_obj;
+ }
+ ret = mlx5dr_table_connect_to_default_miss_tbl(tbl, ft_obj);
+
[v3 12/18] net/mlx5/hws: Add HWS context object
Context is the first mlx5dr object created, all sub object:
table, matcher, rule, action are created using the context.
The context holds the capabilities and send queues used for
configuring the offloads to the HW.
Signed-off-by: Alex Vesker
---
drivers/net/mlx5/hws/mlx5dr_context.c | 223 ++
drivers/net/mlx5/hws/mlx5dr_context.h | 40 +
2 files changed, 263 insertions(+)
create mode 100644 drivers/net/mlx5/hws/mlx5dr_context.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_context.h
diff --git a/drivers/net/mlx5/hws/mlx5dr_context.c
b/drivers/net/mlx5/hws/mlx5dr_context.c
new file mode 100644
index 00..ae86694a51
--- /dev/null
+++ b/drivers/net/mlx5/hws/mlx5dr_context.c
@@ -0,0 +1,223 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2022 NVIDIA Corporation & Affiliates
+ */
+
+#include "mlx5dr_internal.h"
+
+static int mlx5dr_context_pools_init(struct mlx5dr_context *ctx)
+{
+ struct mlx5dr_pool_attr pool_attr = {0};
+ uint8_t max_log_sz;
+ int i;
+
+ if (mlx5dr_pat_init_pattern_cache(&ctx->pattern_cache))
+ return rte_errno;
+
+ /* Create an STC pool per FT type */
+ pool_attr.pool_type = MLX5DR_POOL_TYPE_STC;
+ pool_attr.flags = MLX5DR_POOL_FLAGS_FOR_STC_POOL;
+ max_log_sz = RTE_MIN(MLX5DR_POOL_STC_LOG_SZ,
ctx->caps->stc_alloc_log_max);
+ pool_attr.alloc_log_sz = RTE_MAX(max_log_sz,
ctx->caps->stc_alloc_log_gran);
+
+ for (i = 0; i < MLX5DR_TABLE_TYPE_MAX; i++) {
+ pool_attr.table_type = i;
+ ctx->stc_pool[i] = mlx5dr_pool_create(ctx, &pool_attr);
+ if (!ctx->stc_pool[i]) {
+ DR_LOG(ERR, "Failed to allocate STC pool [%d]", i);
+ goto free_stc_pools;
+ }
+ }
+
+ return 0;
+
+free_stc_pools:
+ for (i = 0; i < MLX5DR_TABLE_TYPE_MAX; i++)
+ if (ctx->stc_pool[i])
+ mlx5dr_pool_destroy(ctx->stc_pool[i]);
+
+ mlx5dr_pat_uninit_pattern_cache(ctx->pattern_cache);
+
+ return rte_errno;
+}
+
+static void mlx5dr_context_pools_uninit(struct mlx5dr_context *ctx)
+{
+ int i;
+
+ mlx5dr_pat_uninit_pattern_cache(ctx->pattern_cache);
+
+ for (i = 0; i < MLX5DR_TABLE_TYPE_MAX; i++) {
+ if (ctx->stc_pool[i])
+ mlx5dr_pool_destroy(ctx->stc_pool[i]);
+ }
+}
+
+static int mlx5dr_context_init_pd(struct mlx5dr_context *ctx,
+ struct ibv_pd *pd)
+{
+ struct mlx5dv_pd mlx5_pd = {0};
+ struct mlx5dv_obj obj;
+ int ret;
+
+ if (pd) {
+ ctx->pd = pd;
+ } else {
+ ctx->pd = mlx5_glue->alloc_pd(ctx->ibv_ctx);
+ if (!ctx->pd) {
+ DR_LOG(ERR, "Failed to allocate PD");
+ rte_errno = errno;
+ return rte_errno;
+ }
+ ctx->flags |= MLX5DR_CONTEXT_FLAG_PRIVATE_PD;
+ }
+
+ obj.pd.in = ctx->pd;
+ obj.pd.out = &mlx5_pd;
+
+ ret = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_PD);
+ if (ret)
+ goto free_private_pd;
+
+ ctx->pd_num = mlx5_pd.pdn;
+
+ return 0;
+
+free_private_pd:
+ if (ctx->flags & MLX5DR_CONTEXT_FLAG_PRIVATE_PD)
+ mlx5_glue->dealloc_pd(ctx->pd);
+
+ return ret;
+}
+
+static int mlx5dr_context_uninit_pd(struct mlx5dr_context *ctx)
+{
+ if (ctx->flags & MLX5DR_CONTEXT_FLAG_PRIVATE_PD)
+ return mlx5_glue->dealloc_pd(ctx->pd);
+
+ return 0;
+}
+
+static void mlx5dr_context_check_hws_supp(struct mlx5dr_context *ctx)
+{
+ struct mlx5dr_cmd_query_caps *caps = ctx->caps;
+
+ /* HWS not supported on device / FW */
+ if (!caps->wqe_based_update) {
+ DR_LOG(INFO, "Required HWS WQE based insertion cap not
supported");
+ return;
+ }
+
+ /* Current solution requires all rules to set reparse bit */
+ if ((!caps->nic_ft.reparse || !caps->fdb_ft.reparse) ||
+ !IS_BIT_SET(caps->rtc_reparse_mode, MLX5_IFC_RTC_REPARSE_ALWAYS)) {
+ DR_LOG(INFO, "Required HWS reparse cap not supported");
+ return;
+ }
+
+ /* FW/HW must support 8DW STE */
+ if (!IS_BIT_SET(caps->ste_format, MLX5_IFC_RTC_STE_FORMAT_8DW)) {
+ DR_LOG(INFO, "Required HWS STE format not supported");
+ return;
+ }
+
+ /* Adding rules by hash and by offset are requirements */
+ if (!IS_BIT_SET(caps->rtc_index_mode,
MLX5_IFC_RTC_STE_UPDATE_MODE_BY_HASH) ||
+ !IS_BIT_SET(caps->rtc_index_mode,
MLX5_IFC_RTC_STE_UPDATE_MODE_BY_OFFSET)) {
+
[v3 14/18] net/mlx5/hws: Add HWS matcher object
HWS matcher resides under the table object, each table can
have multiple chained matcher with different attributes. Each
matcher represents a combination of match and action templates.
Each matcher can contain multiple configurations based on the
templates. Packets are steered from the table to the matcher
and from there to other objects. The matcher allows efficent
HW packet field matching and action execution based on the
configuration done to it.
Signed-off-by: Alex Vesker
---
drivers/net/mlx5/hws/mlx5dr_matcher.c | 922 ++
drivers/net/mlx5/hws/mlx5dr_matcher.h | 76 +++
2 files changed, 998 insertions(+)
create mode 100644 drivers/net/mlx5/hws/mlx5dr_matcher.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_matcher.h
diff --git a/drivers/net/mlx5/hws/mlx5dr_matcher.c
b/drivers/net/mlx5/hws/mlx5dr_matcher.c
new file mode 100644
index 00..835a3908eb
--- /dev/null
+++ b/drivers/net/mlx5/hws/mlx5dr_matcher.c
@@ -0,0 +1,922 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2022 NVIDIA Corporation & Affiliates
+ */
+
+#include "mlx5dr_internal.h"
+
+static bool mlx5dr_matcher_requires_col_tbl(uint8_t log_num_of_rules)
+{
+ /* Collision table concatenation is done only for large rule tables */
+ return log_num_of_rules > MLX5DR_MATCHER_ASSURED_RULES_TH;
+}
+
+static uint8_t mlx5dr_matcher_rules_to_tbl_depth(uint8_t log_num_of_rules)
+{
+ if (mlx5dr_matcher_requires_col_tbl(log_num_of_rules))
+ return MLX5DR_MATCHER_ASSURED_MAIN_TBL_DEPTH;
+
+ /* For small rule tables we use a single deep table to assure insertion
*/
+ return RTE_MIN(log_num_of_rules, MLX5DR_MATCHER_ASSURED_COL_TBL_DEPTH);
+}
+
+static int mlx5dr_matcher_create_end_ft(struct mlx5dr_matcher *matcher)
+{
+ struct mlx5dr_table *tbl = matcher->tbl;
+
+ matcher->end_ft = mlx5dr_table_create_default_ft(tbl);
+ if (!matcher->end_ft) {
+ DR_LOG(ERR, "Failed to create matcher end flow table");
+ return rte_errno;
+ }
+ return 0;
+}
+
+static void mlx5dr_matcher_destroy_end_ft(struct mlx5dr_matcher *matcher)
+{
+ mlx5dr_table_destroy_default_ft(matcher->tbl, matcher->end_ft);
+}
+
+static int mlx5dr_matcher_connect(struct mlx5dr_matcher *matcher)
+{
+ struct mlx5dr_cmd_ft_modify_attr ft_attr = {0};
+ struct mlx5dr_table *tbl = matcher->tbl;
+ struct mlx5dr_matcher *prev = NULL;
+ struct mlx5dr_matcher *next = NULL;
+ struct mlx5dr_matcher *tmp_matcher;
+ struct mlx5dr_devx_obj *ft;
+ int ret;
+
+ /* Find location in matcher list */
+ if (LIST_EMPTY(&tbl->head)) {
+ LIST_INSERT_HEAD(&tbl->head, matcher, next);
+ goto connect;
+ }
+
+ LIST_FOREACH(tmp_matcher, &tbl->head, next) {
+ if (tmp_matcher->attr.priority > matcher->attr.priority) {
+ next = tmp_matcher;
+ break;
+ }
+ prev = tmp_matcher;
+ }
+
+ if (next)
+ LIST_INSERT_BEFORE(next, matcher, next);
+ else
+ LIST_INSERT_AFTER(prev, matcher, next);
+
+connect:
+ ft_attr.modify_fs = MLX5_IFC_MODIFY_FLOW_TABLE_RTC_ID;
+ ft_attr.type = tbl->fw_ft_type;
+
+ /* Connect to next */
+ if (next) {
+ if (next->match_ste.rtc_0)
+ ft_attr.rtc_id_0 = next->match_ste.rtc_0->id;
+ if (next->match_ste.rtc_1)
+ ft_attr.rtc_id_1 = next->match_ste.rtc_1->id;
+
+ ret = mlx5dr_cmd_flow_table_modify(matcher->end_ft, &ft_attr);
+ if (ret) {
+ DR_LOG(ERR, "Failed to connect new matcher to next
RTC");
+ goto remove_from_list;
+ }
+ }
+
+ /* Connect to previous */
+ ft = prev ? prev->end_ft : tbl->ft;
+
+ if (matcher->match_ste.rtc_0)
+ ft_attr.rtc_id_0 = matcher->match_ste.rtc_0->id;
+ if (matcher->match_ste.rtc_1)
+ ft_attr.rtc_id_1 = matcher->match_ste.rtc_1->id;
+
+ ret = mlx5dr_cmd_flow_table_modify(ft, &ft_attr);
+ if (ret) {
+ DR_LOG(ERR, "Failed to connect new matcher to previous FT");
+ goto remove_from_list;
+ }
+
+ return 0;
+
+remove_from_list:
+ LIST_REMOVE(matcher, next);
+ return ret;
+}
+
+static int mlx5dr_matcher_disconnect(struct mlx5dr_matcher *matcher)
+{
+ struct mlx5dr_cmd_ft_modify_attr ft_attr = {0};
+ struct mlx5dr_table *tbl = matcher->tbl;
+ struct mlx5dr_matcher *tmp_matcher;
+ struct mlx5dr_devx_obj *prev_ft;
+ struct mlx5dr_matcher *next;
+ int ret;
+
+ prev_ft = matcher->tbl->ft;
+ LIST_FOR
[v3 15/18] net/mlx5/hws: Add HWS rule object
HWS rule objects reside under the matcher, each rule holds
the configuration for the packet fields to match on and the
set of actions to execute over the packet that has the requested
fields. Rules can be created asynchronously in parallel over multiple
queues to different matchers. Each rule is configured to the HW.
Signed-off-by: Erez Shitrit
Signed-off-by: Alex Vesker
---
drivers/net/mlx5/hws/mlx5dr_rule.c | 528 +
drivers/net/mlx5/hws/mlx5dr_rule.h | 50 +++
2 files changed, 578 insertions(+)
create mode 100644 drivers/net/mlx5/hws/mlx5dr_rule.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_rule.h
diff --git a/drivers/net/mlx5/hws/mlx5dr_rule.c
b/drivers/net/mlx5/hws/mlx5dr_rule.c
new file mode 100644
index 00..b27318e6d4
--- /dev/null
+++ b/drivers/net/mlx5/hws/mlx5dr_rule.c
@@ -0,0 +1,528 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2022 NVIDIA Corporation & Affiliates
+ */
+
+#include "mlx5dr_internal.h"
+
+static void mlx5dr_rule_skip(struct mlx5dr_matcher *matcher,
+const struct rte_flow_item *items,
+bool *skip_rx, bool *skip_tx)
+{
+ struct mlx5dr_match_template *mt = matcher->mt[0];
+ const struct flow_hw_port_info *vport;
+ const struct rte_flow_item_ethdev *v;
+
+ /* Flow_src is the 1st priority */
+ if (matcher->attr.optimize_flow_src) {
+ *skip_tx = matcher->attr.optimize_flow_src ==
MLX5DR_MATCHER_FLOW_SRC_WIRE;
+ *skip_rx = matcher->attr.optimize_flow_src ==
MLX5DR_MATCHER_FLOW_SRC_VPORT;
+ return;
+ }
+
+ /* By default FDB rules are added to both RX and TX */
+ *skip_rx = false;
+ *skip_tx = false;
+
+ if (mt->item_flags & MLX5_FLOW_ITEM_REPRESENTED_PORT) {
+ v = items[mt->vport_item_id].spec;
+ vport = flow_hw_conv_port_id(v->port_id);
+ if (unlikely(!vport)) {
+ DR_LOG(ERR, "Fail to map port ID %d, ignoring",
v->port_id);
+ return;
+ }
+
+ if (!vport->is_wire)
+ /* Match vport ID is not WIRE -> Skip RX */
+ *skip_rx = true;
+ else
+ /* Match vport ID is WIRE -> Skip TX */
+ *skip_tx = true;
+ }
+}
+
+static void mlx5dr_rule_init_dep_wqe(struct mlx5dr_send_ring_dep_wqe *dep_wqe,
+struct mlx5dr_rule *rule,
+const struct rte_flow_item *items,
+void *user_data)
+{
+ struct mlx5dr_matcher *matcher = rule->matcher;
+ struct mlx5dr_table *tbl = matcher->tbl;
+ bool skip_rx, skip_tx;
+
+ dep_wqe->rule = rule;
+ dep_wqe->user_data = user_data;
+
+ switch (tbl->type) {
+ case MLX5DR_TABLE_TYPE_NIC_RX:
+ case MLX5DR_TABLE_TYPE_NIC_TX:
+ dep_wqe->rtc_0 = matcher->match_ste.rtc_0->id;
+ dep_wqe->retry_rtc_0 = matcher->col_matcher ?
+
matcher->col_matcher->match_ste.rtc_0->id : 0;
+ dep_wqe->rtc_1 = 0;
+ dep_wqe->retry_rtc_1 = 0;
+ break;
+
+ case MLX5DR_TABLE_TYPE_FDB:
+ mlx5dr_rule_skip(matcher, items, &skip_rx, &skip_tx);
+
+ if (!skip_rx) {
+ dep_wqe->rtc_0 = matcher->match_ste.rtc_0->id;
+ dep_wqe->retry_rtc_0 = matcher->col_matcher ?
+
matcher->col_matcher->match_ste.rtc_0->id : 0;
+ } else {
+ dep_wqe->rtc_0 = 0;
+ dep_wqe->retry_rtc_0 = 0;
+ }
+
+ if (!skip_tx) {
+ dep_wqe->rtc_1 = matcher->match_ste.rtc_1->id;
+ dep_wqe->retry_rtc_1 = matcher->col_matcher ?
+
matcher->col_matcher->match_ste.rtc_1->id : 0;
+ } else {
+ dep_wqe->rtc_1 = 0;
+ dep_wqe->retry_rtc_1 = 0;
+ }
+
+ break;
+
+ default:
+ assert(false);
+ break;
+ }
+}
+
+static void mlx5dr_rule_gen_comp(struct mlx5dr_send_engine *queue,
+struct mlx5dr_rule *rule,
+bool err,
+void *user_data,
+enum mlx5dr_rule_status rule_status_on_succ)
+{
+ enum rte_flow_op_status comp_status;
+
+ if (!err) {
+ comp_status = RTE_FLOW_OP_SUCCESS;
+ rule->status = rule_status_on_succ;
[v3 17/18] net/mlx5/hws: Add HWS debug layer
From: Hamdan Igbaria
The debug layer is used to generate a debug CSV file
containing details of the context, table, matcher, rules
and other useful debug information.
Signed-off-by: Hamdan Igbaria
Signed-off-by: Alex Vesker
---
drivers/net/mlx5/hws/mlx5dr_debug.c | 462
drivers/net/mlx5/hws/mlx5dr_debug.h | 28 ++
2 files changed, 490 insertions(+)
create mode 100644 drivers/net/mlx5/hws/mlx5dr_debug.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_debug.h
diff --git a/drivers/net/mlx5/hws/mlx5dr_debug.c
b/drivers/net/mlx5/hws/mlx5dr_debug.c
new file mode 100644
index 00..890a761c48
--- /dev/null
+++ b/drivers/net/mlx5/hws/mlx5dr_debug.c
@@ -0,0 +1,462 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2022 NVIDIA Corporation & Affiliates
+ */
+
+#include "mlx5dr_internal.h"
+
+const char *mlx5dr_debug_action_type_str[] = {
+ [MLX5DR_ACTION_TYP_LAST] = "LAST",
+ [MLX5DR_ACTION_TYP_TNL_L2_TO_L2] = "TNL_L2_TO_L2",
+ [MLX5DR_ACTION_TYP_L2_TO_TNL_L2] = "L2_TO_TNL_L2",
+ [MLX5DR_ACTION_TYP_TNL_L3_TO_L2] = "TNL_L3_TO_L2",
+ [MLX5DR_ACTION_TYP_L2_TO_TNL_L3] = "L2_TO_TNL_L3",
+ [MLX5DR_ACTION_TYP_DROP] = "DROP",
+ [MLX5DR_ACTION_TYP_TIR] = "TIR",
+ [MLX5DR_ACTION_TYP_FT] = "FT",
+ [MLX5DR_ACTION_TYP_CTR] = "CTR",
+ [MLX5DR_ACTION_TYP_TAG] = "TAG",
+ [MLX5DR_ACTION_TYP_MODIFY_HDR] = "MODIFY_HDR",
+ [MLX5DR_ACTION_TYP_VPORT] = "VPORT",
+ [MLX5DR_ACTION_TYP_MISS] = "DEFAULT_MISS",
+ [MLX5DR_ACTION_TYP_POP_VLAN] = "POP_VLAN",
+ [MLX5DR_ACTION_TYP_PUSH_VLAN] = "PUSH_VLAN",
+ [MLX5DR_ACTION_TYP_ASO_METER] = "ASO_METER",
+ [MLX5DR_ACTION_TYP_ASO_CT] = "ASO_CT",
+};
+
+static_assert(ARRAY_SIZE(mlx5dr_debug_action_type_str) ==
MLX5DR_ACTION_TYP_MAX,
+ "Missing mlx5dr_debug_action_type_str");
+
+const char *mlx5dr_debug_action_type_to_str(enum mlx5dr_action_type
action_type)
+{
+ return mlx5dr_debug_action_type_str[action_type];
+}
+
+static int
+mlx5dr_debug_dump_matcher_template_definer(FILE *f,
+ struct mlx5dr_match_template *mt)
+{
+ struct mlx5dr_definer *definer = mt->definer;
+ int i, ret;
+
+ ret = fprintf(f, "%d,0x%" PRIx64 ",0x%" PRIx64 ",%d,%d,",
+ MLX5DR_DEBUG_RES_TYPE_MATCHER_TEMPLATE_DEFINER,
+ (uint64_t)(uintptr_t)definer,
+ (uint64_t)(uintptr_t)mt,
+ definer->obj->id,
+ definer->type);
+ if (ret < 0) {
+ rte_errno = EINVAL;
+ return rte_errno;
+ }
+
+ for (i = 0; i < DW_SELECTORS; i++) {
+ ret = fprintf(f, "0x%x%s", definer->dw_selector[i],
+ (i == DW_SELECTORS - 1) ? "," : "-");
+ if (ret < 0) {
+ rte_errno = EINVAL;
+ return rte_errno;
+ }
+ }
+
+ for (i = 0; i < BYTE_SELECTORS; i++) {
+ ret = fprintf(f, "0x%x%s", definer->byte_selector[i],
+ (i == BYTE_SELECTORS - 1) ? "," : "-");
+ if (ret < 0) {
+ rte_errno = EINVAL;
+ return rte_errno;
+ }
+ }
+
+ for (i = 0; i < MLX5DR_JUMBO_TAG_SZ; i++) {
+ ret = fprintf(f, "%02x", definer->mask.jumbo[i]);
+ if (ret < 0) {
+ rte_errno = EINVAL;
+ return rte_errno;
+ }
+ }
+
+ ret = fprintf(f, "\n");
+ if (ret < 0) {
+ rte_errno = EINVAL;
+ return rte_errno;
+ }
+
+ return 0;
+}
+
+static int
+mlx5dr_debug_dump_matcher_match_template(FILE *f, struct mlx5dr_matcher
*matcher)
+{
+ bool is_root = matcher->tbl->level == MLX5DR_ROOT_LEVEL;
+ int i, ret;
+
+ for (i = 0; i < matcher->num_of_mt; i++) {
+ struct mlx5dr_match_template *mt = matcher->mt[i];
+
+ ret = fprintf(f, "%d,0x%" PRIx64 ",0x%" PRIx64 ",%d,%d\n",
+ MLX5DR_DEBUG_RES_TYPE_MATCHER_MATCH_TEMPLATE,
+ (uint64_t)(uintptr_t)mt,
+ (uint64_t)(uintptr_t)matcher,
+ is_root ? 0 : mt->fc_sz,
+ mt->flags);
+ if (ret < 0) {
+ rte_errno = EINVAL;
+ return rte_errno;
+ }
+
+ i
[v3 16/18] net/mlx5/hws: Add HWS action object
Action objects are used for executing different HW actions
over packets. Each action contains the HW resources and parameters
needed for action use over the HW when creating a rule.
Signed-off-by: Erez Shitrit
Signed-off-by: Alex Vesker
---
drivers/net/mlx5/hws/mlx5dr_action.c | 2221 +
drivers/net/mlx5/hws/mlx5dr_action.h | 253 +++
drivers/net/mlx5/hws/mlx5dr_pat_arg.c | 511 ++
drivers/net/mlx5/hws/mlx5dr_pat_arg.h | 83 +
4 files changed, 3068 insertions(+)
create mode 100644 drivers/net/mlx5/hws/mlx5dr_action.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_action.h
create mode 100644 drivers/net/mlx5/hws/mlx5dr_pat_arg.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_pat_arg.h
diff --git a/drivers/net/mlx5/hws/mlx5dr_action.c
b/drivers/net/mlx5/hws/mlx5dr_action.c
new file mode 100644
index 00..d3eb091498
--- /dev/null
+++ b/drivers/net/mlx5/hws/mlx5dr_action.c
@@ -0,0 +1,2221 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2022 NVIDIA Corporation & Affiliates
+ */
+
+#include "mlx5dr_internal.h"
+
+#define WIRE_PORT 0x
+
+#define MLX5DR_ACTION_METER_INIT_COLOR_OFFSET 1
+
+/* This is the maximum allowed action order for each table type:
+ * TX: POP_VLAN, CTR, ASO_METER, AS_CT, PUSH_VLAN, MODIFY, ENCAP, Term
+ * RX: TAG, DECAP, POP_VLAN, CTR, ASO_METER, ASO_CT, PUSH_VLAN, MODIFY,
+ * ENCAP, Term
+ * FDB: DECAP, POP_VLAN, CTR, ASO_METER, ASO_CT, PUSH_VLAN, MODIFY,
+ * ENCAP, Term
+ */
+static const uint32_t
action_order_arr[MLX5DR_TABLE_TYPE_MAX][MLX5DR_ACTION_TYP_MAX] = {
+ [MLX5DR_TABLE_TYPE_NIC_RX] = {
+ BIT(MLX5DR_ACTION_TYP_TAG),
+ BIT(MLX5DR_ACTION_TYP_TNL_L2_TO_L2) |
+ BIT(MLX5DR_ACTION_TYP_TNL_L3_TO_L2),
+ BIT(MLX5DR_ACTION_TYP_POP_VLAN),
+ BIT(MLX5DR_ACTION_TYP_POP_VLAN),
+ BIT(MLX5DR_ACTION_TYP_CTR),
+ BIT(MLX5DR_ACTION_TYP_ASO_METER),
+ BIT(MLX5DR_ACTION_TYP_ASO_CT),
+ BIT(MLX5DR_ACTION_TYP_PUSH_VLAN),
+ BIT(MLX5DR_ACTION_TYP_PUSH_VLAN),
+ BIT(MLX5DR_ACTION_TYP_MODIFY_HDR),
+ BIT(MLX5DR_ACTION_TYP_L2_TO_TNL_L2) |
+ BIT(MLX5DR_ACTION_TYP_L2_TO_TNL_L3),
+ BIT(MLX5DR_ACTION_TYP_FT) |
+ BIT(MLX5DR_ACTION_TYP_MISS) |
+ BIT(MLX5DR_ACTION_TYP_TIR) |
+ BIT(MLX5DR_ACTION_TYP_DROP),
+ BIT(MLX5DR_ACTION_TYP_LAST),
+ },
+ [MLX5DR_TABLE_TYPE_NIC_TX] = {
+ BIT(MLX5DR_ACTION_TYP_POP_VLAN),
+ BIT(MLX5DR_ACTION_TYP_POP_VLAN),
+ BIT(MLX5DR_ACTION_TYP_CTR),
+ BIT(MLX5DR_ACTION_TYP_ASO_METER),
+ BIT(MLX5DR_ACTION_TYP_ASO_CT),
+ BIT(MLX5DR_ACTION_TYP_PUSH_VLAN),
+ BIT(MLX5DR_ACTION_TYP_PUSH_VLAN),
+ BIT(MLX5DR_ACTION_TYP_MODIFY_HDR),
+ BIT(MLX5DR_ACTION_TYP_L2_TO_TNL_L2) |
+ BIT(MLX5DR_ACTION_TYP_L2_TO_TNL_L3),
+ BIT(MLX5DR_ACTION_TYP_FT) |
+ BIT(MLX5DR_ACTION_TYP_MISS) |
+ BIT(MLX5DR_ACTION_TYP_DROP),
+ BIT(MLX5DR_ACTION_TYP_LAST),
+ },
+ [MLX5DR_TABLE_TYPE_FDB] = {
+ BIT(MLX5DR_ACTION_TYP_TNL_L2_TO_L2) |
+ BIT(MLX5DR_ACTION_TYP_TNL_L3_TO_L2),
+ BIT(MLX5DR_ACTION_TYP_POP_VLAN),
+ BIT(MLX5DR_ACTION_TYP_POP_VLAN),
+ BIT(MLX5DR_ACTION_TYP_CTR),
+ BIT(MLX5DR_ACTION_TYP_ASO_METER),
+ BIT(MLX5DR_ACTION_TYP_ASO_CT),
+ BIT(MLX5DR_ACTION_TYP_PUSH_VLAN),
+ BIT(MLX5DR_ACTION_TYP_PUSH_VLAN),
+ BIT(MLX5DR_ACTION_TYP_MODIFY_HDR),
+ BIT(MLX5DR_ACTION_TYP_L2_TO_TNL_L2) |
+ BIT(MLX5DR_ACTION_TYP_L2_TO_TNL_L3),
+ BIT(MLX5DR_ACTION_TYP_FT) |
+ BIT(MLX5DR_ACTION_TYP_MISS) |
+ BIT(MLX5DR_ACTION_TYP_VPORT) |
+ BIT(MLX5DR_ACTION_TYP_DROP),
+ BIT(MLX5DR_ACTION_TYP_LAST),
+ },
+};
+
+static int mlx5dr_action_get_shared_stc_nic(struct mlx5dr_context *ctx,
+ enum mlx5dr_context_shared_stc_type
stc_type,
+ uint8_t tbl_type)
+{
+ struct mlx5dr_cmd_stc_modify_attr stc_attr = {0};
+ struct mlx5dr_action_shared_stc *shared_stc;
+ int ret;
+
+ pthread_spin_lock(&ctx->ctrl_lock);
+ if (ctx->common_res[tbl_type].shared_stc[stc_type]) {
+
rte_atomic32_add(&ctx->common_res[tbl_type].shared_stc[stc_type]->refcount, 1);
+ pthread_spin_unlock(&ctx->ctrl_lock);
+ return 0;
+ }
+
+ shared_stc = simple_calloc(1, sizeof(*shared_stc));
+ if (!shared_stc) {
+ DR_LOG(ERR, "Failed to allocate memory for sha
[v3 18/18] net/mlx5/hws: Enable HWS
Replace stub implenation of HWS with mlx5dr code.
Signed-off-by: Alex Vesker
---
drivers/common/mlx5/linux/meson.build| 2 +
drivers/net/mlx5/hws/meson.build | 18 +
drivers/net/mlx5/{mlx5_dr.h => hws/mlx5dr.h} | 210 --
drivers/net/mlx5/hws/mlx5dr_internal.h | 93 +
drivers/net/mlx5/meson.build | 5 +-
drivers/net/mlx5/mlx5.h | 2 +-
drivers/net/mlx5/mlx5_dr.c | 383 ---
drivers/net/mlx5/mlx5_flow.h | 11 +-
drivers/net/mlx5/mlx5_flow_hw.c | 10 +-
9 files changed, 307 insertions(+), 427 deletions(-)
create mode 100644 drivers/net/mlx5/hws/meson.build
rename drivers/net/mlx5/{mlx5_dr.h => hws/mlx5dr.h} (66%)
create mode 100644 drivers/net/mlx5/hws/mlx5dr_internal.h
delete mode 100644 drivers/net/mlx5/mlx5_dr.c
diff --git a/drivers/common/mlx5/linux/meson.build
b/drivers/common/mlx5/linux/meson.build
index f9d1937571..8c95e7ab56 100644
--- a/drivers/common/mlx5/linux/meson.build
+++ b/drivers/common/mlx5/linux/meson.build
@@ -229,6 +229,8 @@ foreach arg:has_member_args
endforeach
configure_file(output : 'mlx5_autoconf.h', configuration : config)
+MLX5_HAVE_IBV_FLOW_DV_SUPPORT=config.get('HAVE_IBV_FLOW_DV_SUPPORT')
+
# Build Glue Library
if dlopen_ibverbs
dlopen_name = 'mlx5_glue'
diff --git a/drivers/net/mlx5/hws/meson.build b/drivers/net/mlx5/hws/meson.build
new file mode 100644
index 00..f94798dd2d
--- /dev/null
+++ b/drivers/net/mlx5/hws/meson.build
@@ -0,0 +1,18 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2022 NVIDIA Corporation & Affiliates
+
+includes += include_directories('.')
+sources += files(
+ 'mlx5dr_context.c',
+ 'mlx5dr_table.c',
+ 'mlx5dr_matcher.c',
+ 'mlx5dr_rule.c',
+ 'mlx5dr_action.c',
+ 'mlx5dr_buddy.c',
+ 'mlx5dr_pool.c',
+ 'mlx5dr_cmd.c',
+ 'mlx5dr_send.c',
+ 'mlx5dr_definer.c',
+ 'mlx5dr_debug.c',
+ 'mlx5dr_pat_arg.c',
+)
diff --git a/drivers/net/mlx5/mlx5_dr.h b/drivers/net/mlx5/hws/mlx5dr.h
similarity index 66%
rename from drivers/net/mlx5/mlx5_dr.h
rename to drivers/net/mlx5/hws/mlx5dr.h
index d0b2c15652..980bda0d63 100644
--- a/drivers/net/mlx5/mlx5_dr.h
+++ b/drivers/net/mlx5/hws/mlx5dr.h
@@ -1,9 +1,9 @@
/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright (c) 2021 NVIDIA CORPORATION. All rights reserved.
+ * Copyright (c) 2022 NVIDIA Corporation & Affiliates
*/
-#ifndef MLX5_DR_H_
-#define MLX5_DR_H_
+#ifndef MLX5DR_H_
+#define MLX5DR_H_
#include
@@ -26,6 +26,27 @@ enum mlx5dr_matcher_resource_mode {
MLX5DR_MATCHER_RESOURCE_MODE_HTABLE,
};
+enum mlx5dr_action_type {
+ MLX5DR_ACTION_TYP_LAST,
+ MLX5DR_ACTION_TYP_TNL_L2_TO_L2,
+ MLX5DR_ACTION_TYP_L2_TO_TNL_L2,
+ MLX5DR_ACTION_TYP_TNL_L3_TO_L2,
+ MLX5DR_ACTION_TYP_L2_TO_TNL_L3,
+ MLX5DR_ACTION_TYP_DROP,
+ MLX5DR_ACTION_TYP_TIR,
+ MLX5DR_ACTION_TYP_FT,
+ MLX5DR_ACTION_TYP_CTR,
+ MLX5DR_ACTION_TYP_TAG,
+ MLX5DR_ACTION_TYP_MODIFY_HDR,
+ MLX5DR_ACTION_TYP_VPORT,
+ MLX5DR_ACTION_TYP_MISS,
+ MLX5DR_ACTION_TYP_POP_VLAN,
+ MLX5DR_ACTION_TYP_PUSH_VLAN,
+ MLX5DR_ACTION_TYP_ASO_METER,
+ MLX5DR_ACTION_TYP_ASO_CT,
+ MLX5DR_ACTION_TYP_MAX,
+};
+
enum mlx5dr_action_flags {
MLX5DR_ACTION_FLAG_ROOT_RX = 1 << 0,
MLX5DR_ACTION_FLAG_ROOT_TX = 1 << 1,
@@ -33,7 +54,10 @@ enum mlx5dr_action_flags {
MLX5DR_ACTION_FLAG_HWS_RX = 1 << 3,
MLX5DR_ACTION_FLAG_HWS_TX = 1 << 4,
MLX5DR_ACTION_FLAG_HWS_FDB = 1 << 5,
- MLX5DR_ACTION_FLAG_INLINE = 1 << 6,
+ /* Shared action can be used over a few threads, since data is written
+* only once at the creation of the action.
+*/
+ MLX5DR_ACTION_FLAG_SHARED = 1 << 6,
};
enum mlx5dr_action_reformat_type {
@@ -43,6 +67,18 @@ enum mlx5dr_action_reformat_type {
MLX5DR_ACTION_REFORMAT_TYPE_L2_TO_TNL_L3,
};
+enum mlx5dr_action_aso_meter_color {
+ MLX5DR_ACTION_ASO_METER_COLOR_RED = 0x0,
+ MLX5DR_ACTION_ASO_METER_COLOR_YELLOW = 0x1,
+ MLX5DR_ACTION_ASO_METER_COLOR_GREEN = 0x2,
+ MLX5DR_ACTION_ASO_METER_COLOR_UNDEFINED = 0x3,
+};
+
+enum mlx5dr_action_aso_ct_flags {
+ MLX5DR_ACTION_ASO_CT_DIRECTION_INITIATOR = 0 << 0,
+ MLX5DR_ACTION_ASO_CT_DIRECTION_RESPONDER = 1 << 0,
+};
+
enum mlx5dr_match_template_flags {
/* Allow relaxed matching by skipping derived dependent match fields. */
MLX5DR_MATCH_TEMPLATE_FLAG_RELAXED_MATCH = 1,
@@ -56,7 +92,7 @@ enum mlx5dr_send_queue_actions {
struct mlx5dr_context_attr {
uint16_t queues;
uint16_t queue_size;
- s
[v4 00/18] net/mlx5: Add HW steering low level support
Mellanox ConnetX devices supports packet matching, packet modification and
redirection. These functionalities are also referred to as flow-steering.
To configure a steering rule, the rule is written to the device owned
memory, this memory is accessed and cached by the device when processing
a packet.
The highlight of this patchset is supporting HW Steering (HWS) which
is the new technology supported in new ConnectX devices, HWS allows
configuring steering rules directly to the HW using special HW queues
with minimal CPU effort.
This patchset is the internal low layer implementation for HWS used by
the mlx5 PMD. The mlx5dr (direct rule) is layer that bridges between the
PMD and the HW by configuring the HW offloads based on the PMD logic
v2:
Fix check patch and cosmetic changes
v3:
-Fix unsupported items
-Fix compilation with mlx5dv dependency
v4:
-Fix compile on Windows
Alex Vesker (9):
net/mlx5: Add additional glue functions for HWS
net/mlx5/hws: Add HWS send layer
net/mlx5/hws: Add HWS definer layer
net/mlx5/hws: Add HWS context object
net/mlx5/hws: Add HWS table object
net/mlx5/hws: Add HWS matcher object
net/mlx5/hws: Add HWS rule object
net/mlx5/hws: Add HWS action object
net/mlx5/hws: Enable HWS
Bing Zhao (2):
common/mlx5: query set capability of registers
net/mlx5: provide the available tag registers
Dariusz Sosnowski (1):
net/mlx5: add port to metadata conversion
Erez Shitrit (2):
net/mlx5/hws: Add HWS command layer
net/mlx5/hws: Add HWS pool and buddy
Hamdan Igbaria (1):
net/mlx5/hws: Add HWS debug layer
Suanming Mou (3):
net/mlx5: split flow item translation
net/mlx5: split flow item matcher and value translation
net/mlx5: add hardware steering item translation function
doc/guides/nics/mlx5.rst |5 +-
doc/guides/rel_notes/release_22_11.rst |4 +
drivers/common/mlx5/linux/meson.build|2 +
drivers/common/mlx5/linux/mlx5_glue.c| 121 +-
drivers/common/mlx5/linux/mlx5_glue.h| 17 +
drivers/common/mlx5/mlx5_devx_cmds.c | 30 +
drivers/common/mlx5/mlx5_devx_cmds.h |2 +
drivers/common/mlx5/mlx5_prm.h | 652 -
drivers/net/mlx5/hws/meson.build | 18 +
drivers/net/mlx5/{mlx5_dr.h => hws/mlx5dr.h} | 209 +-
drivers/net/mlx5/hws/mlx5dr_action.c | +++
drivers/net/mlx5/hws/mlx5dr_action.h | 253 ++
drivers/net/mlx5/hws/mlx5dr_buddy.c | 201 ++
drivers/net/mlx5/hws/mlx5dr_buddy.h | 22 +
drivers/net/mlx5/hws/mlx5dr_cmd.c| 948 +++
drivers/net/mlx5/hws/mlx5dr_cmd.h| 230 ++
drivers/net/mlx5/hws/mlx5dr_context.c| 223 ++
drivers/net/mlx5/hws/mlx5dr_context.h| 40 +
drivers/net/mlx5/hws/mlx5dr_debug.c | 462 +++
drivers/net/mlx5/hws/mlx5dr_debug.h | 28 +
drivers/net/mlx5/hws/mlx5dr_definer.c| 1968 +
drivers/net/mlx5/hws/mlx5dr_definer.h| 585
drivers/net/mlx5/hws/mlx5dr_internal.h | 93 +
drivers/net/mlx5/hws/mlx5dr_matcher.c| 922 ++
drivers/net/mlx5/hws/mlx5dr_matcher.h| 76 +
drivers/net/mlx5/hws/mlx5dr_pat_arg.c| 511
drivers/net/mlx5/hws/mlx5dr_pat_arg.h| 83 +
drivers/net/mlx5/hws/mlx5dr_pool.c | 672 +
drivers/net/mlx5/hws/mlx5dr_pool.h | 152 +
drivers/net/mlx5/hws/mlx5dr_rule.c | 528
drivers/net/mlx5/hws/mlx5dr_rule.h | 50 +
drivers/net/mlx5/hws/mlx5dr_send.c | 844 ++
drivers/net/mlx5/hws/mlx5dr_send.h | 275 ++
drivers/net/mlx5/hws/mlx5dr_table.c | 248 ++
drivers/net/mlx5/hws/mlx5dr_table.h | 44 +
drivers/net/mlx5/linux/mlx5_os.c | 12 +-
drivers/net/mlx5/meson.build |7 +-
drivers/net/mlx5/mlx5.c |9 +-
drivers/net/mlx5/mlx5.h |8 +-
drivers/net/mlx5/mlx5_defs.h |2 +
drivers/net/mlx5/mlx5_devx.c |2 +-
drivers/net/mlx5/mlx5_dr.c | 383 ---
drivers/net/mlx5/mlx5_flow.c | 29 +-
drivers/net/mlx5/mlx5_flow.h | 174 +-
drivers/net/mlx5/mlx5_flow_dv.c | 2631 +-
drivers/net/mlx5/mlx5_flow_hw.c | 115 +-
46 files changed, 14386 insertions(+), 1726 deletions(-)
create mode 100644 drivers/net/mlx5/hws/meson.build
rename drivers/net/mlx5/{mlx5_dr.h => hws/mlx5dr.h} (66%)
create mode 100644 drivers/net/mlx5/hws/mlx5dr_action.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_action.h
create mode 100644 drivers/net/mlx5/hws/mlx5dr_buddy.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_buddy.h
create mode 100644 drivers/net/mlx5/hws/mlx5dr_cmd.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_cmd.h
create mode 100644 drivers/net/mlx5/hws/mlx5dr_context.c
create mode 100644 drivers/net/ml
[v4 01/18] net/mlx5: split flow item translation
From: Suanming Mou
In order to share the item translation code with hardware steering
mode, this commit splits flow item translation code to a dedicate
function.
Signed-off-by: Suanming Mou
---
drivers/net/mlx5/mlx5_flow_dv.c | 1915 ---
1 file changed, 979 insertions(+), 936 deletions(-)
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 91f287af5c..70a3279e2f 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -13029,8 +13029,7 @@ flow_dv_translate_create_conntrack(struct rte_eth_dev
*dev,
}
/**
- * Fill the flow with DV spec, lock free
- * (mutex should be acquired by caller).
+ * Translate the flow item to matcher.
*
* @param[in] dev
* Pointer to rte_eth_dev structure.
@@ -13040,8 +13039,8 @@ flow_dv_translate_create_conntrack(struct rte_eth_dev
*dev,
* Pointer to the flow attributes.
* @param[in] items
* Pointer to the list of items.
- * @param[in] actions
- * Pointer to the list of actions.
+ * @param[in] matcher
+ * Pointer to the flow matcher.
* @param[out] error
* Pointer to the error structure.
*
@@ -13049,1041 +13048,1086 @@ flow_dv_translate_create_conntrack(struct
rte_eth_dev *dev,
* 0 on success, a negative errno value otherwise and rte_errno is set.
*/
static int
-flow_dv_translate(struct rte_eth_dev *dev,
- struct mlx5_flow *dev_flow,
- const struct rte_flow_attr *attr,
- const struct rte_flow_item items[],
- const struct rte_flow_action actions[],
- struct rte_flow_error *error)
+flow_dv_translate_items(struct rte_eth_dev *dev,
+ struct mlx5_flow *dev_flow,
+ const struct rte_flow_attr *attr,
+ const struct rte_flow_item items[],
+ struct mlx5_flow_dv_matcher *matcher,
+ struct rte_flow_error *error)
{
struct mlx5_priv *priv = dev->data->dev_private;
- struct mlx5_sh_config *dev_conf = &priv->sh->config;
struct rte_flow *flow = dev_flow->flow;
struct mlx5_flow_handle *handle = dev_flow->handle;
struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
- struct mlx5_flow_rss_desc *rss_desc;
+ struct mlx5_flow_rss_desc *rss_desc = &wks->rss_desc;
uint64_t item_flags = 0;
uint64_t last_item = 0;
- uint64_t action_flags = 0;
- struct mlx5_flow_dv_matcher matcher = {
- .mask = {
- .size = sizeof(matcher.mask.buf),
- },
- };
- int actions_n = 0;
- bool actions_end = false;
- union {
- struct mlx5_flow_dv_modify_hdr_resource res;
- uint8_t len[sizeof(struct mlx5_flow_dv_modify_hdr_resource) +
- sizeof(struct mlx5_modification_cmd) *
- (MLX5_MAX_MODIFY_NUM + 1)];
- } mhdr_dummy;
- struct mlx5_flow_dv_modify_hdr_resource *mhdr_res = &mhdr_dummy.res;
- const struct rte_flow_action_count *count = NULL;
- const struct rte_flow_action_age *non_shared_age = NULL;
- union flow_dv_attr flow_attr = { .attr = 0 };
- uint32_t tag_be;
- union mlx5_flow_tbl_key tbl_key;
- uint32_t modify_action_position = UINT32_MAX;
- void *match_mask = matcher.mask.buf;
+ void *match_mask = matcher->mask.buf;
void *match_value = dev_flow->dv.value.buf;
uint8_t next_protocol = 0xff;
- struct rte_vlan_hdr vlan = { 0 };
- struct mlx5_flow_dv_dest_array_resource mdest_res;
- struct mlx5_flow_dv_sample_resource sample_res;
- void *sample_actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS] = {0};
- const struct rte_flow_action_sample *sample = NULL;
- struct mlx5_flow_sub_actions_list *sample_act;
- uint32_t sample_act_pos = UINT32_MAX;
- uint32_t age_act_pos = UINT32_MAX;
- uint32_t num_of_dest = 0;
- int tmp_actions_n = 0;
- uint32_t table;
- int ret = 0;
- const struct mlx5_flow_tunnel *tunnel = NULL;
- struct flow_grp_info grp_info = {
- .external = !!dev_flow->external,
- .transfer = !!attr->transfer,
- .fdb_def_rule = !!priv->fdb_def_rule,
- .skip_scale = dev_flow->skip_scale &
- (1 << MLX5_SCALE_FLOW_GROUP_BIT),
- .std_tbl_fix = true,
- };
+ uint16_t priority = 0;
const struct rte_flow_item *integrity_items[2] = {NULL, NULL};
const struct rte_flow_item *tunnel_item = NULL;
const struct rte_flow_item *gre_item = NULL;
+ int ret = 0;
- if (!wks)
- return rte_flow_error_set(error, ENOMEM,
- RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
- NULL,
- "failed to
[v4 03/18] net/mlx5: add hardware steering item translation function
From: Suanming Mou
As hardware steering root table flows still work under FW steering
mode. This commit provides shared item tranlsation code for hardware
steering root table flows.
Signed-off-by: Suanming Mou
---
drivers/net/mlx5/mlx5_flow.c| 10 +--
drivers/net/mlx5/mlx5_flow.h| 52 -
drivers/net/mlx5/mlx5_flow_dv.c | 134
3 files changed, 155 insertions(+), 41 deletions(-)
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 6fb1d53fc5..742dbd6358 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -7108,7 +7108,7 @@ mlx5_flow_create_devx_sq_miss_flow(struct rte_eth_dev
*dev, uint32_t txq)
struct rte_flow_item_port_id port_spec = {
.id = MLX5_PORT_ESW_MGR,
};
- struct mlx5_rte_flow_item_tx_queue txq_spec = {
+ struct mlx5_rte_flow_item_sq txq_spec = {
.queue = txq,
};
struct rte_flow_item pattern[] = {
@@ -7118,7 +7118,7 @@ mlx5_flow_create_devx_sq_miss_flow(struct rte_eth_dev
*dev, uint32_t txq)
},
{
.type = (enum rte_flow_item_type)
- MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE,
+ MLX5_RTE_FLOW_ITEM_TYPE_SQ,
.spec = &txq_spec,
},
{
@@ -7504,16 +7504,16 @@ mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev,
.egress = 1,
.priority = 0,
};
- struct mlx5_rte_flow_item_tx_queue queue_spec = {
+ struct mlx5_rte_flow_item_sq queue_spec = {
.queue = queue,
};
- struct mlx5_rte_flow_item_tx_queue queue_mask = {
+ struct mlx5_rte_flow_item_sq queue_mask = {
.queue = UINT32_MAX,
};
struct rte_flow_item items[] = {
{
.type = (enum rte_flow_item_type)
- MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE,
+ MLX5_RTE_FLOW_ITEM_TYPE_SQ,
.spec = &queue_spec,
.last = NULL,
.mask = &queue_mask,
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index 2ebb8496f2..288e09d5ba 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -28,7 +28,7 @@
enum mlx5_rte_flow_item_type {
MLX5_RTE_FLOW_ITEM_TYPE_END = INT_MIN,
MLX5_RTE_FLOW_ITEM_TYPE_TAG,
- MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE,
+ MLX5_RTE_FLOW_ITEM_TYPE_SQ,
MLX5_RTE_FLOW_ITEM_TYPE_VLAN,
MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL,
};
@@ -95,7 +95,7 @@ struct mlx5_flow_action_copy_mreg {
};
/* Matches on source queue. */
-struct mlx5_rte_flow_item_tx_queue {
+struct mlx5_rte_flow_item_sq {
uint32_t queue;
};
@@ -159,7 +159,7 @@ enum mlx5_feature_name {
#define MLX5_FLOW_LAYER_GENEVE (1u << 26)
/* Queue items. */
-#define MLX5_FLOW_ITEM_TX_QUEUE (1u << 27)
+#define MLX5_FLOW_ITEM_SQ (1u << 27)
/* Pattern tunnel Layer bits (continued). */
#define MLX5_FLOW_LAYER_GTP (1u << 28)
@@ -196,6 +196,9 @@ enum mlx5_feature_name {
#define MLX5_FLOW_ITEM_PORT_REPRESENTOR (UINT64_C(1) << 41)
#define MLX5_FLOW_ITEM_REPRESENTED_PORT (UINT64_C(1) << 42)
+/* Meter color item */
+#define MLX5_FLOW_ITEM_METER_COLOR (UINT64_C(1) << 44)
+
/* Outer Masks. */
#define MLX5_FLOW_LAYER_OUTER_L3 \
(MLX5_FLOW_LAYER_OUTER_L3_IPV4 | MLX5_FLOW_LAYER_OUTER_L3_IPV6)
@@ -1006,6 +1009,18 @@ flow_items_to_tunnel(const struct rte_flow_item items[])
return items[0].spec;
}
+/* HW steering flow attributes. */
+struct mlx5_flow_attr {
+ uint32_t port_id; /* Port index. */
+ uint32_t group; /* Flow group. */
+ uint32_t priority; /* Original Priority. */
+ /* rss level, used by priority adjustment. */
+ uint32_t rss_level;
+ /* Action flags, used by priority adjustment. */
+ uint32_t act_flags;
+ uint32_t tbl_type; /* Flow table type. */
+};
+
/* Flow structure. */
struct rte_flow {
uint32_t dev_handles;
@@ -1766,6 +1781,32 @@ mlx5_translate_tunnel_etypes(uint64_t pattern_flags)
int flow_hw_q_flow_flush(struct rte_eth_dev *dev,
struct rte_flow_error *error);
+
+/*
+ * Convert rte_mtr_color to mlx5 color.
+ *
+ * @param[in] rcol
+ * rte_mtr_color.
+ *
+ * @return
+ * mlx5 color.
+ */
+static inline int
+rte_col_2_mlx5_col(enum rte_color rcol)
+{
+ switch (rcol) {
+ case RTE_COLOR_GREEN:
+ return MLX5_FLOW_COLOR_GREEN;
+ case RTE_COLOR_YELLOW:
+ return MLX5_FLOW_COLOR_YELLOW;
+ case RTE_COLOR_RED:
+ return MLX5_FLOW_COLOR_RED;
+ default:
+ break;
+ }
+ return MLX5_FLOW_COLOR_UNDEFINED;
+}
+
int mlx5_flow_group_to_table(struct rte_eth_dev *dev,
const struct mlx5_f
[v4 02/18] net/mlx5: split flow item matcher and value translation
From: Suanming Mou
As hardware steering mode translates flow matcher and value in two
different stages, split the flow item matcher and value translation
to help reuse the code.
Signed-off-by: Suanming Mou
---
drivers/net/mlx5/mlx5_flow.h| 32 +
drivers/net/mlx5/mlx5_flow_dv.c | 2314 +++
2 files changed, 1185 insertions(+), 1161 deletions(-)
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index 0fa1735b1a..2ebb8496f2 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -1264,6 +1264,38 @@ struct mlx5_flow_workspace {
uint32_t skip_matcher_reg:1;
/* Indicates if need to skip matcher register in translate. */
uint32_t mark:1; /* Indicates if flow contains mark action. */
+ uint32_t vport_meta_tag; /* Used for vport index match. */
+};
+
+/* Matcher translate type. */
+enum MLX5_SET_MATCHER {
+ MLX5_SET_MATCHER_SW_V = 1 << 0,
+ MLX5_SET_MATCHER_SW_M = 1 << 1,
+ MLX5_SET_MATCHER_HS_V = 1 << 2,
+ MLX5_SET_MATCHER_HS_M = 1 << 3,
+};
+
+#define MLX5_SET_MATCHER_SW (MLX5_SET_MATCHER_SW_V | MLX5_SET_MATCHER_SW_M)
+#define MLX5_SET_MATCHER_HS (MLX5_SET_MATCHER_HS_V | MLX5_SET_MATCHER_HS_M)
+#define MLX5_SET_MATCHER_V (MLX5_SET_MATCHER_SW_V | MLX5_SET_MATCHER_HS_V)
+#define MLX5_SET_MATCHER_M (MLX5_SET_MATCHER_SW_M | MLX5_SET_MATCHER_HS_M)
+
+/* Flow matcher workspace intermediate data. */
+struct mlx5_dv_matcher_workspace {
+ uint8_t priority; /* Flow priority. */
+ uint64_t last_item; /* Last item in pattern. */
+ uint64_t item_flags; /* Flow item pattern flags. */
+ uint64_t action_flags; /* Flow action flags. */
+ bool external; /* External flow or not. */
+ uint32_t vlan_tag:12; /* Flow item VLAN tag. */
+ uint8_t next_protocol; /* Tunnel next protocol */
+ uint32_t geneve_tlv_option; /* Flow item Geneve TLV option. */
+ uint32_t group; /* Flow group. */
+ uint16_t udp_dport; /* Flow item UDP port. */
+ const struct rte_flow_attr *attr; /* Flow attribute. */
+ struct mlx5_flow_rss_desc *rss_desc; /* RSS descriptor. */
+ const struct rte_flow_item *tunnel_item; /* Flow tunnel item. */
+ const struct rte_flow_item *gre_item; /* Flow GRE item. */
};
struct mlx5_flow_split_info {
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 70a3279e2f..0589cafc30 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -63,6 +63,25 @@
#define MLX5DV_FLOW_VLAN_PCP_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK)
#define MLX5DV_FLOW_VLAN_VID_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_VID_MASK)
+#define MLX5_ITEM_VALID(item, key_type) \
+ (((MLX5_SET_MATCHER_SW & (key_type)) && !((item)->spec)) || \
+((MLX5_SET_MATCHER_HS_V == (key_type)) && !((item)->spec)) || \
+((MLX5_SET_MATCHER_HS_M == (key_type)) && !((item)->mask)))
+
+#define MLX5_ITEM_UPDATE(item, key_type, v, m, gm) \
+ do { \
+ if ((key_type) == MLX5_SET_MATCHER_SW_V) { \
+ v = (item)->spec; \
+ m = (item)->mask ? (item)->mask : (gm); \
+ } else if ((key_type) == MLX5_SET_MATCHER_HS_V) { \
+ v = (item)->spec; \
+ m = (v); \
+ } else { \
+ v = (item)->mask ? (item)->mask : (gm); \
+ m = (v); \
+ } \
+ } while (0)
+
union flow_dv_attr {
struct {
uint32_t valid:1;
@@ -8323,70 +8342,61 @@ flow_dv_check_valid_spec(void *match_mask, void
*match_value)
static inline void
flow_dv_set_match_ip_version(uint32_t group,
void *headers_v,
-void *headers_m,
+uint32_t key_type,
uint8_t ip_version)
{
- if (group == 0)
- MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
+ if (group == 0 && (key_type & MLX5_SET_MATCHER_M))
+ MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, 0xf);
else
- MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version,
+ MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version,
ip_version);
- MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, ip_version);
MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype, 0);
- MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype, 0);
}
/**
- * Add Ethernet item to matcher and to the value.
+ * Add Ethernet item to the value.
*
- * @param[in, out] matcher
- * Flow matcher.
* @param[in, out] key
* Flow matcher value.
* @param[in] item
* Flow pattern to translate.
* @param[in] inner
* Item is inner pattern.
+ * @param[in] grpup
+ * Flow matcher group.
+ * @param[in] key_type
+ * Set flow matcher mask or value.
*/
static voi
[v4 04/18] net/mlx5: add port to metadata conversion
From: Dariusz Sosnowski
This patch initial version of functions used to:
- convert between ethdev port_id and internal tag/mask value,
- convert between IB context and internal tag/mask value.
Signed-off-by: Dariusz Sosnowski
---
drivers/net/mlx5/linux/mlx5_os.c | 10 +-
drivers/net/mlx5/mlx5.c | 1 +
drivers/net/mlx5/mlx5_flow.c | 6
drivers/net/mlx5/mlx5_flow.h | 52
drivers/net/mlx5/mlx5_flow_hw.c | 29 ++
5 files changed, 97 insertions(+), 1 deletion(-)
diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index 60677eb8d7..98c6374547 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -1541,8 +1541,16 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
if (!priv->hrxqs)
goto error;
rte_rwlock_init(&priv->ind_tbls_lock);
- if (priv->sh->config.dv_flow_en == 2)
+ if (priv->sh->config.dv_flow_en == 2) {
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+ if (priv->vport_meta_mask)
+ flow_hw_set_port_info(eth_dev);
return eth_dev;
+#else
+ DRV_LOG(ERR, "DV support is missing for HWS.");
+ goto error;
+#endif
+ }
/* Port representor shares the same max priority with pf port. */
if (!priv->sh->flow_priority_check_flag) {
/* Supported Verbs flow priority number detection. */
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 752b60d769..1d10932619 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -1944,6 +1944,7 @@ mlx5_dev_close(struct rte_eth_dev *dev)
mlx5_flex_item_port_cleanup(dev);
#if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
flow_hw_resource_release(dev);
+ flow_hw_clear_port_info(dev);
#endif
if (priv->rxq_privs != NULL) {
/* XXX race condition if mlx5_rx_burst() is still running. */
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 742dbd6358..9d94da0868 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -33,6 +33,12 @@
#include "mlx5_common_os.h"
#include "rte_pmd_mlx5.h"
+/*
+ * Shared array for quick translation between port_id and vport mask/values
+ * used for HWS rules.
+ */
+struct flow_hw_port_info mlx5_flow_hw_port_infos[RTE_MAX_ETHPORTS];
+
struct tunnel_default_miss_ctx {
uint16_t *queue;
__extension__
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index 288e09d5ba..17102623c1 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -1323,6 +1323,58 @@ struct mlx5_flow_split_info {
uint64_t prefix_layers; /**< Prefix subflow layers. */
};
+struct flow_hw_port_info {
+ uint32_t regc_mask;
+ uint32_t regc_value;
+ uint32_t is_wire:1;
+};
+
+extern struct flow_hw_port_info mlx5_flow_hw_port_infos[RTE_MAX_ETHPORTS];
+
+/*
+ * Get metadata match tag and mask for given rte_eth_dev port.
+ * Used in HWS rule creation.
+ */
+static __rte_always_inline const struct flow_hw_port_info *
+flow_hw_conv_port_id(const uint16_t port_id)
+{
+ struct flow_hw_port_info *port_info;
+
+ if (port_id >= RTE_MAX_ETHPORTS)
+ return NULL;
+ port_info = &mlx5_flow_hw_port_infos[port_id];
+ return !!port_info->regc_mask ? port_info : NULL;
+}
+
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+/*
+ * Get metadata match tag and mask for the uplink port represented
+ * by given IB context. Used in HWS context creation.
+ */
+static __rte_always_inline const struct flow_hw_port_info *
+flow_hw_get_wire_port(struct ibv_context *ibctx)
+{
+ struct ibv_device *ibdev = ibctx->device;
+ uint16_t port_id;
+
+ MLX5_ETH_FOREACH_DEV(port_id, NULL) {
+ const struct mlx5_priv *priv =
+ rte_eth_devices[port_id].data->dev_private;
+
+ if (priv && priv->master) {
+ struct ibv_context *port_ibctx = priv->sh->cdev->ctx;
+
+ if (port_ibctx->device == ibdev)
+ return flow_hw_conv_port_id(port_id);
+ }
+ }
+ return NULL;
+}
+#endif
+
+void flow_hw_set_port_info(struct rte_eth_dev *dev);
+void flow_hw_clear_port_info(struct rte_eth_dev *dev);
+
typedef int (*mlx5_flow_validate_t)(struct rte_eth_dev *dev,
const struct rte_flow_attr *attr,
const struct rte_flow_item items[],
diff --git a/drivers/net/mlx5/mlx5_flow_hw.c b/drivers/net/mlx5/mlx5_flow_hw.c
index 12498794a5..fe809a83b9 100644
--- a/drivers/net/mlx5/mlx5_flow_hw.c
+++ b/drivers/net/mlx5/mlx5_flow_hw.c
@@ -2208,6 +2208,35 @@ flow_hw_resource_release(struct rte_eth_dev *dev)
priv->nb_queue = 0;
}
+/* Sets vport tag and mask, for given port, used in HWS ru
[v4 06/18] net/mlx5: provide the available tag registers
From: Bing Zhao
The available tags that can be used by the application are fixed
after startup.
A global array is used to store the information and transfer the TAG
item directly from the ID to the REG_C_x.
Signed-off-by: Bing Zhao
---
drivers/net/mlx5/linux/mlx5_os.c | 2 +
drivers/net/mlx5/mlx5.c | 2 +
drivers/net/mlx5/mlx5.h | 1 +
drivers/net/mlx5/mlx5_defs.h | 2 +
drivers/net/mlx5/mlx5_flow.c | 11 +
drivers/net/mlx5/mlx5_flow.h | 27
drivers/net/mlx5/mlx5_flow_hw.c | 76
7 files changed, 121 insertions(+)
diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index 98c6374547..aed55e6a62 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -1545,6 +1545,8 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
#ifdef HAVE_IBV_FLOW_DV_SUPPORT
if (priv->vport_meta_mask)
flow_hw_set_port_info(eth_dev);
+ /* Only HWS requires this information. */
+ flow_hw_init_tags_set(eth_dev);
return eth_dev;
#else
DRV_LOG(ERR, "DV support is missing for HWS.");
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 1d10932619..b39ef1ecbe 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -1945,6 +1945,8 @@ mlx5_dev_close(struct rte_eth_dev *dev)
#if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
flow_hw_resource_release(dev);
flow_hw_clear_port_info(dev);
+ if (priv->sh->config.dv_flow_en == 2)
+ flow_hw_clear_tags_set(dev);
#endif
if (priv->rxq_privs != NULL) {
/* XXX race condition if mlx5_rx_burst() is still running. */
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 3c9e6bad53..741be2df98 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -1200,6 +1200,7 @@ struct mlx5_dev_ctx_shared {
uint32_t drop_action_check_flag:1; /* Check Flag for drop action. */
uint32_t flow_priority_check_flag:1; /* Check Flag for flow priority. */
uint32_t metadata_regc_check_flag:1; /* Check Flag for metadata REGC. */
+ uint32_t hws_tags:1; /* Check if tags info for HWS initialized. */
uint32_t max_port; /* Maximal IB device port index. */
struct mlx5_bond_info bond; /* Bonding information. */
struct mlx5_common_device *cdev; /* Backend mlx5 device. */
diff --git a/drivers/net/mlx5/mlx5_defs.h b/drivers/net/mlx5/mlx5_defs.h
index 018d3f0f0c..585afb0a98 100644
--- a/drivers/net/mlx5/mlx5_defs.h
+++ b/drivers/net/mlx5/mlx5_defs.h
@@ -139,6 +139,8 @@
#define MLX5_XMETA_MODE_META32 2
/* Provide info on patrial hw miss. Implies MLX5_XMETA_MODE_META16 */
#define MLX5_XMETA_MODE_MISS_INFO 3
+/* Only valid in HWS, 32bits extended META without MARK support in FDB. */
+#define MLX5_XMETA_MODE_META32_HWS 4
/* Tx accurate scheduling on timestamps parameters. */
#define MLX5_TXPP_WAIT_INIT_TS 1000ul /* How long to wait timestamp. */
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 9d94da0868..dd3d2bb1a4 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -39,6 +39,17 @@
*/
struct flow_hw_port_info mlx5_flow_hw_port_infos[RTE_MAX_ETHPORTS];
+/*
+ * A global structure to save the available REG_C_x for tags usage.
+ * The Meter color REG (ASO) and the last available one will be reserved
+ * for PMD internal usage.
+ * Since there is no "port" concept in the driver, it is assumed that the
+ * available tags set will be the minimum intersection.
+ * 3 - in FDB mode / 5 - in legacy mode
+ */
+uint32_t mlx5_flow_hw_avl_tags_init_cnt;
+enum modify_reg mlx5_flow_hw_avl_tags[MLX5_FLOW_HW_TAGS_MAX] = {REG_NON};
+
struct tunnel_default_miss_ctx {
uint16_t *queue;
__extension__
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index 17102623c1..2002f6ef4b 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -1331,6 +1331,10 @@ struct flow_hw_port_info {
extern struct flow_hw_port_info mlx5_flow_hw_port_infos[RTE_MAX_ETHPORTS];
+#define MLX5_FLOW_HW_TAGS_MAX 8
+extern uint32_t mlx5_flow_hw_avl_tags_init_cnt;
+extern enum modify_reg mlx5_flow_hw_avl_tags[];
+
/*
* Get metadata match tag and mask for given rte_eth_dev port.
* Used in HWS rule creation.
@@ -1372,9 +1376,32 @@ flow_hw_get_wire_port(struct ibv_context *ibctx)
}
#endif
+/*
+ * Convert metadata or tag to the actual register.
+ * META: Can only be used to match in the FDB in this stage, fixed C_1.
+ * TAG: C_x expect meter color reg and the reserved ones.
+ * TODO: Per port / device, FDB or NIC for Meta matching.
+ */
+static __rte_always_inline int
+flow_hw_get_reg_id(enum rte_flow_item_type type, uint32_t id)
+{
+ switch (type) {
+ case RTE_FLOW_ITEM_TYPE_META:
+ return REG_
[v4 05/18] common/mlx5: query set capability of registers
From: Bing Zhao
In the flow table capabilities, new fields are added to query the
capability to set, add, copy to a REG_C_x.
The set capability are queried and saved for the future usage.
Signed-off-by: Bing Zhao
---
drivers/common/mlx5/mlx5_devx_cmds.c | 30 +++
drivers/common/mlx5/mlx5_devx_cmds.h | 2 ++
drivers/common/mlx5/mlx5_prm.h | 45 +---
3 files changed, 73 insertions(+), 4 deletions(-)
diff --git a/drivers/common/mlx5/mlx5_devx_cmds.c
b/drivers/common/mlx5/mlx5_devx_cmds.c
index 76f0b6724f..9c185366d0 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.c
+++ b/drivers/common/mlx5/mlx5_devx_cmds.c
@@ -1064,6 +1064,24 @@ mlx5_devx_cmd_query_hca_attr(void *ctx,
attr->modify_outer_ip_ecn = MLX5_GET
(flow_table_nic_cap, hcattr,
ft_header_modify_nic_receive.outer_ip_ecn);
+ attr->set_reg_c = 0xff;
+ if (attr->nic_flow_table) {
+#define GET_RX_REG_X_BITS \
+ MLX5_GET(flow_table_nic_cap, hcattr, \
+ft_header_modify_nic_receive.metadata_reg_c_x)
+#define GET_TX_REG_X_BITS \
+ MLX5_GET(flow_table_nic_cap, hcattr, \
+ft_header_modify_nic_transmit.metadata_reg_c_x)
+
+ uint32_t tx_reg, rx_reg;
+
+ tx_reg = GET_TX_REG_X_BITS;
+ rx_reg = GET_RX_REG_X_BITS;
+ attr->set_reg_c &= (rx_reg & tx_reg);
+
+#undef GET_RX_REG_X_BITS
+#undef GET_TX_REG_X_BITS
+ }
attr->pkt_integrity_match = mlx5_devx_query_pkt_integrity_match(hcattr);
attr->inner_ipv4_ihl = MLX5_GET
(flow_table_nic_cap, hcattr,
@@ -1163,6 +1181,18 @@ mlx5_devx_cmd_query_hca_attr(void *ctx,
attr->esw_mgr_vport_id =
MLX5_GET(esw_cap, hcattr, esw_manager_vport_number);
}
+ if (attr->eswitch_manager) {
+ uint32_t esw_reg;
+
+ hcattr = mlx5_devx_get_hca_cap(ctx, in, out, &rc,
+ MLX5_GET_HCA_CAP_OP_MOD_ESW_FLOW_TABLE |
+ MLX5_HCA_CAP_OPMOD_GET_CUR);
+ if (!hcattr)
+ return rc;
+ esw_reg = MLX5_GET(flow_table_esw_cap, hcattr,
+ ft_header_modify_esw_fdb.metadata_reg_c_x);
+ attr->set_reg_c &= esw_reg;
+ }
return 0;
error:
rc = (rc > 0) ? -rc : rc;
diff --git a/drivers/common/mlx5/mlx5_devx_cmds.h
b/drivers/common/mlx5/mlx5_devx_cmds.h
index cceaf3411d..a10aa3331b 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.h
+++ b/drivers/common/mlx5/mlx5_devx_cmds.h
@@ -263,6 +263,8 @@ struct mlx5_hca_attr {
uint32_t crypto_wrapped_import_method:1;
uint16_t esw_mgr_vport_id; /* E-Switch Mgr vport ID . */
uint16_t max_wqe_sz_sq;
+ uint32_t set_reg_c:8;
+ uint32_t nic_flow_table:1;
uint32_t modify_outer_ip_ecn:1;
};
diff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h
index 9c1c93f916..ca4763f53d 100644
--- a/drivers/common/mlx5/mlx5_prm.h
+++ b/drivers/common/mlx5/mlx5_prm.h
@@ -1295,6 +1295,7 @@ enum {
MLX5_GET_HCA_CAP_OP_MOD_QOS_CAP = 0xc << 1,
MLX5_GET_HCA_CAP_OP_MOD_ROCE = 0x4 << 1,
MLX5_GET_HCA_CAP_OP_MOD_NIC_FLOW_TABLE = 0x7 << 1,
+ MLX5_GET_HCA_CAP_OP_MOD_ESW_FLOW_TABLE = 0x8 << 1,
MLX5_SET_HCA_CAP_OP_MOD_ESW = 0x9 << 1,
MLX5_GET_HCA_CAP_OP_MOD_VDPA_EMULATION = 0x13 << 1,
MLX5_GET_HCA_CAP_OP_MOD_CRYPTO = 0x1A << 1,
@@ -1892,6 +1893,7 @@ struct mlx5_ifc_roce_caps_bits {
};
struct mlx5_ifc_ft_fields_support_bits {
+ /* set_action_field_support */
u8 outer_dmac[0x1];
u8 outer_smac[0x1];
u8 outer_ether_type[0x1];
@@ -1919,7 +1921,7 @@ struct mlx5_ifc_ft_fields_support_bits {
u8 outer_gre_key[0x1];
u8 outer_vxlan_vni[0x1];
u8 reserved_at_1a[0x5];
- u8 source_eswitch_port[0x1];
+ u8 source_eswitch_port[0x1]; /* end of DW0 */
u8 inner_dmac[0x1];
u8 inner_smac[0x1];
u8 inner_ether_type[0x1];
@@ -1943,8 +1945,33 @@ struct mlx5_ifc_ft_fields_support_bits {
u8 inner_tcp_sport[0x1];
u8 inner_tcp_dport[0x1];
u8 inner_tcp_flags[0x1];
- u8 reserved_at_37[0x9];
- u8 reserved_at_40[0x40];
+ u8 reserved_at_37[0x9]; /* end of DW1 */
+ u8 reserved_at_40[0x20]; /* end of DW2 */
+ u8 reserved_at_60[0x18];
+ union {
+ struct {
+ u8 metadata_reg_c_7[0x1];
+ u8 metadata_reg_c_6[0x1];
+ u8 metadata_reg_c_5[0x1];
+ u8 metadata_reg_c_4[0x1];
+ u8 metadata_reg_c_3[0x1];
+ u8 metadata_reg_c_2[0x1];
+ u8 metadata_reg_c_1[0x1];
+ u8 metadata_reg_c_0[0x1];
+ };
+ u8 metadata_reg_c_x[0
[v4 07/18] net/mlx5: Add additional glue functions for HWS
Add missing glue support for HWS mlx5dr layer. The new glue
functions are needed for mlx5dv create matcher and action,
which are used as the kernel root table as well as for
capabilities query like device name and ports info.
Signed-off-by: Alex Vesker
---
drivers/common/mlx5/linux/mlx5_glue.c | 121 --
drivers/common/mlx5/linux/mlx5_glue.h | 17
2 files changed, 131 insertions(+), 7 deletions(-)
diff --git a/drivers/common/mlx5/linux/mlx5_glue.c
b/drivers/common/mlx5/linux/mlx5_glue.c
index 450dd6a06a..9f5953fbce 100644
--- a/drivers/common/mlx5/linux/mlx5_glue.c
+++ b/drivers/common/mlx5/linux/mlx5_glue.c
@@ -111,6 +111,12 @@ mlx5_glue_query_device_ex(struct ibv_context *context,
return ibv_query_device_ex(context, input, attr);
}
+static const char *
+mlx5_glue_get_device_name(struct ibv_device *device)
+{
+ return ibv_get_device_name(device);
+}
+
static int
mlx5_glue_query_rt_values_ex(struct ibv_context *context,
struct ibv_values_ex *values)
@@ -620,6 +626,20 @@ mlx5_glue_dv_create_qp(struct ibv_context *context,
#endif
}
+static void *
+__mlx5_glue_dv_create_flow_matcher(struct ibv_context *context,
+ struct mlx5dv_flow_matcher_attr *matcher_attr)
+{
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+ return mlx5dv_create_flow_matcher(context, matcher_attr);
+#else
+ (void)context;
+ (void)matcher_attr;
+ errno = ENOTSUP;
+ return NULL;
+#endif
+}
+
static void *
mlx5_glue_dv_create_flow_matcher(struct ibv_context *context,
struct mlx5dv_flow_matcher_attr *matcher_attr,
@@ -633,7 +653,7 @@ mlx5_glue_dv_create_flow_matcher(struct ibv_context
*context,
matcher_attr->match_mask);
#else
(void)tbl;
- return mlx5dv_create_flow_matcher(context, matcher_attr);
+ return __mlx5_glue_dv_create_flow_matcher(context, matcher_attr);
#endif
#else
(void)context;
@@ -644,6 +664,26 @@ mlx5_glue_dv_create_flow_matcher(struct ibv_context
*context,
#endif
}
+static void *
+__mlx5_glue_dv_create_flow(void *matcher,
+ void *match_value,
+ size_t num_actions,
+ void *actions)
+{
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+ return mlx5dv_create_flow(matcher,
+ match_value,
+ num_actions,
+ (struct mlx5dv_flow_action_attr *)actions);
+#else
+ (void)matcher;
+ (void)match_value;
+ (void)num_actions;
+ (void)actions;
+ return NULL;
+#endif
+}
+
static void *
mlx5_glue_dv_create_flow(void *matcher,
void *match_value,
@@ -663,8 +703,8 @@ mlx5_glue_dv_create_flow(void *matcher,
for (i = 0; i < num_actions; i++)
actions_attr[i] =
*((struct mlx5dv_flow_action_attr *)(actions[i]));
- return mlx5dv_create_flow(matcher, match_value,
- num_actions, actions_attr);
+ return __mlx5_glue_dv_create_flow(matcher, match_value,
+ num_actions, actions_attr);
#endif
#else
(void)matcher;
@@ -735,6 +775,26 @@ mlx5_glue_dv_create_flow_action_dest_devx_tir(void *tir)
#endif
}
+static void *
+__mlx5_glue_dv_create_flow_action_modify_header
+ (struct ibv_context *ctx,
+size_t actions_sz,
+uint64_t actions[],
+enum mlx5dv_flow_table_type ft_type)
+{
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+ return mlx5dv_create_flow_action_modify_header
+ (ctx, actions_sz, actions, ft_type);
+#else
+ (void)ctx;
+ (void)ft_type;
+ (void)actions_sz;
+ (void)actions;
+ errno = ENOTSUP;
+ return NULL;
+#endif
+}
+
static void *
mlx5_glue_dv_create_flow_action_modify_header
(struct ibv_context *ctx,
@@ -758,7 +818,7 @@ mlx5_glue_dv_create_flow_action_modify_header
if (!action)
return NULL;
action->type = MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION;
- action->action = mlx5dv_create_flow_action_modify_header
+ action->action = __mlx5_glue_dv_create_flow_action_modify_header
(ctx, actions_sz, actions, ft_type);
return action;
#endif
@@ -774,6 +834,27 @@ mlx5_glue_dv_create_flow_action_modify_header
#endif
}
+static void *
+__mlx5_glue_dv_create_flow_action_packet_reformat
+ (struct ibv_context *ctx,
+size_t data_sz, void *data,
+enum mlx5dv_flow_action_packet_reformat_type reformat_type,
+enum mlx5dv_flow_table_type ft_type)
+{
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+ return mlx5dv_create_flow_action_
[v4 08/18] net/mlx5/hws: Add HWS command layer
From: Erez Shitrit
The command layer is used to communicate with the FW,
query capabilities and allocate FW resources needed for HWS.
Signed-off-by: Erez Shitrit
Signed-off-by: Alex Vesker
---
drivers/common/mlx5/mlx5_prm.h| 607 ++-
drivers/net/mlx5/hws/mlx5dr_cmd.c | 948 ++
drivers/net/mlx5/hws/mlx5dr_cmd.h | 230
3 files changed, 1775 insertions(+), 10 deletions(-)
create mode 100644 drivers/net/mlx5/hws/mlx5dr_cmd.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_cmd.h
diff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h
index ca4763f53d..371942ae50 100644
--- a/drivers/common/mlx5/mlx5_prm.h
+++ b/drivers/common/mlx5/mlx5_prm.h
@@ -289,6 +289,8 @@
/* The alignment needed for CQ buffer. */
#define MLX5_CQE_BUF_ALIGNMENT rte_mem_page_size()
+#define MAX_ACTIONS_DATA_IN_HEADER_MODIFY 512
+
/* Completion mode. */
enum mlx5_completion_mode {
MLX5_COMP_ONLY_ERR = 0x0,
@@ -677,6 +679,10 @@ enum {
MLX5_MODIFICATION_TYPE_SET = 0x1,
MLX5_MODIFICATION_TYPE_ADD = 0x2,
MLX5_MODIFICATION_TYPE_COPY = 0x3,
+ MLX5_MODIFICATION_TYPE_INSERT = 0x4,
+ MLX5_MODIFICATION_TYPE_REMOVE = 0x5,
+ MLX5_MODIFICATION_TYPE_NOP = 0x6,
+ MLX5_MODIFICATION_TYPE_REMOVE_WORDS = 0x7,
};
/* The field of packet to be modified. */
@@ -,6 +1117,10 @@ enum {
MLX5_CMD_OP_QUERY_TIS = 0x915,
MLX5_CMD_OP_CREATE_RQT = 0x916,
MLX5_CMD_OP_MODIFY_RQT = 0x917,
+ MLX5_CMD_OP_CREATE_FLOW_TABLE = 0x930,
+ MLX5_CMD_OP_CREATE_FLOW_GROUP = 0x933,
+ MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY = 0x936,
+ MLX5_CMD_OP_MODIFY_FLOW_TABLE = 0x93c,
MLX5_CMD_OP_ALLOC_FLOW_COUNTER = 0x939,
MLX5_CMD_OP_QUERY_FLOW_COUNTER = 0x93b,
MLX5_CMD_OP_CREATE_GENERAL_OBJECT = 0xa00,
@@ -1299,6 +1309,7 @@ enum {
MLX5_SET_HCA_CAP_OP_MOD_ESW = 0x9 << 1,
MLX5_GET_HCA_CAP_OP_MOD_VDPA_EMULATION = 0x13 << 1,
MLX5_GET_HCA_CAP_OP_MOD_CRYPTO = 0x1A << 1,
+ MLX5_GET_HCA_CAP_OP_MOD_WQE_BASED_FLOW_TABLE = 0x1B << 1,
MLX5_GET_HCA_CAP_OP_MOD_PARSE_GRAPH_NODE_CAP = 0x1C << 1,
MLX5_GET_HCA_CAP_OP_MOD_GENERAL_DEVICE_2 = 0x20 << 1,
};
@@ -1317,6 +1328,14 @@ enum {
(1ULL << MLX5_GENERAL_OBJ_TYPE_GENEVE_TLV_OPT)
#define MLX5_GENERAL_OBJ_TYPES_CAP_CONN_TRACK_OFFLOAD \
(1ULL << MLX5_GENERAL_OBJ_TYPE_CONN_TRACK_OFFLOAD)
+#define MLX5_GENERAL_OBJ_TYPES_CAP_RTC \
+ (1ULL << MLX5_GENERAL_OBJ_TYPE_RTC)
+#define MLX5_GENERAL_OBJ_TYPES_CAP_STC \
+ (1ULL << MLX5_GENERAL_OBJ_TYPE_STC)
+#define MLX5_GENERAL_OBJ_TYPES_CAP_STE \
+ (1ULL << MLX5_GENERAL_OBJ_TYPE_STE)
+#define MLX5_GENERAL_OBJ_TYPES_CAP_DEFINER \
+ (1ULL << MLX5_GENERAL_OBJ_TYPE_DEFINER)
#define MLX5_GENERAL_OBJ_TYPES_CAP_DEK \
(1ULL << MLX5_GENERAL_OBJ_TYPE_DEK)
#define MLX5_GENERAL_OBJ_TYPES_CAP_IMPORT_KEK \
@@ -1373,6 +1392,11 @@ enum {
#define MLX5_HCA_FLEX_VXLAN_GPE_ENABLED (1UL << 7)
#define MLX5_HCA_FLEX_ICMP_ENABLED (1UL << 8)
#define MLX5_HCA_FLEX_ICMPV6_ENABLED (1UL << 9)
+#define MLX5_HCA_FLEX_GTPU_ENABLED (1UL << 11)
+#define MLX5_HCA_FLEX_GTPU_DW_2_ENABLED (1UL << 16)
+#define MLX5_HCA_FLEX_GTPU_FIRST_EXT_DW_0_ENABLED (1UL << 17)
+#define MLX5_HCA_FLEX_GTPU_DW_0_ENABLED (1UL << 18)
+#define MLX5_HCA_FLEX_GTPU_TEID_ENABLED (1UL << 19)
/* The device steering logic format. */
#define MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 0x0
@@ -1505,7 +1529,8 @@ struct mlx5_ifc_cmd_hca_cap_bits {
u8 wol_u[0x1];
u8 wol_p[0x1];
u8 stat_rate_support[0x10];
- u8 reserved_at_1f0[0xc];
+ u8 reserved_at_1ef[0xb];
+ u8 wqe_based_flow_table_update_cap[0x1];
u8 cqe_version[0x4];
u8 compact_address_vector[0x1];
u8 striding_rq[0x1];
@@ -1681,7 +1706,9 @@ struct mlx5_ifc_cmd_hca_cap_bits {
u8 cqe_compression[0x1];
u8 cqe_compression_timeout[0x10];
u8 cqe_compression_max_num[0x10];
- u8 reserved_at_5e0[0x10];
+ u8 reserved_at_5e0[0x8];
+ u8 flex_parser_id_gtpu_dw_0[0x4];
+ u8 reserved_at_5ec[0x4];
u8 tag_matching[0x1];
u8 rndv_offload_rc[0x1];
u8 rndv_offload_dc[0x1];
@@ -1691,17 +1718,38 @@ struct mlx5_ifc_cmd_hca_cap_bits {
u8 affiliate_nic_vport_criteria[0x8];
u8 native_port_num[0x8];
u8 num_vhca_ports[0x8];
- u8 reserved_at_618[0x6];
+ u8 flex_parser_id_gtpu_teid[0x4];
+ u8 reserved_at_61c[0x2];
u8 sw_owner_id[0x1];
u8 reserved_at_61f[0x6C];
u8 wait_on_data[0x1];
u8 wait_on_time[0x1];
- u8 reserved_at_68d[0xBB];
+ u8 reserved_at_68d[0x37];
+ u8 flex_parser_id_geneve_opt_0[0x
[v4 09/18] net/mlx5/hws: Add HWS pool and buddy
From: Erez Shitrit
HWS needs to manage different types of device memory in
an efficient and quick way. For this, memory pools are
being used.
Signed-off-by: Erez Shitrit
Signed-off-by: Alex Vesker
---
drivers/net/mlx5/hws/mlx5dr_buddy.c | 201 +
drivers/net/mlx5/hws/mlx5dr_buddy.h | 22 +
drivers/net/mlx5/hws/mlx5dr_pool.c | 672
drivers/net/mlx5/hws/mlx5dr_pool.h | 152 +++
4 files changed, 1047 insertions(+)
create mode 100644 drivers/net/mlx5/hws/mlx5dr_buddy.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_buddy.h
create mode 100644 drivers/net/mlx5/hws/mlx5dr_pool.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_pool.h
diff --git a/drivers/net/mlx5/hws/mlx5dr_buddy.c
b/drivers/net/mlx5/hws/mlx5dr_buddy.c
new file mode 100644
index 00..9dba95f0b1
--- /dev/null
+++ b/drivers/net/mlx5/hws/mlx5dr_buddy.c
@@ -0,0 +1,201 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2022 NVIDIA Corporation & Affiliates
+ */
+
+#include
+#include
+#include "mlx5dr_internal.h"
+#include "mlx5dr_buddy.h"
+
+static struct rte_bitmap *bitmap_alloc0(int s)
+{
+ struct rte_bitmap *bitmap;
+ uint32_t bmp_size;
+ void *mem;
+
+ bmp_size = rte_bitmap_get_memory_footprint(s);
+ mem = rte_zmalloc("create_bmap", bmp_size, RTE_CACHE_LINE_SIZE);
+ if (!mem) {
+ DR_LOG(ERR, "No mem for bitmap");
+ rte_errno = ENOMEM;
+ return NULL;
+ }
+
+ bitmap = rte_bitmap_init(s, mem, bmp_size);
+ if (!bitmap) {
+ DR_LOG(ERR, "%s Failed to initialize bitmap", __func__);
+ rte_errno = EINVAL;
+ goto err_mem_alloc;
+ }
+
+ return bitmap;
+
+err_mem_alloc:
+ rte_free(mem);
+ return NULL;
+}
+
+static void bitmap_set_bit(struct rte_bitmap *bmp, uint32_t pos)
+{
+ rte_bitmap_set(bmp, pos);
+}
+
+static void bitmap_clear_bit(struct rte_bitmap *bmp, uint32_t pos)
+{
+ rte_bitmap_clear(bmp, pos);
+}
+
+static bool bitmap_test_bit(struct rte_bitmap *bmp, unsigned long n)
+{
+ return !!rte_bitmap_get(bmp, n);
+}
+
+static unsigned long bitmap_ffs(struct rte_bitmap *bmap,
+ unsigned long n, unsigned long m)
+{
+ uint64_t out_slab = 0;
+ uint32_t pos = 0; /* Compilation warn */
+
+ __rte_bitmap_scan_init(bmap);
+ if (!rte_bitmap_scan(bmap, &pos, &out_slab)) {
+ DR_LOG(ERR, "Failed to get slab from bitmap.");
+ return m;
+ }
+ pos = pos + __builtin_ctzll(out_slab);
+
+ if (pos < n) {
+ DR_LOG(ERR, "Unexpected bit (%d < %"PRIx64") from bitmap", pos,
n);
+ return m;
+ }
+ return pos;
+}
+
+static unsigned long mlx5dr_buddy_find_first_bit(struct rte_bitmap *addr,
+uint32_t size)
+{
+ return bitmap_ffs(addr, 0, size);
+}
+
+static int mlx5dr_buddy_init(struct mlx5dr_buddy_mem *buddy, uint32_t
max_order)
+{
+ int i, s;
+
+ buddy->max_order = max_order;
+
+ buddy->bits = simple_calloc(buddy->max_order + 1, sizeof(long *));
+ if (!buddy->bits) {
+ rte_errno = ENOMEM;
+ return -1;
+ }
+
+ buddy->num_free = simple_calloc(buddy->max_order + 1,
sizeof(*buddy->num_free));
+ if (!buddy->num_free) {
+ rte_errno = ENOMEM;
+ goto err_out_free_bits;
+ }
+
+ for (i = 0; i <= (int)buddy->max_order; ++i) {
+ s = 1 << (buddy->max_order - i);
+ buddy->bits[i] = bitmap_alloc0(s);
+ if (!buddy->bits[i])
+ goto err_out_free_num_free;
+ }
+
+ bitmap_set_bit(buddy->bits[buddy->max_order], 0);
+
+ buddy->num_free[buddy->max_order] = 1;
+
+ return 0;
+
+err_out_free_num_free:
+ for (i = 0; i <= (int)buddy->max_order; ++i)
+ rte_free(buddy->bits[i]);
+
+ simple_free(buddy->num_free);
+
+err_out_free_bits:
+ simple_free(buddy->bits);
+ return -1;
+}
+
+struct mlx5dr_buddy_mem *mlx5dr_buddy_create(uint32_t max_order)
+{
+ struct mlx5dr_buddy_mem *buddy;
+
+ buddy = simple_calloc(1, sizeof(*buddy));
+ if (!buddy) {
+ rte_errno = ENOMEM;
+ return NULL;
+ }
+
+ if (mlx5dr_buddy_init(buddy, max_order))
+ goto free_buddy;
+
+ return buddy;
+
+free_buddy:
+ simple_free(buddy);
+ return NULL;
+}
+
+void mlx5dr_buddy_cleanup(struct mlx5dr_buddy_mem *buddy)
+{
+ int i;
+
+ for (i = 0; i <= (int)buddy->max_order; ++i) {
+ rte_free(buddy->bits[i]);
+ }
+
+ simple_free(buddy->num_free);
+ simple_free(bud
[v4 10/18] net/mlx5/hws: Add HWS send layer
HWS configures flows to the HW using a QP, each WQE has
the details of the flow we want to offload. The send layer
allocates the resources needed to send the request to the HW
as well as managing the queues, getting completions and
handling failures.
Signed-off-by: Mark Bloch
Signed-off-by: Alex Vesker
---
drivers/net/mlx5/hws/mlx5dr_send.c | 844 +
drivers/net/mlx5/hws/mlx5dr_send.h | 275 ++
2 files changed, 1119 insertions(+)
create mode 100644 drivers/net/mlx5/hws/mlx5dr_send.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_send.h
diff --git a/drivers/net/mlx5/hws/mlx5dr_send.c
b/drivers/net/mlx5/hws/mlx5dr_send.c
new file mode 100644
index 00..26904a9040
--- /dev/null
+++ b/drivers/net/mlx5/hws/mlx5dr_send.c
@@ -0,0 +1,844 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2022 NVIDIA Corporation & Affiliates
+ */
+
+#include "mlx5dr_internal.h"
+
+struct mlx5dr_send_ring_dep_wqe *
+mlx5dr_send_add_new_dep_wqe(struct mlx5dr_send_engine *queue)
+{
+ struct mlx5dr_send_ring_sq *send_sq = &queue->send_ring->send_sq;
+ unsigned int idx = send_sq->head_dep_idx++ & (queue->num_entries - 1);
+
+ memset(&send_sq->dep_wqe[idx].wqe_data.tag, 0, MLX5DR_MATCH_TAG_SZ);
+
+ return &send_sq->dep_wqe[idx];
+}
+
+void mlx5dr_send_abort_new_dep_wqe(struct mlx5dr_send_engine *queue)
+{
+ queue->send_ring->send_sq.head_dep_idx--;
+}
+
+void mlx5dr_send_all_dep_wqe(struct mlx5dr_send_engine *queue)
+{
+ struct mlx5dr_send_ring_sq *send_sq = &queue->send_ring->send_sq;
+ struct mlx5dr_send_ste_attr ste_attr = {0};
+ struct mlx5dr_send_ring_dep_wqe *dep_wqe;
+
+ ste_attr.send_attr.opmod = MLX5DR_WQE_GTA_OPMOD_STE;
+ ste_attr.send_attr.opcode = MLX5DR_WQE_OPCODE_TBL_ACCESS;
+ ste_attr.send_attr.len = MLX5DR_WQE_SZ_GTA_CTRL +
MLX5DR_WQE_SZ_GTA_DATA;
+ ste_attr.gta_opcode = MLX5DR_WQE_GTA_OP_ACTIVATE;
+
+ /* Fence first from previous depend WQEs */
+ ste_attr.send_attr.fence = 1;
+
+ while (send_sq->head_dep_idx != send_sq->tail_dep_idx) {
+ dep_wqe = &send_sq->dep_wqe[send_sq->tail_dep_idx++ &
(queue->num_entries - 1)];
+
+ /* Notify HW on the last WQE */
+ ste_attr.send_attr.notify_hw = (send_sq->tail_dep_idx ==
send_sq->head_dep_idx);
+ ste_attr.send_attr.user_data = dep_wqe->user_data;
+ ste_attr.send_attr.rule = dep_wqe->rule;
+
+ ste_attr.rtc_0 = dep_wqe->rtc_0;
+ ste_attr.rtc_1 = dep_wqe->rtc_1;
+ ste_attr.retry_rtc_0 = dep_wqe->retry_rtc_0;
+ ste_attr.retry_rtc_1 = dep_wqe->retry_rtc_1;
+ ste_attr.used_id_rtc_0 = &dep_wqe->rule->rtc_0;
+ ste_attr.used_id_rtc_1 = &dep_wqe->rule->rtc_1;
+ ste_attr.wqe_ctrl = &dep_wqe->wqe_ctrl;
+ ste_attr.wqe_data = &dep_wqe->wqe_data;
+
+ mlx5dr_send_ste(queue, &ste_attr);
+
+ /* Fencing is done only on the first WQE */
+ ste_attr.send_attr.fence = 0;
+ }
+}
+
+struct mlx5dr_send_engine_post_ctrl
+mlx5dr_send_engine_post_start(struct mlx5dr_send_engine *queue)
+{
+ struct mlx5dr_send_engine_post_ctrl ctrl;
+
+ ctrl.queue = queue;
+ /* Currently only one send ring is supported */
+ ctrl.send_ring = &queue->send_ring[0];
+ ctrl.num_wqebbs = 0;
+
+ return ctrl;
+}
+
+void mlx5dr_send_engine_post_req_wqe(struct mlx5dr_send_engine_post_ctrl *ctrl,
+char **buf, size_t *len)
+{
+ struct mlx5dr_send_ring_sq *send_sq = &ctrl->send_ring->send_sq;
+ unsigned int idx;
+
+ idx = (send_sq->cur_post + ctrl->num_wqebbs) & send_sq->buf_mask;
+
+ *buf = send_sq->buf + (idx << MLX5_SEND_WQE_SHIFT);
+ *len = MLX5_SEND_WQE_BB;
+
+ if (!ctrl->num_wqebbs) {
+ *buf += sizeof(struct mlx5dr_wqe_ctrl_seg);
+ *len -= sizeof(struct mlx5dr_wqe_ctrl_seg);
+ }
+
+ ctrl->num_wqebbs++;
+}
+
+static void mlx5dr_send_engine_post_ring(struct mlx5dr_send_ring_sq *sq,
+struct mlx5dv_devx_uar *uar,
+struct mlx5dr_wqe_ctrl_seg *wqe_ctrl)
+{
+ rte_compiler_barrier();
+ sq->db[MLX5_SND_DBR] = rte_cpu_to_be_32(sq->cur_post);
+
+ rte_wmb();
+ mlx5dr_uar_write64_relaxed(*((uint64_t *)wqe_ctrl), uar->reg_addr);
+ rte_wmb();
+}
+
+static void
+mlx5dr_send_wqe_set_tag(struct mlx5dr_wqe_gta_data_seg_ste *wqe_data,
+ struct mlx5dr_rule_match_tag *tag,
+ bool is_jumbo)
+{
+ if (is_jumbo) {
+ /* Clear previous possibly dirty control */
+
[v4 12/18] net/mlx5/hws: Add HWS context object
Context is the first mlx5dr object created, all sub object:
table, matcher, rule, action are created using the context.
The context holds the capabilities and send queues used for
configuring the offloads to the HW.
Signed-off-by: Alex Vesker
---
drivers/net/mlx5/hws/mlx5dr_context.c | 223 ++
drivers/net/mlx5/hws/mlx5dr_context.h | 40 +
2 files changed, 263 insertions(+)
create mode 100644 drivers/net/mlx5/hws/mlx5dr_context.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_context.h
diff --git a/drivers/net/mlx5/hws/mlx5dr_context.c
b/drivers/net/mlx5/hws/mlx5dr_context.c
new file mode 100644
index 00..ae86694a51
--- /dev/null
+++ b/drivers/net/mlx5/hws/mlx5dr_context.c
@@ -0,0 +1,223 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2022 NVIDIA Corporation & Affiliates
+ */
+
+#include "mlx5dr_internal.h"
+
+static int mlx5dr_context_pools_init(struct mlx5dr_context *ctx)
+{
+ struct mlx5dr_pool_attr pool_attr = {0};
+ uint8_t max_log_sz;
+ int i;
+
+ if (mlx5dr_pat_init_pattern_cache(&ctx->pattern_cache))
+ return rte_errno;
+
+ /* Create an STC pool per FT type */
+ pool_attr.pool_type = MLX5DR_POOL_TYPE_STC;
+ pool_attr.flags = MLX5DR_POOL_FLAGS_FOR_STC_POOL;
+ max_log_sz = RTE_MIN(MLX5DR_POOL_STC_LOG_SZ,
ctx->caps->stc_alloc_log_max);
+ pool_attr.alloc_log_sz = RTE_MAX(max_log_sz,
ctx->caps->stc_alloc_log_gran);
+
+ for (i = 0; i < MLX5DR_TABLE_TYPE_MAX; i++) {
+ pool_attr.table_type = i;
+ ctx->stc_pool[i] = mlx5dr_pool_create(ctx, &pool_attr);
+ if (!ctx->stc_pool[i]) {
+ DR_LOG(ERR, "Failed to allocate STC pool [%d]", i);
+ goto free_stc_pools;
+ }
+ }
+
+ return 0;
+
+free_stc_pools:
+ for (i = 0; i < MLX5DR_TABLE_TYPE_MAX; i++)
+ if (ctx->stc_pool[i])
+ mlx5dr_pool_destroy(ctx->stc_pool[i]);
+
+ mlx5dr_pat_uninit_pattern_cache(ctx->pattern_cache);
+
+ return rte_errno;
+}
+
+static void mlx5dr_context_pools_uninit(struct mlx5dr_context *ctx)
+{
+ int i;
+
+ mlx5dr_pat_uninit_pattern_cache(ctx->pattern_cache);
+
+ for (i = 0; i < MLX5DR_TABLE_TYPE_MAX; i++) {
+ if (ctx->stc_pool[i])
+ mlx5dr_pool_destroy(ctx->stc_pool[i]);
+ }
+}
+
+static int mlx5dr_context_init_pd(struct mlx5dr_context *ctx,
+ struct ibv_pd *pd)
+{
+ struct mlx5dv_pd mlx5_pd = {0};
+ struct mlx5dv_obj obj;
+ int ret;
+
+ if (pd) {
+ ctx->pd = pd;
+ } else {
+ ctx->pd = mlx5_glue->alloc_pd(ctx->ibv_ctx);
+ if (!ctx->pd) {
+ DR_LOG(ERR, "Failed to allocate PD");
+ rte_errno = errno;
+ return rte_errno;
+ }
+ ctx->flags |= MLX5DR_CONTEXT_FLAG_PRIVATE_PD;
+ }
+
+ obj.pd.in = ctx->pd;
+ obj.pd.out = &mlx5_pd;
+
+ ret = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_PD);
+ if (ret)
+ goto free_private_pd;
+
+ ctx->pd_num = mlx5_pd.pdn;
+
+ return 0;
+
+free_private_pd:
+ if (ctx->flags & MLX5DR_CONTEXT_FLAG_PRIVATE_PD)
+ mlx5_glue->dealloc_pd(ctx->pd);
+
+ return ret;
+}
+
+static int mlx5dr_context_uninit_pd(struct mlx5dr_context *ctx)
+{
+ if (ctx->flags & MLX5DR_CONTEXT_FLAG_PRIVATE_PD)
+ return mlx5_glue->dealloc_pd(ctx->pd);
+
+ return 0;
+}
+
+static void mlx5dr_context_check_hws_supp(struct mlx5dr_context *ctx)
+{
+ struct mlx5dr_cmd_query_caps *caps = ctx->caps;
+
+ /* HWS not supported on device / FW */
+ if (!caps->wqe_based_update) {
+ DR_LOG(INFO, "Required HWS WQE based insertion cap not
supported");
+ return;
+ }
+
+ /* Current solution requires all rules to set reparse bit */
+ if ((!caps->nic_ft.reparse || !caps->fdb_ft.reparse) ||
+ !IS_BIT_SET(caps->rtc_reparse_mode, MLX5_IFC_RTC_REPARSE_ALWAYS)) {
+ DR_LOG(INFO, "Required HWS reparse cap not supported");
+ return;
+ }
+
+ /* FW/HW must support 8DW STE */
+ if (!IS_BIT_SET(caps->ste_format, MLX5_IFC_RTC_STE_FORMAT_8DW)) {
+ DR_LOG(INFO, "Required HWS STE format not supported");
+ return;
+ }
+
+ /* Adding rules by hash and by offset are requirements */
+ if (!IS_BIT_SET(caps->rtc_index_mode,
MLX5_IFC_RTC_STE_UPDATE_MODE_BY_HASH) ||
+ !IS_BIT_SET(caps->rtc_index_mode,
MLX5_IFC_RTC_STE_UPDATE_MODE_BY_OFFSET)) {
+
[v4 11/18] net/mlx5/hws: Add HWS definer layer
Definers are HW objects that are used for matching, rte items
are translated to definers, each definer holds the fields and
bit-masks used for HW flow matching. The definer layer is used
for finding the most efficient definer for each set of items.
In addition to definer creation we also calculate the field
copy (fc) array used for efficient items to WQE conversion.
Signed-off-by: Mark Bloch
Signed-off-by: Alex Vesker
---
drivers/net/mlx5/hws/mlx5dr_definer.c | 1968 +
drivers/net/mlx5/hws/mlx5dr_definer.h | 585
2 files changed, 2553 insertions(+)
create mode 100644 drivers/net/mlx5/hws/mlx5dr_definer.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_definer.h
diff --git a/drivers/net/mlx5/hws/mlx5dr_definer.c
b/drivers/net/mlx5/hws/mlx5dr_definer.c
new file mode 100644
index 00..6b98eb8c96
--- /dev/null
+++ b/drivers/net/mlx5/hws/mlx5dr_definer.c
@@ -0,0 +1,1968 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2022 NVIDIA Corporation & Affiliates
+ */
+
+#include "mlx5dr_internal.h"
+
+#define GTP_PDU_SC 0x85
+#define BAD_PORT 0xBAD
+#define ETH_TYPE_IPV4_VXLAN0x0800
+#define ETH_TYPE_IPV6_VXLAN0x86DD
+#define ETH_VXLAN_DEFAULT_PORT 4789
+
+#define STE_NO_VLAN0x0
+#define STE_SVLAN 0x1
+#define STE_CVLAN 0x2
+#define STE_IPV4 0x1
+#define STE_IPV6 0x2
+#define STE_TCP0x1
+#define STE_UDP0x2
+#define STE_ICMP 0x3
+
+/* Setter function based on bit offset and mask, for 32bit DW*/
+#define _DR_SET_32(p, v, byte_off, bit_off, mask) \
+ do { \
+ u32 _v = v; \
+ *((rte_be32_t *)(p) + ((byte_off) / 4)) = \
+ rte_cpu_to_be_32((rte_be_to_cpu_32(*((u32 *)(p) + \
+ ((byte_off) / 4))) & \
+ (~((mask) << (bit_off | \
+(((_v) & (mask)) << \
+ (bit_off))); \
+ } while (0)
+
+/* Setter function based on bit offset and mask */
+#define DR_SET(p, v, byte_off, bit_off, mask) \
+ do { \
+ if (unlikely((bit_off) < 0)) { \
+ u32 _bit_off = -1 * (bit_off); \
+ u32 second_dw_mask = (mask) & ((1 << _bit_off) - 1); \
+ _DR_SET_32(p, (v) >> _bit_off, byte_off, 0, (mask) >>
_bit_off); \
+ _DR_SET_32(p, (v) & second_dw_mask, (byte_off) +
DW_SIZE, \
+ (bit_off) % BITS_IN_DW, second_dw_mask); \
+ } else { \
+ _DR_SET_32(p, v, byte_off, (bit_off), (mask)); \
+ } \
+ } while (0)
+
+/* Setter function based on byte offset to directly set FULL BE32 value */
+#define DR_SET_BE32(p, v, byte_off, bit_off, mask) \
+ (*((rte_be32_t *)((uint8_t *)(p) + (byte_off))) = (v))
+
+/* Setter function based on byte offset to directly set FULL BE32 value from
ptr */
+#define DR_SET_BE32P(p, v_ptr, byte_off, bit_off, mask) \
+ memcpy((uint8_t *)(p) + (byte_off), v_ptr, 4)
+
+/* Setter function based on byte offset to directly set FULL BE16 value */
+#define DR_SET_BE16(p, v, byte_off, bit_off, mask) \
+ (*((rte_be16_t *)((uint8_t *)(p) + (byte_off))) = (v))
+
+/* Setter function based on byte offset to directly set FULL BE16 value from
ptr */
+#define DR_SET_BE16P(p, v_ptr, byte_off, bit_off, mask) \
+ memcpy((uint8_t *)(p) + (byte_off), v_ptr, 2)
+
+#define DR_CALC_FNAME(field, inner) \
+ ((inner) ? MLX5DR_DEFINER_FNAME_##field##_I : \
+ MLX5DR_DEFINER_FNAME_##field##_O)
+
+#define DR_CALC_SET_HDR(fc, hdr, field) \
+ do { \
+ (fc)->bit_mask = __mlx5_mask(definer_hl, hdr.field); \
+ (fc)->bit_off = __mlx5_dw_bit_off(definer_hl, hdr.field); \
+ (fc)->byte_off = MLX5_BYTE_OFF(definer_hl, hdr.field); \
+ } while (0)
+
+/* Helper to calculate data used by DR_SET */
+#define DR_CALC_SET(fc, hdr, field, is_inner) \
+ do { \
+ if (is_inner) { \
+ DR_CALC_SET_HDR(fc, hdr##_inner, field); \
+ } else { \
+ DR_CALC_SET_HDR(fc, hdr##_outer, field); \
+ } \
+ } while (0)
+
+ #define DR_GET(typ, p, fld) \
+ ((rte_be_to_cpu_32(*((const rte_be32_t *)(p) + \
+ __mlx5_dw_off(typ, fld))) >> __mlx5_dw_bit_off(typ, fld)) & \
+ __mlx5_mask(typ, fld))
+
+struct mlx5dr_definer_sel_ctrl {
+ uint8_t allowed_full_dw; /* Full DW selectors cover all offsets */
+ uint8_t allowed_lim_dw; /* Limited DW selectors cover offset < 64 */
+ uint8_t allowed_bytes; /* Bytes selectors, up to offset 255 */
+ uint8_t used_full_dw;
+ uint8_t used_lim_dw;
+ uint8_t used_bytes;
+ uint8_t full_dw_selector[DW_SELECTORS];
[v4 13/18] net/mlx5/hws: Add HWS table object
HWS table resides under the context object, each context can
have multiple tables with different steering types RX/TX/FDB.
The table is not only a logical object but it is also represented
in the HW, packets can be steered to the table and from there
to other tables.
Signed-off-by: Erez Shitrit
Signed-off-by: Alex Vesker
---
drivers/net/mlx5/hws/mlx5dr_table.c | 248
drivers/net/mlx5/hws/mlx5dr_table.h | 44 +
2 files changed, 292 insertions(+)
create mode 100644 drivers/net/mlx5/hws/mlx5dr_table.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_table.h
diff --git a/drivers/net/mlx5/hws/mlx5dr_table.c
b/drivers/net/mlx5/hws/mlx5dr_table.c
new file mode 100644
index 00..d3f77e4780
--- /dev/null
+++ b/drivers/net/mlx5/hws/mlx5dr_table.c
@@ -0,0 +1,248 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2022 NVIDIA Corporation & Affiliates
+ */
+
+#include "mlx5dr_internal.h"
+
+static void mlx5dr_table_init_next_ft_attr(struct mlx5dr_table *tbl,
+ struct mlx5dr_cmd_ft_create_attr
*ft_attr)
+{
+ ft_attr->type = tbl->fw_ft_type;
+ if (tbl->type == MLX5DR_TABLE_TYPE_FDB)
+ ft_attr->level = tbl->ctx->caps->fdb_ft.max_level - 1;
+ else
+ ft_attr->level = tbl->ctx->caps->nic_ft.max_level - 1;
+ ft_attr->rtc_valid = true;
+}
+
+/* Call this under ctx->ctrl_lock */
+static int
+mlx5dr_table_up_default_fdb_miss_tbl(struct mlx5dr_table *tbl)
+{
+ struct mlx5dr_cmd_ft_create_attr ft_attr = {0};
+ struct mlx5dr_cmd_forward_tbl *default_miss;
+ struct mlx5dr_context *ctx = tbl->ctx;
+ uint8_t tbl_type = tbl->type;
+ uint32_t vport;
+
+ if (tbl->type != MLX5DR_TABLE_TYPE_FDB)
+ return 0;
+
+ if (ctx->common_res[tbl_type].default_miss) {
+ ctx->common_res[tbl_type].default_miss->refcount++;
+ return 0;
+ }
+
+ ft_attr.type = tbl->fw_ft_type;
+ ft_attr.level = tbl->ctx->caps->fdb_ft.max_level; /* The last level */
+ ft_attr.rtc_valid = false;
+
+ assert(ctx->caps->eswitch_manager);
+ vport = ctx->caps->eswitch_manager_vport_number;
+
+ default_miss = mlx5dr_cmd_miss_ft_create(ctx->ibv_ctx, &ft_attr, vport);
+ if (!default_miss) {
+ DR_LOG(ERR, "Failed to default miss table type: 0x%x",
tbl_type);
+ return rte_errno;
+ }
+
+ ctx->common_res[tbl_type].default_miss = default_miss;
+ ctx->common_res[tbl_type].default_miss->refcount++;
+ return 0;
+}
+
+/* Called under pthread_spin_lock(&ctx->ctrl_lock) */
+static void mlx5dr_table_down_default_fdb_miss_tbl(struct mlx5dr_table *tbl)
+{
+ struct mlx5dr_cmd_forward_tbl *default_miss;
+ struct mlx5dr_context *ctx = tbl->ctx;
+ uint8_t tbl_type = tbl->type;
+
+ if (tbl->type != MLX5DR_TABLE_TYPE_FDB)
+ return;
+
+ default_miss = ctx->common_res[tbl_type].default_miss;
+ if (--default_miss->refcount)
+ return;
+
+ mlx5dr_cmd_miss_ft_destroy(default_miss);
+
+ simple_free(default_miss);
+ ctx->common_res[tbl_type].default_miss = NULL;
+}
+
+static int
+mlx5dr_table_connect_to_default_miss_tbl(struct mlx5dr_table *tbl,
+struct mlx5dr_devx_obj *ft)
+{
+ struct mlx5dr_cmd_ft_modify_attr ft_attr = {0};
+ int ret;
+
+ assert(tbl->type == MLX5DR_TABLE_TYPE_FDB);
+
+ mlx5dr_cmd_set_attr_connect_miss_tbl(tbl->ctx,
+tbl->fw_ft_type,
+tbl->type,
+&ft_attr);
+
+ /* Connect to next */
+ ret = mlx5dr_cmd_flow_table_modify(ft, &ft_attr);
+ if (ret) {
+ DR_LOG(ERR, "Failed to connect FT to default FDB FT");
+ return errno;
+ }
+
+ return 0;
+}
+
+struct mlx5dr_devx_obj *
+mlx5dr_table_create_default_ft(struct mlx5dr_table *tbl)
+{
+ struct mlx5dr_cmd_ft_create_attr ft_attr = {0};
+ struct mlx5dr_devx_obj *ft_obj;
+ int ret;
+
+ mlx5dr_table_init_next_ft_attr(tbl, &ft_attr);
+
+ ft_obj = mlx5dr_cmd_flow_table_create(tbl->ctx->ibv_ctx, &ft_attr);
+ if (ft_obj && tbl->type == MLX5DR_TABLE_TYPE_FDB) {
+ /* Take/create ref over the default miss */
+ ret = mlx5dr_table_up_default_fdb_miss_tbl(tbl);
+ if (ret) {
+ DR_LOG(ERR, "Failed to get default fdb miss");
+ goto free_ft_obj;
+ }
+ ret = mlx5dr_table_connect_to_default_miss_tbl(tbl, ft_obj);
+
[v4 14/18] net/mlx5/hws: Add HWS matcher object
HWS matcher resides under the table object, each table can
have multiple chained matcher with different attributes. Each
matcher represents a combination of match and action templates.
Each matcher can contain multiple configurations based on the
templates. Packets are steered from the table to the matcher
and from there to other objects. The matcher allows efficent
HW packet field matching and action execution based on the
configuration done to it.
Signed-off-by: Alex Vesker
---
drivers/net/mlx5/hws/mlx5dr_matcher.c | 922 ++
drivers/net/mlx5/hws/mlx5dr_matcher.h | 76 +++
2 files changed, 998 insertions(+)
create mode 100644 drivers/net/mlx5/hws/mlx5dr_matcher.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_matcher.h
diff --git a/drivers/net/mlx5/hws/mlx5dr_matcher.c
b/drivers/net/mlx5/hws/mlx5dr_matcher.c
new file mode 100644
index 00..835a3908eb
--- /dev/null
+++ b/drivers/net/mlx5/hws/mlx5dr_matcher.c
@@ -0,0 +1,922 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2022 NVIDIA Corporation & Affiliates
+ */
+
+#include "mlx5dr_internal.h"
+
+static bool mlx5dr_matcher_requires_col_tbl(uint8_t log_num_of_rules)
+{
+ /* Collision table concatenation is done only for large rule tables */
+ return log_num_of_rules > MLX5DR_MATCHER_ASSURED_RULES_TH;
+}
+
+static uint8_t mlx5dr_matcher_rules_to_tbl_depth(uint8_t log_num_of_rules)
+{
+ if (mlx5dr_matcher_requires_col_tbl(log_num_of_rules))
+ return MLX5DR_MATCHER_ASSURED_MAIN_TBL_DEPTH;
+
+ /* For small rule tables we use a single deep table to assure insertion
*/
+ return RTE_MIN(log_num_of_rules, MLX5DR_MATCHER_ASSURED_COL_TBL_DEPTH);
+}
+
+static int mlx5dr_matcher_create_end_ft(struct mlx5dr_matcher *matcher)
+{
+ struct mlx5dr_table *tbl = matcher->tbl;
+
+ matcher->end_ft = mlx5dr_table_create_default_ft(tbl);
+ if (!matcher->end_ft) {
+ DR_LOG(ERR, "Failed to create matcher end flow table");
+ return rte_errno;
+ }
+ return 0;
+}
+
+static void mlx5dr_matcher_destroy_end_ft(struct mlx5dr_matcher *matcher)
+{
+ mlx5dr_table_destroy_default_ft(matcher->tbl, matcher->end_ft);
+}
+
+static int mlx5dr_matcher_connect(struct mlx5dr_matcher *matcher)
+{
+ struct mlx5dr_cmd_ft_modify_attr ft_attr = {0};
+ struct mlx5dr_table *tbl = matcher->tbl;
+ struct mlx5dr_matcher *prev = NULL;
+ struct mlx5dr_matcher *next = NULL;
+ struct mlx5dr_matcher *tmp_matcher;
+ struct mlx5dr_devx_obj *ft;
+ int ret;
+
+ /* Find location in matcher list */
+ if (LIST_EMPTY(&tbl->head)) {
+ LIST_INSERT_HEAD(&tbl->head, matcher, next);
+ goto connect;
+ }
+
+ LIST_FOREACH(tmp_matcher, &tbl->head, next) {
+ if (tmp_matcher->attr.priority > matcher->attr.priority) {
+ next = tmp_matcher;
+ break;
+ }
+ prev = tmp_matcher;
+ }
+
+ if (next)
+ LIST_INSERT_BEFORE(next, matcher, next);
+ else
+ LIST_INSERT_AFTER(prev, matcher, next);
+
+connect:
+ ft_attr.modify_fs = MLX5_IFC_MODIFY_FLOW_TABLE_RTC_ID;
+ ft_attr.type = tbl->fw_ft_type;
+
+ /* Connect to next */
+ if (next) {
+ if (next->match_ste.rtc_0)
+ ft_attr.rtc_id_0 = next->match_ste.rtc_0->id;
+ if (next->match_ste.rtc_1)
+ ft_attr.rtc_id_1 = next->match_ste.rtc_1->id;
+
+ ret = mlx5dr_cmd_flow_table_modify(matcher->end_ft, &ft_attr);
+ if (ret) {
+ DR_LOG(ERR, "Failed to connect new matcher to next
RTC");
+ goto remove_from_list;
+ }
+ }
+
+ /* Connect to previous */
+ ft = prev ? prev->end_ft : tbl->ft;
+
+ if (matcher->match_ste.rtc_0)
+ ft_attr.rtc_id_0 = matcher->match_ste.rtc_0->id;
+ if (matcher->match_ste.rtc_1)
+ ft_attr.rtc_id_1 = matcher->match_ste.rtc_1->id;
+
+ ret = mlx5dr_cmd_flow_table_modify(ft, &ft_attr);
+ if (ret) {
+ DR_LOG(ERR, "Failed to connect new matcher to previous FT");
+ goto remove_from_list;
+ }
+
+ return 0;
+
+remove_from_list:
+ LIST_REMOVE(matcher, next);
+ return ret;
+}
+
+static int mlx5dr_matcher_disconnect(struct mlx5dr_matcher *matcher)
+{
+ struct mlx5dr_cmd_ft_modify_attr ft_attr = {0};
+ struct mlx5dr_table *tbl = matcher->tbl;
+ struct mlx5dr_matcher *tmp_matcher;
+ struct mlx5dr_devx_obj *prev_ft;
+ struct mlx5dr_matcher *next;
+ int ret;
+
+ prev_ft = matcher->tbl->ft;
+ LIST_FOR
[v4 15/18] net/mlx5/hws: Add HWS rule object
HWS rule objects reside under the matcher, each rule holds
the configuration for the packet fields to match on and the
set of actions to execute over the packet that has the requested
fields. Rules can be created asynchronously in parallel over multiple
queues to different matchers. Each rule is configured to the HW.
Signed-off-by: Erez Shitrit
Signed-off-by: Alex Vesker
---
drivers/net/mlx5/hws/mlx5dr_rule.c | 528 +
drivers/net/mlx5/hws/mlx5dr_rule.h | 50 +++
2 files changed, 578 insertions(+)
create mode 100644 drivers/net/mlx5/hws/mlx5dr_rule.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_rule.h
diff --git a/drivers/net/mlx5/hws/mlx5dr_rule.c
b/drivers/net/mlx5/hws/mlx5dr_rule.c
new file mode 100644
index 00..b27318e6d4
--- /dev/null
+++ b/drivers/net/mlx5/hws/mlx5dr_rule.c
@@ -0,0 +1,528 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2022 NVIDIA Corporation & Affiliates
+ */
+
+#include "mlx5dr_internal.h"
+
+static void mlx5dr_rule_skip(struct mlx5dr_matcher *matcher,
+const struct rte_flow_item *items,
+bool *skip_rx, bool *skip_tx)
+{
+ struct mlx5dr_match_template *mt = matcher->mt[0];
+ const struct flow_hw_port_info *vport;
+ const struct rte_flow_item_ethdev *v;
+
+ /* Flow_src is the 1st priority */
+ if (matcher->attr.optimize_flow_src) {
+ *skip_tx = matcher->attr.optimize_flow_src ==
MLX5DR_MATCHER_FLOW_SRC_WIRE;
+ *skip_rx = matcher->attr.optimize_flow_src ==
MLX5DR_MATCHER_FLOW_SRC_VPORT;
+ return;
+ }
+
+ /* By default FDB rules are added to both RX and TX */
+ *skip_rx = false;
+ *skip_tx = false;
+
+ if (mt->item_flags & MLX5_FLOW_ITEM_REPRESENTED_PORT) {
+ v = items[mt->vport_item_id].spec;
+ vport = flow_hw_conv_port_id(v->port_id);
+ if (unlikely(!vport)) {
+ DR_LOG(ERR, "Fail to map port ID %d, ignoring",
v->port_id);
+ return;
+ }
+
+ if (!vport->is_wire)
+ /* Match vport ID is not WIRE -> Skip RX */
+ *skip_rx = true;
+ else
+ /* Match vport ID is WIRE -> Skip TX */
+ *skip_tx = true;
+ }
+}
+
+static void mlx5dr_rule_init_dep_wqe(struct mlx5dr_send_ring_dep_wqe *dep_wqe,
+struct mlx5dr_rule *rule,
+const struct rte_flow_item *items,
+void *user_data)
+{
+ struct mlx5dr_matcher *matcher = rule->matcher;
+ struct mlx5dr_table *tbl = matcher->tbl;
+ bool skip_rx, skip_tx;
+
+ dep_wqe->rule = rule;
+ dep_wqe->user_data = user_data;
+
+ switch (tbl->type) {
+ case MLX5DR_TABLE_TYPE_NIC_RX:
+ case MLX5DR_TABLE_TYPE_NIC_TX:
+ dep_wqe->rtc_0 = matcher->match_ste.rtc_0->id;
+ dep_wqe->retry_rtc_0 = matcher->col_matcher ?
+
matcher->col_matcher->match_ste.rtc_0->id : 0;
+ dep_wqe->rtc_1 = 0;
+ dep_wqe->retry_rtc_1 = 0;
+ break;
+
+ case MLX5DR_TABLE_TYPE_FDB:
+ mlx5dr_rule_skip(matcher, items, &skip_rx, &skip_tx);
+
+ if (!skip_rx) {
+ dep_wqe->rtc_0 = matcher->match_ste.rtc_0->id;
+ dep_wqe->retry_rtc_0 = matcher->col_matcher ?
+
matcher->col_matcher->match_ste.rtc_0->id : 0;
+ } else {
+ dep_wqe->rtc_0 = 0;
+ dep_wqe->retry_rtc_0 = 0;
+ }
+
+ if (!skip_tx) {
+ dep_wqe->rtc_1 = matcher->match_ste.rtc_1->id;
+ dep_wqe->retry_rtc_1 = matcher->col_matcher ?
+
matcher->col_matcher->match_ste.rtc_1->id : 0;
+ } else {
+ dep_wqe->rtc_1 = 0;
+ dep_wqe->retry_rtc_1 = 0;
+ }
+
+ break;
+
+ default:
+ assert(false);
+ break;
+ }
+}
+
+static void mlx5dr_rule_gen_comp(struct mlx5dr_send_engine *queue,
+struct mlx5dr_rule *rule,
+bool err,
+void *user_data,
+enum mlx5dr_rule_status rule_status_on_succ)
+{
+ enum rte_flow_op_status comp_status;
+
+ if (!err) {
+ comp_status = RTE_FLOW_OP_SUCCESS;
+ rule->status = rule_status_on_succ;
[v4 16/18] net/mlx5/hws: Add HWS action object
Action objects are used for executing different HW actions
over packets. Each action contains the HW resources and parameters
needed for action use over the HW when creating a rule.
Signed-off-by: Erez Shitrit
Signed-off-by: Alex Vesker
---
drivers/net/mlx5/hws/mlx5dr_action.c | +
drivers/net/mlx5/hws/mlx5dr_action.h | 253 +++
drivers/net/mlx5/hws/mlx5dr_pat_arg.c | 511 ++
drivers/net/mlx5/hws/mlx5dr_pat_arg.h | 83 +
4 files changed, 3069 insertions(+)
create mode 100644 drivers/net/mlx5/hws/mlx5dr_action.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_action.h
create mode 100644 drivers/net/mlx5/hws/mlx5dr_pat_arg.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_pat_arg.h
diff --git a/drivers/net/mlx5/hws/mlx5dr_action.c
b/drivers/net/mlx5/hws/mlx5dr_action.c
new file mode 100644
index 00..61b3a58bf2
--- /dev/null
+++ b/drivers/net/mlx5/hws/mlx5dr_action.c
@@ -0,0 +1, @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2022 NVIDIA Corporation & Affiliates
+ */
+
+#include "mlx5dr_internal.h"
+
+#define WIRE_PORT 0x
+
+#define MLX5DR_ACTION_METER_INIT_COLOR_OFFSET 1
+
+/* This is the maximum allowed action order for each table type:
+ * TX: POP_VLAN, CTR, ASO_METER, AS_CT, PUSH_VLAN, MODIFY, ENCAP, Term
+ * RX: TAG, DECAP, POP_VLAN, CTR, ASO_METER, ASO_CT, PUSH_VLAN, MODIFY,
+ * ENCAP, Term
+ * FDB: DECAP, POP_VLAN, CTR, ASO_METER, ASO_CT, PUSH_VLAN, MODIFY,
+ * ENCAP, Term
+ */
+static const uint32_t
action_order_arr[MLX5DR_TABLE_TYPE_MAX][MLX5DR_ACTION_TYP_MAX] = {
+ [MLX5DR_TABLE_TYPE_NIC_RX] = {
+ BIT(MLX5DR_ACTION_TYP_TAG),
+ BIT(MLX5DR_ACTION_TYP_TNL_L2_TO_L2) |
+ BIT(MLX5DR_ACTION_TYP_TNL_L3_TO_L2),
+ BIT(MLX5DR_ACTION_TYP_POP_VLAN),
+ BIT(MLX5DR_ACTION_TYP_POP_VLAN),
+ BIT(MLX5DR_ACTION_TYP_CTR),
+ BIT(MLX5DR_ACTION_TYP_ASO_METER),
+ BIT(MLX5DR_ACTION_TYP_ASO_CT),
+ BIT(MLX5DR_ACTION_TYP_PUSH_VLAN),
+ BIT(MLX5DR_ACTION_TYP_PUSH_VLAN),
+ BIT(MLX5DR_ACTION_TYP_MODIFY_HDR),
+ BIT(MLX5DR_ACTION_TYP_L2_TO_TNL_L2) |
+ BIT(MLX5DR_ACTION_TYP_L2_TO_TNL_L3),
+ BIT(MLX5DR_ACTION_TYP_FT) |
+ BIT(MLX5DR_ACTION_TYP_MISS) |
+ BIT(MLX5DR_ACTION_TYP_TIR) |
+ BIT(MLX5DR_ACTION_TYP_DROP),
+ BIT(MLX5DR_ACTION_TYP_LAST),
+ },
+ [MLX5DR_TABLE_TYPE_NIC_TX] = {
+ BIT(MLX5DR_ACTION_TYP_POP_VLAN),
+ BIT(MLX5DR_ACTION_TYP_POP_VLAN),
+ BIT(MLX5DR_ACTION_TYP_CTR),
+ BIT(MLX5DR_ACTION_TYP_ASO_METER),
+ BIT(MLX5DR_ACTION_TYP_ASO_CT),
+ BIT(MLX5DR_ACTION_TYP_PUSH_VLAN),
+ BIT(MLX5DR_ACTION_TYP_PUSH_VLAN),
+ BIT(MLX5DR_ACTION_TYP_MODIFY_HDR),
+ BIT(MLX5DR_ACTION_TYP_L2_TO_TNL_L2) |
+ BIT(MLX5DR_ACTION_TYP_L2_TO_TNL_L3),
+ BIT(MLX5DR_ACTION_TYP_FT) |
+ BIT(MLX5DR_ACTION_TYP_MISS) |
+ BIT(MLX5DR_ACTION_TYP_DROP),
+ BIT(MLX5DR_ACTION_TYP_LAST),
+ },
+ [MLX5DR_TABLE_TYPE_FDB] = {
+ BIT(MLX5DR_ACTION_TYP_TNL_L2_TO_L2) |
+ BIT(MLX5DR_ACTION_TYP_TNL_L3_TO_L2),
+ BIT(MLX5DR_ACTION_TYP_POP_VLAN),
+ BIT(MLX5DR_ACTION_TYP_POP_VLAN),
+ BIT(MLX5DR_ACTION_TYP_CTR),
+ BIT(MLX5DR_ACTION_TYP_ASO_METER),
+ BIT(MLX5DR_ACTION_TYP_ASO_CT),
+ BIT(MLX5DR_ACTION_TYP_PUSH_VLAN),
+ BIT(MLX5DR_ACTION_TYP_PUSH_VLAN),
+ BIT(MLX5DR_ACTION_TYP_MODIFY_HDR),
+ BIT(MLX5DR_ACTION_TYP_L2_TO_TNL_L2) |
+ BIT(MLX5DR_ACTION_TYP_L2_TO_TNL_L3),
+ BIT(MLX5DR_ACTION_TYP_FT) |
+ BIT(MLX5DR_ACTION_TYP_MISS) |
+ BIT(MLX5DR_ACTION_TYP_VPORT) |
+ BIT(MLX5DR_ACTION_TYP_DROP),
+ BIT(MLX5DR_ACTION_TYP_LAST),
+ },
+};
+
+static int mlx5dr_action_get_shared_stc_nic(struct mlx5dr_context *ctx,
+ enum mlx5dr_context_shared_stc_type
stc_type,
+ uint8_t tbl_type)
+{
+ struct mlx5dr_cmd_stc_modify_attr stc_attr = {0};
+ struct mlx5dr_action_shared_stc *shared_stc;
+ int ret;
+
+ pthread_spin_lock(&ctx->ctrl_lock);
+ if (ctx->common_res[tbl_type].shared_stc[stc_type]) {
+
rte_atomic32_add(&ctx->common_res[tbl_type].shared_stc[stc_type]->refcount, 1);
+ pthread_spin_unlock(&ctx->ctrl_lock);
+ return 0;
+ }
+
+ shared_stc = simple_calloc(1, sizeof(*shared_stc));
+ if (!shared_stc) {
+ DR_LOG(ERR, "Failed to allocate memory for sha
[v4 18/18] net/mlx5/hws: Enable HWS
Replace stub implenation of HWS with mlx5dr code.
Signed-off-by: Alex Vesker
---
doc/guides/nics/mlx5.rst | 5 +-
doc/guides/rel_notes/release_22_11.rst | 4 +
drivers/common/mlx5/linux/meson.build| 2 +
drivers/net/mlx5/hws/meson.build | 18 +
drivers/net/mlx5/{mlx5_dr.h => hws/mlx5dr.h} | 209 --
drivers/net/mlx5/hws/mlx5dr_internal.h | 93 +
drivers/net/mlx5/meson.build | 7 +-
drivers/net/mlx5/mlx5.c | 6 +-
drivers/net/mlx5/mlx5.h | 7 +-
drivers/net/mlx5/mlx5_devx.c | 2 +-
drivers/net/mlx5/mlx5_dr.c | 383 ---
drivers/net/mlx5/mlx5_flow.c | 2 +
drivers/net/mlx5/mlx5_flow.h | 11 +-
drivers/net/mlx5/mlx5_flow_hw.c | 10 +-
14 files changed, 327 insertions(+), 432 deletions(-)
create mode 100644 drivers/net/mlx5/hws/meson.build
rename drivers/net/mlx5/{mlx5_dr.h => hws/mlx5dr.h} (66%)
create mode 100644 drivers/net/mlx5/hws/mlx5dr_internal.h
delete mode 100644 drivers/net/mlx5/mlx5_dr.c
diff --git a/doc/guides/nics/mlx5.rst b/doc/guides/nics/mlx5.rst
index bb436892a0..303eb17714 100644
--- a/doc/guides/nics/mlx5.rst
+++ b/doc/guides/nics/mlx5.rst
@@ -539,7 +539,10 @@ Limitations
- WQE based high scaling and safer flow insertion/destruction.
- Set ``dv_flow_en`` to 2 in order to enable HW steering.
-- Async queue-based ``rte_flow_q`` APIs supported only.
+- Async queue-based ``rte_flow_async`` APIs supported only.
+- NIC ConnectX-5 and before are not supported.
+- Partial match with item template is not supported.
+- IPv6 5-tuple matching is not supported.
- Match on GRE header supports the following fields:
diff --git a/doc/guides/rel_notes/release_22_11.rst
b/doc/guides/rel_notes/release_22_11.rst
index a3700bbb34..eed7acc838 100644
--- a/doc/guides/rel_notes/release_22_11.rst
+++ b/doc/guides/rel_notes/release_22_11.rst
@@ -237,6 +237,10 @@ New Features
sysfs entries to adjust the minimum and maximum uncore frequency values,
which works on Linux with Intel hardware only.
+* **Updated Nvidia mlx5 driver.**
+
+ * Added fully support for queue based async HW steering to the PMD.
+
* **Rewritten pmdinfo script.**
The ``dpdk-pmdinfo.py`` script was rewritten to produce valid JSON only.
diff --git a/drivers/common/mlx5/linux/meson.build
b/drivers/common/mlx5/linux/meson.build
index e77b46d157..3db7a1770f 100644
--- a/drivers/common/mlx5/linux/meson.build
+++ b/drivers/common/mlx5/linux/meson.build
@@ -229,6 +229,8 @@ foreach arg:has_member_args
endforeach
configure_file(output : 'mlx5_autoconf.h', configuration : config)
+MLX5_HAVE_IBV_FLOW_DV_SUPPORT=config.get('HAVE_IBV_FLOW_DV_SUPPORT')
+
# Build Glue Library
if dlopen_ibverbs
dlopen_name = 'mlx5_glue'
diff --git a/drivers/net/mlx5/hws/meson.build b/drivers/net/mlx5/hws/meson.build
new file mode 100644
index 00..f94798dd2d
--- /dev/null
+++ b/drivers/net/mlx5/hws/meson.build
@@ -0,0 +1,18 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2022 NVIDIA Corporation & Affiliates
+
+includes += include_directories('.')
+sources += files(
+ 'mlx5dr_context.c',
+ 'mlx5dr_table.c',
+ 'mlx5dr_matcher.c',
+ 'mlx5dr_rule.c',
+ 'mlx5dr_action.c',
+ 'mlx5dr_buddy.c',
+ 'mlx5dr_pool.c',
+ 'mlx5dr_cmd.c',
+ 'mlx5dr_send.c',
+ 'mlx5dr_definer.c',
+ 'mlx5dr_debug.c',
+ 'mlx5dr_pat_arg.c',
+)
diff --git a/drivers/net/mlx5/mlx5_dr.h b/drivers/net/mlx5/hws/mlx5dr.h
similarity index 66%
rename from drivers/net/mlx5/mlx5_dr.h
rename to drivers/net/mlx5/hws/mlx5dr.h
index d0b2c15652..664dadbcde 100644
--- a/drivers/net/mlx5/mlx5_dr.h
+++ b/drivers/net/mlx5/hws/mlx5dr.h
@@ -1,9 +1,9 @@
/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright (c) 2021 NVIDIA CORPORATION. All rights reserved.
+ * Copyright (c) 2022 NVIDIA Corporation & Affiliates
*/
-#ifndef MLX5_DR_H_
-#define MLX5_DR_H_
+#ifndef MLX5DR_H_
+#define MLX5DR_H_
#include
@@ -11,6 +11,7 @@ struct mlx5dr_context;
struct mlx5dr_table;
struct mlx5dr_matcher;
struct mlx5dr_rule;
+struct ibv_context;
enum mlx5dr_table_type {
MLX5DR_TABLE_TYPE_NIC_RX,
@@ -26,6 +27,27 @@ enum mlx5dr_matcher_resource_mode {
MLX5DR_MATCHER_RESOURCE_MODE_HTABLE,
};
+enum mlx5dr_action_type {
+ MLX5DR_ACTION_TYP_LAST,
+ MLX5DR_ACTION_TYP_TNL_L2_TO_L2,
+ MLX5DR_ACTION_TYP_L2_TO_TNL_L2,
+ MLX5DR_ACTION_TYP_TNL_L3_TO_L2,
+ MLX5DR_ACTION_TYP_L2_TO_TNL_L3,
+ MLX5DR_ACTION_TYP_DROP,
+ MLX5DR_ACTION_TYP_TIR,
+ MLX5DR_ACTION_TYP_FT,
+ MLX5DR_ACTION_TYP_CTR,
+ MLX5DR_ACTION_TYP_TAG,
[v4 17/18] net/mlx5/hws: Add HWS debug layer
From: Hamdan Igbaria
The debug layer is used to generate a debug CSV file
containing details of the context, table, matcher, rules
and other useful debug information.
Signed-off-by: Hamdan Igbaria
Signed-off-by: Alex Vesker
---
drivers/net/mlx5/hws/mlx5dr_debug.c | 462
drivers/net/mlx5/hws/mlx5dr_debug.h | 28 ++
2 files changed, 490 insertions(+)
create mode 100644 drivers/net/mlx5/hws/mlx5dr_debug.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_debug.h
diff --git a/drivers/net/mlx5/hws/mlx5dr_debug.c
b/drivers/net/mlx5/hws/mlx5dr_debug.c
new file mode 100644
index 00..890a761c48
--- /dev/null
+++ b/drivers/net/mlx5/hws/mlx5dr_debug.c
@@ -0,0 +1,462 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2022 NVIDIA Corporation & Affiliates
+ */
+
+#include "mlx5dr_internal.h"
+
+const char *mlx5dr_debug_action_type_str[] = {
+ [MLX5DR_ACTION_TYP_LAST] = "LAST",
+ [MLX5DR_ACTION_TYP_TNL_L2_TO_L2] = "TNL_L2_TO_L2",
+ [MLX5DR_ACTION_TYP_L2_TO_TNL_L2] = "L2_TO_TNL_L2",
+ [MLX5DR_ACTION_TYP_TNL_L3_TO_L2] = "TNL_L3_TO_L2",
+ [MLX5DR_ACTION_TYP_L2_TO_TNL_L3] = "L2_TO_TNL_L3",
+ [MLX5DR_ACTION_TYP_DROP] = "DROP",
+ [MLX5DR_ACTION_TYP_TIR] = "TIR",
+ [MLX5DR_ACTION_TYP_FT] = "FT",
+ [MLX5DR_ACTION_TYP_CTR] = "CTR",
+ [MLX5DR_ACTION_TYP_TAG] = "TAG",
+ [MLX5DR_ACTION_TYP_MODIFY_HDR] = "MODIFY_HDR",
+ [MLX5DR_ACTION_TYP_VPORT] = "VPORT",
+ [MLX5DR_ACTION_TYP_MISS] = "DEFAULT_MISS",
+ [MLX5DR_ACTION_TYP_POP_VLAN] = "POP_VLAN",
+ [MLX5DR_ACTION_TYP_PUSH_VLAN] = "PUSH_VLAN",
+ [MLX5DR_ACTION_TYP_ASO_METER] = "ASO_METER",
+ [MLX5DR_ACTION_TYP_ASO_CT] = "ASO_CT",
+};
+
+static_assert(ARRAY_SIZE(mlx5dr_debug_action_type_str) ==
MLX5DR_ACTION_TYP_MAX,
+ "Missing mlx5dr_debug_action_type_str");
+
+const char *mlx5dr_debug_action_type_to_str(enum mlx5dr_action_type
action_type)
+{
+ return mlx5dr_debug_action_type_str[action_type];
+}
+
+static int
+mlx5dr_debug_dump_matcher_template_definer(FILE *f,
+ struct mlx5dr_match_template *mt)
+{
+ struct mlx5dr_definer *definer = mt->definer;
+ int i, ret;
+
+ ret = fprintf(f, "%d,0x%" PRIx64 ",0x%" PRIx64 ",%d,%d,",
+ MLX5DR_DEBUG_RES_TYPE_MATCHER_TEMPLATE_DEFINER,
+ (uint64_t)(uintptr_t)definer,
+ (uint64_t)(uintptr_t)mt,
+ definer->obj->id,
+ definer->type);
+ if (ret < 0) {
+ rte_errno = EINVAL;
+ return rte_errno;
+ }
+
+ for (i = 0; i < DW_SELECTORS; i++) {
+ ret = fprintf(f, "0x%x%s", definer->dw_selector[i],
+ (i == DW_SELECTORS - 1) ? "," : "-");
+ if (ret < 0) {
+ rte_errno = EINVAL;
+ return rte_errno;
+ }
+ }
+
+ for (i = 0; i < BYTE_SELECTORS; i++) {
+ ret = fprintf(f, "0x%x%s", definer->byte_selector[i],
+ (i == BYTE_SELECTORS - 1) ? "," : "-");
+ if (ret < 0) {
+ rte_errno = EINVAL;
+ return rte_errno;
+ }
+ }
+
+ for (i = 0; i < MLX5DR_JUMBO_TAG_SZ; i++) {
+ ret = fprintf(f, "%02x", definer->mask.jumbo[i]);
+ if (ret < 0) {
+ rte_errno = EINVAL;
+ return rte_errno;
+ }
+ }
+
+ ret = fprintf(f, "\n");
+ if (ret < 0) {
+ rte_errno = EINVAL;
+ return rte_errno;
+ }
+
+ return 0;
+}
+
+static int
+mlx5dr_debug_dump_matcher_match_template(FILE *f, struct mlx5dr_matcher
*matcher)
+{
+ bool is_root = matcher->tbl->level == MLX5DR_ROOT_LEVEL;
+ int i, ret;
+
+ for (i = 0; i < matcher->num_of_mt; i++) {
+ struct mlx5dr_match_template *mt = matcher->mt[i];
+
+ ret = fprintf(f, "%d,0x%" PRIx64 ",0x%" PRIx64 ",%d,%d\n",
+ MLX5DR_DEBUG_RES_TYPE_MATCHER_MATCH_TEMPLATE,
+ (uint64_t)(uintptr_t)mt,
+ (uint64_t)(uintptr_t)matcher,
+ is_root ? 0 : mt->fc_sz,
+ mt->flags);
+ if (ret < 0) {
+ rte_errno = EINVAL;
+ return rte_errno;
+ }
+
+ i
[v5 00/18] net/mlx5: Add HW steering low level support
Mellanox ConnetX devices supports packet matching, packet modification and
redirection. These functionalities are also referred to as flow-steering.
To configure a steering rule, the rule is written to the device owned
memory, this memory is accessed and cached by the device when processing
a packet.
The highlight of this patchset is supporting HW Steering (HWS) which
is the new technology supported in new ConnectX devices, HWS allows
configuring steering rules directly to the HW using special HW queues
with minimal CPU effort.
This patchset is the internal low layer implementation for HWS used by
the mlx5 PMD. The mlx5dr (direct rule) is layer that bridges between the
PMD and the HW by configuring the HW offloads based on the PMD logic
v2:
Fix check patch and cosmetic changes
v3:
-Fix unsupported items
-Fix compilation with mlx5dv dependency
v4:
-Fix compile on Windows
v5:
-Fix compile on old rdma-core or no rdma core
Alex Vesker (8):
net/mlx5: Add additional glue functions for HWS
net/mlx5/hws: Add HWS send layer
net/mlx5/hws: Add HWS definer layer
net/mlx5/hws: Add HWS context object
net/mlx5/hws: Add HWS table object
net/mlx5/hws: Add HWS matcher object
net/mlx5/hws: Add HWS rule object
net/mlx5/hws: Enable HWS
Bing Zhao (2):
common/mlx5: query set capability of registers
net/mlx5: provide the available tag registers
Dariusz Sosnowski (1):
net/mlx5: add port to metadata conversion
Erez Shitrit (3):
net/mlx5/hws: Add HWS command layer
net/mlx5/hws: Add HWS pool and buddy
net/mlx5/hws: Add HWS action object
Hamdan Igbaria (1):
net/mlx5/hws: Add HWS debug layer
Suanming Mou (3):
net/mlx5: split flow item translation
net/mlx5: split flow item matcher and value translation
net/mlx5: add hardware steering item translation function
doc/guides/nics/mlx5.rst |5 +-
doc/guides/rel_notes/release_22_11.rst |4 +
drivers/common/mlx5/linux/meson.build|5 +
drivers/common/mlx5/linux/mlx5_glue.c| 121 +-
drivers/common/mlx5/linux/mlx5_glue.h| 17 +
drivers/common/mlx5/mlx5_devx_cmds.c | 30 +
drivers/common/mlx5/mlx5_devx_cmds.h |2 +
drivers/common/mlx5/mlx5_prm.h | 652 -
drivers/net/mlx5/hws/meson.build | 18 +
drivers/net/mlx5/{mlx5_dr.h => hws/mlx5dr.h} | 209 +-
drivers/net/mlx5/hws/mlx5dr_action.c | 2237 +++
drivers/net/mlx5/hws/mlx5dr_action.h | 253 ++
drivers/net/mlx5/hws/mlx5dr_buddy.c | 201 ++
drivers/net/mlx5/hws/mlx5dr_buddy.h | 22 +
drivers/net/mlx5/hws/mlx5dr_cmd.c| 948 +++
drivers/net/mlx5/hws/mlx5dr_cmd.h| 230 ++
drivers/net/mlx5/hws/mlx5dr_context.c| 223 ++
drivers/net/mlx5/hws/mlx5dr_context.h| 40 +
drivers/net/mlx5/hws/mlx5dr_debug.c | 462 +++
drivers/net/mlx5/hws/mlx5dr_debug.h | 28 +
drivers/net/mlx5/hws/mlx5dr_definer.c| 1968 +
drivers/net/mlx5/hws/mlx5dr_definer.h| 585
drivers/net/mlx5/hws/mlx5dr_internal.h | 93 +
drivers/net/mlx5/hws/mlx5dr_matcher.c| 919 ++
drivers/net/mlx5/hws/mlx5dr_matcher.h| 76 +
drivers/net/mlx5/hws/mlx5dr_pat_arg.c| 511
drivers/net/mlx5/hws/mlx5dr_pat_arg.h| 83 +
drivers/net/mlx5/hws/mlx5dr_pool.c | 672 +
drivers/net/mlx5/hws/mlx5dr_pool.h | 152 +
drivers/net/mlx5/hws/mlx5dr_rule.c | 528
drivers/net/mlx5/hws/mlx5dr_rule.h | 50 +
drivers/net/mlx5/hws/mlx5dr_send.c | 844 ++
drivers/net/mlx5/hws/mlx5dr_send.h | 275 ++
drivers/net/mlx5/hws/mlx5dr_table.c | 248 ++
drivers/net/mlx5/hws/mlx5dr_table.h | 44 +
drivers/net/mlx5/linux/mlx5_os.c | 12 +-
drivers/net/mlx5/meson.build |7 +-
drivers/net/mlx5/mlx5.c |9 +-
drivers/net/mlx5/mlx5.h |8 +-
drivers/net/mlx5/mlx5_defs.h |2 +
drivers/net/mlx5/mlx5_devx.c |2 +-
drivers/net/mlx5/mlx5_dr.c | 383 ---
drivers/net/mlx5/mlx5_flow.c | 29 +-
drivers/net/mlx5/mlx5_flow.h | 174 +-
drivers/net/mlx5/mlx5_flow_dv.c | 2631 +-
drivers/net/mlx5/mlx5_flow_hw.c | 115 +-
46 files changed, 14401 insertions(+), 1726 deletions(-)
create mode 100644 drivers/net/mlx5/hws/meson.build
rename drivers/net/mlx5/{mlx5_dr.h => hws/mlx5dr.h} (66%)
create mode 100644 drivers/net/mlx5/hws/mlx5dr_action.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_action.h
create mode 100644 drivers/net/mlx5/hws/mlx5dr_buddy.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_buddy.h
create mode 100644 drivers/net/mlx5/hws/mlx5dr_cmd.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_cmd.h
create mode 100644 drivers/net/mlx5/hws/mlx5dr_con
[v5 01/18] net/mlx5: split flow item translation
From: Suanming Mou
In order to share the item translation code with hardware steering
mode, this commit splits flow item translation code to a dedicate
function.
Signed-off-by: Suanming Mou
---
drivers/net/mlx5/mlx5_flow_dv.c | 1915 ---
1 file changed, 979 insertions(+), 936 deletions(-)
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 91f287af5c..70a3279e2f 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -13029,8 +13029,7 @@ flow_dv_translate_create_conntrack(struct rte_eth_dev
*dev,
}
/**
- * Fill the flow with DV spec, lock free
- * (mutex should be acquired by caller).
+ * Translate the flow item to matcher.
*
* @param[in] dev
* Pointer to rte_eth_dev structure.
@@ -13040,8 +13039,8 @@ flow_dv_translate_create_conntrack(struct rte_eth_dev
*dev,
* Pointer to the flow attributes.
* @param[in] items
* Pointer to the list of items.
- * @param[in] actions
- * Pointer to the list of actions.
+ * @param[in] matcher
+ * Pointer to the flow matcher.
* @param[out] error
* Pointer to the error structure.
*
@@ -13049,1041 +13048,1086 @@ flow_dv_translate_create_conntrack(struct
rte_eth_dev *dev,
* 0 on success, a negative errno value otherwise and rte_errno is set.
*/
static int
-flow_dv_translate(struct rte_eth_dev *dev,
- struct mlx5_flow *dev_flow,
- const struct rte_flow_attr *attr,
- const struct rte_flow_item items[],
- const struct rte_flow_action actions[],
- struct rte_flow_error *error)
+flow_dv_translate_items(struct rte_eth_dev *dev,
+ struct mlx5_flow *dev_flow,
+ const struct rte_flow_attr *attr,
+ const struct rte_flow_item items[],
+ struct mlx5_flow_dv_matcher *matcher,
+ struct rte_flow_error *error)
{
struct mlx5_priv *priv = dev->data->dev_private;
- struct mlx5_sh_config *dev_conf = &priv->sh->config;
struct rte_flow *flow = dev_flow->flow;
struct mlx5_flow_handle *handle = dev_flow->handle;
struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
- struct mlx5_flow_rss_desc *rss_desc;
+ struct mlx5_flow_rss_desc *rss_desc = &wks->rss_desc;
uint64_t item_flags = 0;
uint64_t last_item = 0;
- uint64_t action_flags = 0;
- struct mlx5_flow_dv_matcher matcher = {
- .mask = {
- .size = sizeof(matcher.mask.buf),
- },
- };
- int actions_n = 0;
- bool actions_end = false;
- union {
- struct mlx5_flow_dv_modify_hdr_resource res;
- uint8_t len[sizeof(struct mlx5_flow_dv_modify_hdr_resource) +
- sizeof(struct mlx5_modification_cmd) *
- (MLX5_MAX_MODIFY_NUM + 1)];
- } mhdr_dummy;
- struct mlx5_flow_dv_modify_hdr_resource *mhdr_res = &mhdr_dummy.res;
- const struct rte_flow_action_count *count = NULL;
- const struct rte_flow_action_age *non_shared_age = NULL;
- union flow_dv_attr flow_attr = { .attr = 0 };
- uint32_t tag_be;
- union mlx5_flow_tbl_key tbl_key;
- uint32_t modify_action_position = UINT32_MAX;
- void *match_mask = matcher.mask.buf;
+ void *match_mask = matcher->mask.buf;
void *match_value = dev_flow->dv.value.buf;
uint8_t next_protocol = 0xff;
- struct rte_vlan_hdr vlan = { 0 };
- struct mlx5_flow_dv_dest_array_resource mdest_res;
- struct mlx5_flow_dv_sample_resource sample_res;
- void *sample_actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS] = {0};
- const struct rte_flow_action_sample *sample = NULL;
- struct mlx5_flow_sub_actions_list *sample_act;
- uint32_t sample_act_pos = UINT32_MAX;
- uint32_t age_act_pos = UINT32_MAX;
- uint32_t num_of_dest = 0;
- int tmp_actions_n = 0;
- uint32_t table;
- int ret = 0;
- const struct mlx5_flow_tunnel *tunnel = NULL;
- struct flow_grp_info grp_info = {
- .external = !!dev_flow->external,
- .transfer = !!attr->transfer,
- .fdb_def_rule = !!priv->fdb_def_rule,
- .skip_scale = dev_flow->skip_scale &
- (1 << MLX5_SCALE_FLOW_GROUP_BIT),
- .std_tbl_fix = true,
- };
+ uint16_t priority = 0;
const struct rte_flow_item *integrity_items[2] = {NULL, NULL};
const struct rte_flow_item *tunnel_item = NULL;
const struct rte_flow_item *gre_item = NULL;
+ int ret = 0;
- if (!wks)
- return rte_flow_error_set(error, ENOMEM,
- RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
- NULL,
- "failed to
[v5 03/18] net/mlx5: add hardware steering item translation function
From: Suanming Mou
As hardware steering root table flows still work under FW steering
mode. This commit provides shared item tranlsation code for hardware
steering root table flows.
Signed-off-by: Suanming Mou
---
drivers/net/mlx5/mlx5_flow.c| 10 +--
drivers/net/mlx5/mlx5_flow.h| 52 -
drivers/net/mlx5/mlx5_flow_dv.c | 134
3 files changed, 155 insertions(+), 41 deletions(-)
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 6fb1d53fc5..742dbd6358 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -7108,7 +7108,7 @@ mlx5_flow_create_devx_sq_miss_flow(struct rte_eth_dev
*dev, uint32_t txq)
struct rte_flow_item_port_id port_spec = {
.id = MLX5_PORT_ESW_MGR,
};
- struct mlx5_rte_flow_item_tx_queue txq_spec = {
+ struct mlx5_rte_flow_item_sq txq_spec = {
.queue = txq,
};
struct rte_flow_item pattern[] = {
@@ -7118,7 +7118,7 @@ mlx5_flow_create_devx_sq_miss_flow(struct rte_eth_dev
*dev, uint32_t txq)
},
{
.type = (enum rte_flow_item_type)
- MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE,
+ MLX5_RTE_FLOW_ITEM_TYPE_SQ,
.spec = &txq_spec,
},
{
@@ -7504,16 +7504,16 @@ mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev,
.egress = 1,
.priority = 0,
};
- struct mlx5_rte_flow_item_tx_queue queue_spec = {
+ struct mlx5_rte_flow_item_sq queue_spec = {
.queue = queue,
};
- struct mlx5_rte_flow_item_tx_queue queue_mask = {
+ struct mlx5_rte_flow_item_sq queue_mask = {
.queue = UINT32_MAX,
};
struct rte_flow_item items[] = {
{
.type = (enum rte_flow_item_type)
- MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE,
+ MLX5_RTE_FLOW_ITEM_TYPE_SQ,
.spec = &queue_spec,
.last = NULL,
.mask = &queue_mask,
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index 2ebb8496f2..288e09d5ba 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -28,7 +28,7 @@
enum mlx5_rte_flow_item_type {
MLX5_RTE_FLOW_ITEM_TYPE_END = INT_MIN,
MLX5_RTE_FLOW_ITEM_TYPE_TAG,
- MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE,
+ MLX5_RTE_FLOW_ITEM_TYPE_SQ,
MLX5_RTE_FLOW_ITEM_TYPE_VLAN,
MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL,
};
@@ -95,7 +95,7 @@ struct mlx5_flow_action_copy_mreg {
};
/* Matches on source queue. */
-struct mlx5_rte_flow_item_tx_queue {
+struct mlx5_rte_flow_item_sq {
uint32_t queue;
};
@@ -159,7 +159,7 @@ enum mlx5_feature_name {
#define MLX5_FLOW_LAYER_GENEVE (1u << 26)
/* Queue items. */
-#define MLX5_FLOW_ITEM_TX_QUEUE (1u << 27)
+#define MLX5_FLOW_ITEM_SQ (1u << 27)
/* Pattern tunnel Layer bits (continued). */
#define MLX5_FLOW_LAYER_GTP (1u << 28)
@@ -196,6 +196,9 @@ enum mlx5_feature_name {
#define MLX5_FLOW_ITEM_PORT_REPRESENTOR (UINT64_C(1) << 41)
#define MLX5_FLOW_ITEM_REPRESENTED_PORT (UINT64_C(1) << 42)
+/* Meter color item */
+#define MLX5_FLOW_ITEM_METER_COLOR (UINT64_C(1) << 44)
+
/* Outer Masks. */
#define MLX5_FLOW_LAYER_OUTER_L3 \
(MLX5_FLOW_LAYER_OUTER_L3_IPV4 | MLX5_FLOW_LAYER_OUTER_L3_IPV6)
@@ -1006,6 +1009,18 @@ flow_items_to_tunnel(const struct rte_flow_item items[])
return items[0].spec;
}
+/* HW steering flow attributes. */
+struct mlx5_flow_attr {
+ uint32_t port_id; /* Port index. */
+ uint32_t group; /* Flow group. */
+ uint32_t priority; /* Original Priority. */
+ /* rss level, used by priority adjustment. */
+ uint32_t rss_level;
+ /* Action flags, used by priority adjustment. */
+ uint32_t act_flags;
+ uint32_t tbl_type; /* Flow table type. */
+};
+
/* Flow structure. */
struct rte_flow {
uint32_t dev_handles;
@@ -1766,6 +1781,32 @@ mlx5_translate_tunnel_etypes(uint64_t pattern_flags)
int flow_hw_q_flow_flush(struct rte_eth_dev *dev,
struct rte_flow_error *error);
+
+/*
+ * Convert rte_mtr_color to mlx5 color.
+ *
+ * @param[in] rcol
+ * rte_mtr_color.
+ *
+ * @return
+ * mlx5 color.
+ */
+static inline int
+rte_col_2_mlx5_col(enum rte_color rcol)
+{
+ switch (rcol) {
+ case RTE_COLOR_GREEN:
+ return MLX5_FLOW_COLOR_GREEN;
+ case RTE_COLOR_YELLOW:
+ return MLX5_FLOW_COLOR_YELLOW;
+ case RTE_COLOR_RED:
+ return MLX5_FLOW_COLOR_RED;
+ default:
+ break;
+ }
+ return MLX5_FLOW_COLOR_UNDEFINED;
+}
+
int mlx5_flow_group_to_table(struct rte_eth_dev *dev,
const struct mlx5_f
[v5 02/18] net/mlx5: split flow item matcher and value translation
From: Suanming Mou
As hardware steering mode translates flow matcher and value in two
different stages, split the flow item matcher and value translation
to help reuse the code.
Signed-off-by: Suanming Mou
---
drivers/net/mlx5/mlx5_flow.h| 32 +
drivers/net/mlx5/mlx5_flow_dv.c | 2314 +++
2 files changed, 1185 insertions(+), 1161 deletions(-)
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index 0fa1735b1a..2ebb8496f2 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -1264,6 +1264,38 @@ struct mlx5_flow_workspace {
uint32_t skip_matcher_reg:1;
/* Indicates if need to skip matcher register in translate. */
uint32_t mark:1; /* Indicates if flow contains mark action. */
+ uint32_t vport_meta_tag; /* Used for vport index match. */
+};
+
+/* Matcher translate type. */
+enum MLX5_SET_MATCHER {
+ MLX5_SET_MATCHER_SW_V = 1 << 0,
+ MLX5_SET_MATCHER_SW_M = 1 << 1,
+ MLX5_SET_MATCHER_HS_V = 1 << 2,
+ MLX5_SET_MATCHER_HS_M = 1 << 3,
+};
+
+#define MLX5_SET_MATCHER_SW (MLX5_SET_MATCHER_SW_V | MLX5_SET_MATCHER_SW_M)
+#define MLX5_SET_MATCHER_HS (MLX5_SET_MATCHER_HS_V | MLX5_SET_MATCHER_HS_M)
+#define MLX5_SET_MATCHER_V (MLX5_SET_MATCHER_SW_V | MLX5_SET_MATCHER_HS_V)
+#define MLX5_SET_MATCHER_M (MLX5_SET_MATCHER_SW_M | MLX5_SET_MATCHER_HS_M)
+
+/* Flow matcher workspace intermediate data. */
+struct mlx5_dv_matcher_workspace {
+ uint8_t priority; /* Flow priority. */
+ uint64_t last_item; /* Last item in pattern. */
+ uint64_t item_flags; /* Flow item pattern flags. */
+ uint64_t action_flags; /* Flow action flags. */
+ bool external; /* External flow or not. */
+ uint32_t vlan_tag:12; /* Flow item VLAN tag. */
+ uint8_t next_protocol; /* Tunnel next protocol */
+ uint32_t geneve_tlv_option; /* Flow item Geneve TLV option. */
+ uint32_t group; /* Flow group. */
+ uint16_t udp_dport; /* Flow item UDP port. */
+ const struct rte_flow_attr *attr; /* Flow attribute. */
+ struct mlx5_flow_rss_desc *rss_desc; /* RSS descriptor. */
+ const struct rte_flow_item *tunnel_item; /* Flow tunnel item. */
+ const struct rte_flow_item *gre_item; /* Flow GRE item. */
};
struct mlx5_flow_split_info {
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 70a3279e2f..0589cafc30 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -63,6 +63,25 @@
#define MLX5DV_FLOW_VLAN_PCP_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK)
#define MLX5DV_FLOW_VLAN_VID_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_VID_MASK)
+#define MLX5_ITEM_VALID(item, key_type) \
+ (((MLX5_SET_MATCHER_SW & (key_type)) && !((item)->spec)) || \
+((MLX5_SET_MATCHER_HS_V == (key_type)) && !((item)->spec)) || \
+((MLX5_SET_MATCHER_HS_M == (key_type)) && !((item)->mask)))
+
+#define MLX5_ITEM_UPDATE(item, key_type, v, m, gm) \
+ do { \
+ if ((key_type) == MLX5_SET_MATCHER_SW_V) { \
+ v = (item)->spec; \
+ m = (item)->mask ? (item)->mask : (gm); \
+ } else if ((key_type) == MLX5_SET_MATCHER_HS_V) { \
+ v = (item)->spec; \
+ m = (v); \
+ } else { \
+ v = (item)->mask ? (item)->mask : (gm); \
+ m = (v); \
+ } \
+ } while (0)
+
union flow_dv_attr {
struct {
uint32_t valid:1;
@@ -8323,70 +8342,61 @@ flow_dv_check_valid_spec(void *match_mask, void
*match_value)
static inline void
flow_dv_set_match_ip_version(uint32_t group,
void *headers_v,
-void *headers_m,
+uint32_t key_type,
uint8_t ip_version)
{
- if (group == 0)
- MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
+ if (group == 0 && (key_type & MLX5_SET_MATCHER_M))
+ MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, 0xf);
else
- MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version,
+ MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version,
ip_version);
- MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, ip_version);
MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype, 0);
- MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype, 0);
}
/**
- * Add Ethernet item to matcher and to the value.
+ * Add Ethernet item to the value.
*
- * @param[in, out] matcher
- * Flow matcher.
* @param[in, out] key
* Flow matcher value.
* @param[in] item
* Flow pattern to translate.
* @param[in] inner
* Item is inner pattern.
+ * @param[in] grpup
+ * Flow matcher group.
+ * @param[in] key_type
+ * Set flow matcher mask or value.
*/
static voi
[v5 04/18] net/mlx5: add port to metadata conversion
From: Dariusz Sosnowski
This patch initial version of functions used to:
- convert between ethdev port_id and internal tag/mask value,
- convert between IB context and internal tag/mask value.
Signed-off-by: Dariusz Sosnowski
---
drivers/net/mlx5/linux/mlx5_os.c | 10 +-
drivers/net/mlx5/mlx5.c | 1 +
drivers/net/mlx5/mlx5_flow.c | 6
drivers/net/mlx5/mlx5_flow.h | 52
drivers/net/mlx5/mlx5_flow_hw.c | 29 ++
5 files changed, 97 insertions(+), 1 deletion(-)
diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index 60677eb8d7..98c6374547 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -1541,8 +1541,16 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
if (!priv->hrxqs)
goto error;
rte_rwlock_init(&priv->ind_tbls_lock);
- if (priv->sh->config.dv_flow_en == 2)
+ if (priv->sh->config.dv_flow_en == 2) {
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+ if (priv->vport_meta_mask)
+ flow_hw_set_port_info(eth_dev);
return eth_dev;
+#else
+ DRV_LOG(ERR, "DV support is missing for HWS.");
+ goto error;
+#endif
+ }
/* Port representor shares the same max priority with pf port. */
if (!priv->sh->flow_priority_check_flag) {
/* Supported Verbs flow priority number detection. */
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 752b60d769..1d10932619 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -1944,6 +1944,7 @@ mlx5_dev_close(struct rte_eth_dev *dev)
mlx5_flex_item_port_cleanup(dev);
#if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
flow_hw_resource_release(dev);
+ flow_hw_clear_port_info(dev);
#endif
if (priv->rxq_privs != NULL) {
/* XXX race condition if mlx5_rx_burst() is still running. */
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 742dbd6358..9d94da0868 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -33,6 +33,12 @@
#include "mlx5_common_os.h"
#include "rte_pmd_mlx5.h"
+/*
+ * Shared array for quick translation between port_id and vport mask/values
+ * used for HWS rules.
+ */
+struct flow_hw_port_info mlx5_flow_hw_port_infos[RTE_MAX_ETHPORTS];
+
struct tunnel_default_miss_ctx {
uint16_t *queue;
__extension__
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index 288e09d5ba..17102623c1 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -1323,6 +1323,58 @@ struct mlx5_flow_split_info {
uint64_t prefix_layers; /**< Prefix subflow layers. */
};
+struct flow_hw_port_info {
+ uint32_t regc_mask;
+ uint32_t regc_value;
+ uint32_t is_wire:1;
+};
+
+extern struct flow_hw_port_info mlx5_flow_hw_port_infos[RTE_MAX_ETHPORTS];
+
+/*
+ * Get metadata match tag and mask for given rte_eth_dev port.
+ * Used in HWS rule creation.
+ */
+static __rte_always_inline const struct flow_hw_port_info *
+flow_hw_conv_port_id(const uint16_t port_id)
+{
+ struct flow_hw_port_info *port_info;
+
+ if (port_id >= RTE_MAX_ETHPORTS)
+ return NULL;
+ port_info = &mlx5_flow_hw_port_infos[port_id];
+ return !!port_info->regc_mask ? port_info : NULL;
+}
+
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+/*
+ * Get metadata match tag and mask for the uplink port represented
+ * by given IB context. Used in HWS context creation.
+ */
+static __rte_always_inline const struct flow_hw_port_info *
+flow_hw_get_wire_port(struct ibv_context *ibctx)
+{
+ struct ibv_device *ibdev = ibctx->device;
+ uint16_t port_id;
+
+ MLX5_ETH_FOREACH_DEV(port_id, NULL) {
+ const struct mlx5_priv *priv =
+ rte_eth_devices[port_id].data->dev_private;
+
+ if (priv && priv->master) {
+ struct ibv_context *port_ibctx = priv->sh->cdev->ctx;
+
+ if (port_ibctx->device == ibdev)
+ return flow_hw_conv_port_id(port_id);
+ }
+ }
+ return NULL;
+}
+#endif
+
+void flow_hw_set_port_info(struct rte_eth_dev *dev);
+void flow_hw_clear_port_info(struct rte_eth_dev *dev);
+
typedef int (*mlx5_flow_validate_t)(struct rte_eth_dev *dev,
const struct rte_flow_attr *attr,
const struct rte_flow_item items[],
diff --git a/drivers/net/mlx5/mlx5_flow_hw.c b/drivers/net/mlx5/mlx5_flow_hw.c
index 12498794a5..fe809a83b9 100644
--- a/drivers/net/mlx5/mlx5_flow_hw.c
+++ b/drivers/net/mlx5/mlx5_flow_hw.c
@@ -2208,6 +2208,35 @@ flow_hw_resource_release(struct rte_eth_dev *dev)
priv->nb_queue = 0;
}
+/* Sets vport tag and mask, for given port, used in HWS ru
[v5 05/18] common/mlx5: query set capability of registers
From: Bing Zhao
In the flow table capabilities, new fields are added to query the
capability to set, add, copy to a REG_C_x.
The set capability are queried and saved for the future usage.
Signed-off-by: Bing Zhao
---
drivers/common/mlx5/mlx5_devx_cmds.c | 30 +++
drivers/common/mlx5/mlx5_devx_cmds.h | 2 ++
drivers/common/mlx5/mlx5_prm.h | 45 +---
3 files changed, 73 insertions(+), 4 deletions(-)
diff --git a/drivers/common/mlx5/mlx5_devx_cmds.c
b/drivers/common/mlx5/mlx5_devx_cmds.c
index 76f0b6724f..9c185366d0 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.c
+++ b/drivers/common/mlx5/mlx5_devx_cmds.c
@@ -1064,6 +1064,24 @@ mlx5_devx_cmd_query_hca_attr(void *ctx,
attr->modify_outer_ip_ecn = MLX5_GET
(flow_table_nic_cap, hcattr,
ft_header_modify_nic_receive.outer_ip_ecn);
+ attr->set_reg_c = 0xff;
+ if (attr->nic_flow_table) {
+#define GET_RX_REG_X_BITS \
+ MLX5_GET(flow_table_nic_cap, hcattr, \
+ft_header_modify_nic_receive.metadata_reg_c_x)
+#define GET_TX_REG_X_BITS \
+ MLX5_GET(flow_table_nic_cap, hcattr, \
+ft_header_modify_nic_transmit.metadata_reg_c_x)
+
+ uint32_t tx_reg, rx_reg;
+
+ tx_reg = GET_TX_REG_X_BITS;
+ rx_reg = GET_RX_REG_X_BITS;
+ attr->set_reg_c &= (rx_reg & tx_reg);
+
+#undef GET_RX_REG_X_BITS
+#undef GET_TX_REG_X_BITS
+ }
attr->pkt_integrity_match = mlx5_devx_query_pkt_integrity_match(hcattr);
attr->inner_ipv4_ihl = MLX5_GET
(flow_table_nic_cap, hcattr,
@@ -1163,6 +1181,18 @@ mlx5_devx_cmd_query_hca_attr(void *ctx,
attr->esw_mgr_vport_id =
MLX5_GET(esw_cap, hcattr, esw_manager_vport_number);
}
+ if (attr->eswitch_manager) {
+ uint32_t esw_reg;
+
+ hcattr = mlx5_devx_get_hca_cap(ctx, in, out, &rc,
+ MLX5_GET_HCA_CAP_OP_MOD_ESW_FLOW_TABLE |
+ MLX5_HCA_CAP_OPMOD_GET_CUR);
+ if (!hcattr)
+ return rc;
+ esw_reg = MLX5_GET(flow_table_esw_cap, hcattr,
+ ft_header_modify_esw_fdb.metadata_reg_c_x);
+ attr->set_reg_c &= esw_reg;
+ }
return 0;
error:
rc = (rc > 0) ? -rc : rc;
diff --git a/drivers/common/mlx5/mlx5_devx_cmds.h
b/drivers/common/mlx5/mlx5_devx_cmds.h
index cceaf3411d..a10aa3331b 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.h
+++ b/drivers/common/mlx5/mlx5_devx_cmds.h
@@ -263,6 +263,8 @@ struct mlx5_hca_attr {
uint32_t crypto_wrapped_import_method:1;
uint16_t esw_mgr_vport_id; /* E-Switch Mgr vport ID . */
uint16_t max_wqe_sz_sq;
+ uint32_t set_reg_c:8;
+ uint32_t nic_flow_table:1;
uint32_t modify_outer_ip_ecn:1;
};
diff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h
index 9c1c93f916..ca4763f53d 100644
--- a/drivers/common/mlx5/mlx5_prm.h
+++ b/drivers/common/mlx5/mlx5_prm.h
@@ -1295,6 +1295,7 @@ enum {
MLX5_GET_HCA_CAP_OP_MOD_QOS_CAP = 0xc << 1,
MLX5_GET_HCA_CAP_OP_MOD_ROCE = 0x4 << 1,
MLX5_GET_HCA_CAP_OP_MOD_NIC_FLOW_TABLE = 0x7 << 1,
+ MLX5_GET_HCA_CAP_OP_MOD_ESW_FLOW_TABLE = 0x8 << 1,
MLX5_SET_HCA_CAP_OP_MOD_ESW = 0x9 << 1,
MLX5_GET_HCA_CAP_OP_MOD_VDPA_EMULATION = 0x13 << 1,
MLX5_GET_HCA_CAP_OP_MOD_CRYPTO = 0x1A << 1,
@@ -1892,6 +1893,7 @@ struct mlx5_ifc_roce_caps_bits {
};
struct mlx5_ifc_ft_fields_support_bits {
+ /* set_action_field_support */
u8 outer_dmac[0x1];
u8 outer_smac[0x1];
u8 outer_ether_type[0x1];
@@ -1919,7 +1921,7 @@ struct mlx5_ifc_ft_fields_support_bits {
u8 outer_gre_key[0x1];
u8 outer_vxlan_vni[0x1];
u8 reserved_at_1a[0x5];
- u8 source_eswitch_port[0x1];
+ u8 source_eswitch_port[0x1]; /* end of DW0 */
u8 inner_dmac[0x1];
u8 inner_smac[0x1];
u8 inner_ether_type[0x1];
@@ -1943,8 +1945,33 @@ struct mlx5_ifc_ft_fields_support_bits {
u8 inner_tcp_sport[0x1];
u8 inner_tcp_dport[0x1];
u8 inner_tcp_flags[0x1];
- u8 reserved_at_37[0x9];
- u8 reserved_at_40[0x40];
+ u8 reserved_at_37[0x9]; /* end of DW1 */
+ u8 reserved_at_40[0x20]; /* end of DW2 */
+ u8 reserved_at_60[0x18];
+ union {
+ struct {
+ u8 metadata_reg_c_7[0x1];
+ u8 metadata_reg_c_6[0x1];
+ u8 metadata_reg_c_5[0x1];
+ u8 metadata_reg_c_4[0x1];
+ u8 metadata_reg_c_3[0x1];
+ u8 metadata_reg_c_2[0x1];
+ u8 metadata_reg_c_1[0x1];
+ u8 metadata_reg_c_0[0x1];
+ };
+ u8 metadata_reg_c_x[0
[v5 06/18] net/mlx5: provide the available tag registers
From: Bing Zhao
The available tags that can be used by the application are fixed
after startup.
A global array is used to store the information and transfer the TAG
item directly from the ID to the REG_C_x.
Signed-off-by: Bing Zhao
---
drivers/net/mlx5/linux/mlx5_os.c | 2 +
drivers/net/mlx5/mlx5.c | 2 +
drivers/net/mlx5/mlx5.h | 1 +
drivers/net/mlx5/mlx5_defs.h | 2 +
drivers/net/mlx5/mlx5_flow.c | 11 +
drivers/net/mlx5/mlx5_flow.h | 27
drivers/net/mlx5/mlx5_flow_hw.c | 76
7 files changed, 121 insertions(+)
diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index 98c6374547..aed55e6a62 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -1545,6 +1545,8 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
#ifdef HAVE_IBV_FLOW_DV_SUPPORT
if (priv->vport_meta_mask)
flow_hw_set_port_info(eth_dev);
+ /* Only HWS requires this information. */
+ flow_hw_init_tags_set(eth_dev);
return eth_dev;
#else
DRV_LOG(ERR, "DV support is missing for HWS.");
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 1d10932619..b39ef1ecbe 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -1945,6 +1945,8 @@ mlx5_dev_close(struct rte_eth_dev *dev)
#if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
flow_hw_resource_release(dev);
flow_hw_clear_port_info(dev);
+ if (priv->sh->config.dv_flow_en == 2)
+ flow_hw_clear_tags_set(dev);
#endif
if (priv->rxq_privs != NULL) {
/* XXX race condition if mlx5_rx_burst() is still running. */
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 3c9e6bad53..741be2df98 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -1200,6 +1200,7 @@ struct mlx5_dev_ctx_shared {
uint32_t drop_action_check_flag:1; /* Check Flag for drop action. */
uint32_t flow_priority_check_flag:1; /* Check Flag for flow priority. */
uint32_t metadata_regc_check_flag:1; /* Check Flag for metadata REGC. */
+ uint32_t hws_tags:1; /* Check if tags info for HWS initialized. */
uint32_t max_port; /* Maximal IB device port index. */
struct mlx5_bond_info bond; /* Bonding information. */
struct mlx5_common_device *cdev; /* Backend mlx5 device. */
diff --git a/drivers/net/mlx5/mlx5_defs.h b/drivers/net/mlx5/mlx5_defs.h
index 018d3f0f0c..585afb0a98 100644
--- a/drivers/net/mlx5/mlx5_defs.h
+++ b/drivers/net/mlx5/mlx5_defs.h
@@ -139,6 +139,8 @@
#define MLX5_XMETA_MODE_META32 2
/* Provide info on patrial hw miss. Implies MLX5_XMETA_MODE_META16 */
#define MLX5_XMETA_MODE_MISS_INFO 3
+/* Only valid in HWS, 32bits extended META without MARK support in FDB. */
+#define MLX5_XMETA_MODE_META32_HWS 4
/* Tx accurate scheduling on timestamps parameters. */
#define MLX5_TXPP_WAIT_INIT_TS 1000ul /* How long to wait timestamp. */
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 9d94da0868..dd3d2bb1a4 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -39,6 +39,17 @@
*/
struct flow_hw_port_info mlx5_flow_hw_port_infos[RTE_MAX_ETHPORTS];
+/*
+ * A global structure to save the available REG_C_x for tags usage.
+ * The Meter color REG (ASO) and the last available one will be reserved
+ * for PMD internal usage.
+ * Since there is no "port" concept in the driver, it is assumed that the
+ * available tags set will be the minimum intersection.
+ * 3 - in FDB mode / 5 - in legacy mode
+ */
+uint32_t mlx5_flow_hw_avl_tags_init_cnt;
+enum modify_reg mlx5_flow_hw_avl_tags[MLX5_FLOW_HW_TAGS_MAX] = {REG_NON};
+
struct tunnel_default_miss_ctx {
uint16_t *queue;
__extension__
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index 17102623c1..2002f6ef4b 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -1331,6 +1331,10 @@ struct flow_hw_port_info {
extern struct flow_hw_port_info mlx5_flow_hw_port_infos[RTE_MAX_ETHPORTS];
+#define MLX5_FLOW_HW_TAGS_MAX 8
+extern uint32_t mlx5_flow_hw_avl_tags_init_cnt;
+extern enum modify_reg mlx5_flow_hw_avl_tags[];
+
/*
* Get metadata match tag and mask for given rte_eth_dev port.
* Used in HWS rule creation.
@@ -1372,9 +1376,32 @@ flow_hw_get_wire_port(struct ibv_context *ibctx)
}
#endif
+/*
+ * Convert metadata or tag to the actual register.
+ * META: Can only be used to match in the FDB in this stage, fixed C_1.
+ * TAG: C_x expect meter color reg and the reserved ones.
+ * TODO: Per port / device, FDB or NIC for Meta matching.
+ */
+static __rte_always_inline int
+flow_hw_get_reg_id(enum rte_flow_item_type type, uint32_t id)
+{
+ switch (type) {
+ case RTE_FLOW_ITEM_TYPE_META:
+ return REG_
[v5 07/18] net/mlx5: Add additional glue functions for HWS
Add missing glue support for HWS mlx5dr layer. The new glue
functions are needed for mlx5dv create matcher and action,
which are used as the kernel root table as well as for
capabilities query like device name and ports info.
Signed-off-by: Alex Vesker
---
drivers/common/mlx5/linux/mlx5_glue.c | 121 --
drivers/common/mlx5/linux/mlx5_glue.h | 17
2 files changed, 131 insertions(+), 7 deletions(-)
diff --git a/drivers/common/mlx5/linux/mlx5_glue.c
b/drivers/common/mlx5/linux/mlx5_glue.c
index 450dd6a06a..9f5953fbce 100644
--- a/drivers/common/mlx5/linux/mlx5_glue.c
+++ b/drivers/common/mlx5/linux/mlx5_glue.c
@@ -111,6 +111,12 @@ mlx5_glue_query_device_ex(struct ibv_context *context,
return ibv_query_device_ex(context, input, attr);
}
+static const char *
+mlx5_glue_get_device_name(struct ibv_device *device)
+{
+ return ibv_get_device_name(device);
+}
+
static int
mlx5_glue_query_rt_values_ex(struct ibv_context *context,
struct ibv_values_ex *values)
@@ -620,6 +626,20 @@ mlx5_glue_dv_create_qp(struct ibv_context *context,
#endif
}
+static void *
+__mlx5_glue_dv_create_flow_matcher(struct ibv_context *context,
+ struct mlx5dv_flow_matcher_attr *matcher_attr)
+{
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+ return mlx5dv_create_flow_matcher(context, matcher_attr);
+#else
+ (void)context;
+ (void)matcher_attr;
+ errno = ENOTSUP;
+ return NULL;
+#endif
+}
+
static void *
mlx5_glue_dv_create_flow_matcher(struct ibv_context *context,
struct mlx5dv_flow_matcher_attr *matcher_attr,
@@ -633,7 +653,7 @@ mlx5_glue_dv_create_flow_matcher(struct ibv_context
*context,
matcher_attr->match_mask);
#else
(void)tbl;
- return mlx5dv_create_flow_matcher(context, matcher_attr);
+ return __mlx5_glue_dv_create_flow_matcher(context, matcher_attr);
#endif
#else
(void)context;
@@ -644,6 +664,26 @@ mlx5_glue_dv_create_flow_matcher(struct ibv_context
*context,
#endif
}
+static void *
+__mlx5_glue_dv_create_flow(void *matcher,
+ void *match_value,
+ size_t num_actions,
+ void *actions)
+{
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+ return mlx5dv_create_flow(matcher,
+ match_value,
+ num_actions,
+ (struct mlx5dv_flow_action_attr *)actions);
+#else
+ (void)matcher;
+ (void)match_value;
+ (void)num_actions;
+ (void)actions;
+ return NULL;
+#endif
+}
+
static void *
mlx5_glue_dv_create_flow(void *matcher,
void *match_value,
@@ -663,8 +703,8 @@ mlx5_glue_dv_create_flow(void *matcher,
for (i = 0; i < num_actions; i++)
actions_attr[i] =
*((struct mlx5dv_flow_action_attr *)(actions[i]));
- return mlx5dv_create_flow(matcher, match_value,
- num_actions, actions_attr);
+ return __mlx5_glue_dv_create_flow(matcher, match_value,
+ num_actions, actions_attr);
#endif
#else
(void)matcher;
@@ -735,6 +775,26 @@ mlx5_glue_dv_create_flow_action_dest_devx_tir(void *tir)
#endif
}
+static void *
+__mlx5_glue_dv_create_flow_action_modify_header
+ (struct ibv_context *ctx,
+size_t actions_sz,
+uint64_t actions[],
+enum mlx5dv_flow_table_type ft_type)
+{
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+ return mlx5dv_create_flow_action_modify_header
+ (ctx, actions_sz, actions, ft_type);
+#else
+ (void)ctx;
+ (void)ft_type;
+ (void)actions_sz;
+ (void)actions;
+ errno = ENOTSUP;
+ return NULL;
+#endif
+}
+
static void *
mlx5_glue_dv_create_flow_action_modify_header
(struct ibv_context *ctx,
@@ -758,7 +818,7 @@ mlx5_glue_dv_create_flow_action_modify_header
if (!action)
return NULL;
action->type = MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION;
- action->action = mlx5dv_create_flow_action_modify_header
+ action->action = __mlx5_glue_dv_create_flow_action_modify_header
(ctx, actions_sz, actions, ft_type);
return action;
#endif
@@ -774,6 +834,27 @@ mlx5_glue_dv_create_flow_action_modify_header
#endif
}
+static void *
+__mlx5_glue_dv_create_flow_action_packet_reformat
+ (struct ibv_context *ctx,
+size_t data_sz, void *data,
+enum mlx5dv_flow_action_packet_reformat_type reformat_type,
+enum mlx5dv_flow_table_type ft_type)
+{
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+ return mlx5dv_create_flow_action_
[v5 08/18] net/mlx5/hws: Add HWS command layer
From: Erez Shitrit
The command layer is used to communicate with the FW,
query capabilities and allocate FW resources needed for HWS.
Signed-off-by: Erez Shitrit
Signed-off-by: Alex Vesker
---
drivers/common/mlx5/mlx5_prm.h| 607 ++-
drivers/net/mlx5/hws/mlx5dr_cmd.c | 948 ++
drivers/net/mlx5/hws/mlx5dr_cmd.h | 230
3 files changed, 1775 insertions(+), 10 deletions(-)
create mode 100644 drivers/net/mlx5/hws/mlx5dr_cmd.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_cmd.h
diff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h
index ca4763f53d..371942ae50 100644
--- a/drivers/common/mlx5/mlx5_prm.h
+++ b/drivers/common/mlx5/mlx5_prm.h
@@ -289,6 +289,8 @@
/* The alignment needed for CQ buffer. */
#define MLX5_CQE_BUF_ALIGNMENT rte_mem_page_size()
+#define MAX_ACTIONS_DATA_IN_HEADER_MODIFY 512
+
/* Completion mode. */
enum mlx5_completion_mode {
MLX5_COMP_ONLY_ERR = 0x0,
@@ -677,6 +679,10 @@ enum {
MLX5_MODIFICATION_TYPE_SET = 0x1,
MLX5_MODIFICATION_TYPE_ADD = 0x2,
MLX5_MODIFICATION_TYPE_COPY = 0x3,
+ MLX5_MODIFICATION_TYPE_INSERT = 0x4,
+ MLX5_MODIFICATION_TYPE_REMOVE = 0x5,
+ MLX5_MODIFICATION_TYPE_NOP = 0x6,
+ MLX5_MODIFICATION_TYPE_REMOVE_WORDS = 0x7,
};
/* The field of packet to be modified. */
@@ -,6 +1117,10 @@ enum {
MLX5_CMD_OP_QUERY_TIS = 0x915,
MLX5_CMD_OP_CREATE_RQT = 0x916,
MLX5_CMD_OP_MODIFY_RQT = 0x917,
+ MLX5_CMD_OP_CREATE_FLOW_TABLE = 0x930,
+ MLX5_CMD_OP_CREATE_FLOW_GROUP = 0x933,
+ MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY = 0x936,
+ MLX5_CMD_OP_MODIFY_FLOW_TABLE = 0x93c,
MLX5_CMD_OP_ALLOC_FLOW_COUNTER = 0x939,
MLX5_CMD_OP_QUERY_FLOW_COUNTER = 0x93b,
MLX5_CMD_OP_CREATE_GENERAL_OBJECT = 0xa00,
@@ -1299,6 +1309,7 @@ enum {
MLX5_SET_HCA_CAP_OP_MOD_ESW = 0x9 << 1,
MLX5_GET_HCA_CAP_OP_MOD_VDPA_EMULATION = 0x13 << 1,
MLX5_GET_HCA_CAP_OP_MOD_CRYPTO = 0x1A << 1,
+ MLX5_GET_HCA_CAP_OP_MOD_WQE_BASED_FLOW_TABLE = 0x1B << 1,
MLX5_GET_HCA_CAP_OP_MOD_PARSE_GRAPH_NODE_CAP = 0x1C << 1,
MLX5_GET_HCA_CAP_OP_MOD_GENERAL_DEVICE_2 = 0x20 << 1,
};
@@ -1317,6 +1328,14 @@ enum {
(1ULL << MLX5_GENERAL_OBJ_TYPE_GENEVE_TLV_OPT)
#define MLX5_GENERAL_OBJ_TYPES_CAP_CONN_TRACK_OFFLOAD \
(1ULL << MLX5_GENERAL_OBJ_TYPE_CONN_TRACK_OFFLOAD)
+#define MLX5_GENERAL_OBJ_TYPES_CAP_RTC \
+ (1ULL << MLX5_GENERAL_OBJ_TYPE_RTC)
+#define MLX5_GENERAL_OBJ_TYPES_CAP_STC \
+ (1ULL << MLX5_GENERAL_OBJ_TYPE_STC)
+#define MLX5_GENERAL_OBJ_TYPES_CAP_STE \
+ (1ULL << MLX5_GENERAL_OBJ_TYPE_STE)
+#define MLX5_GENERAL_OBJ_TYPES_CAP_DEFINER \
+ (1ULL << MLX5_GENERAL_OBJ_TYPE_DEFINER)
#define MLX5_GENERAL_OBJ_TYPES_CAP_DEK \
(1ULL << MLX5_GENERAL_OBJ_TYPE_DEK)
#define MLX5_GENERAL_OBJ_TYPES_CAP_IMPORT_KEK \
@@ -1373,6 +1392,11 @@ enum {
#define MLX5_HCA_FLEX_VXLAN_GPE_ENABLED (1UL << 7)
#define MLX5_HCA_FLEX_ICMP_ENABLED (1UL << 8)
#define MLX5_HCA_FLEX_ICMPV6_ENABLED (1UL << 9)
+#define MLX5_HCA_FLEX_GTPU_ENABLED (1UL << 11)
+#define MLX5_HCA_FLEX_GTPU_DW_2_ENABLED (1UL << 16)
+#define MLX5_HCA_FLEX_GTPU_FIRST_EXT_DW_0_ENABLED (1UL << 17)
+#define MLX5_HCA_FLEX_GTPU_DW_0_ENABLED (1UL << 18)
+#define MLX5_HCA_FLEX_GTPU_TEID_ENABLED (1UL << 19)
/* The device steering logic format. */
#define MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 0x0
@@ -1505,7 +1529,8 @@ struct mlx5_ifc_cmd_hca_cap_bits {
u8 wol_u[0x1];
u8 wol_p[0x1];
u8 stat_rate_support[0x10];
- u8 reserved_at_1f0[0xc];
+ u8 reserved_at_1ef[0xb];
+ u8 wqe_based_flow_table_update_cap[0x1];
u8 cqe_version[0x4];
u8 compact_address_vector[0x1];
u8 striding_rq[0x1];
@@ -1681,7 +1706,9 @@ struct mlx5_ifc_cmd_hca_cap_bits {
u8 cqe_compression[0x1];
u8 cqe_compression_timeout[0x10];
u8 cqe_compression_max_num[0x10];
- u8 reserved_at_5e0[0x10];
+ u8 reserved_at_5e0[0x8];
+ u8 flex_parser_id_gtpu_dw_0[0x4];
+ u8 reserved_at_5ec[0x4];
u8 tag_matching[0x1];
u8 rndv_offload_rc[0x1];
u8 rndv_offload_dc[0x1];
@@ -1691,17 +1718,38 @@ struct mlx5_ifc_cmd_hca_cap_bits {
u8 affiliate_nic_vport_criteria[0x8];
u8 native_port_num[0x8];
u8 num_vhca_ports[0x8];
- u8 reserved_at_618[0x6];
+ u8 flex_parser_id_gtpu_teid[0x4];
+ u8 reserved_at_61c[0x2];
u8 sw_owner_id[0x1];
u8 reserved_at_61f[0x6C];
u8 wait_on_data[0x1];
u8 wait_on_time[0x1];
- u8 reserved_at_68d[0xBB];
+ u8 reserved_at_68d[0x37];
+ u8 flex_parser_id_geneve_opt_0[0x
[v5 09/18] net/mlx5/hws: Add HWS pool and buddy
From: Erez Shitrit
HWS needs to manage different types of device memory in
an efficient and quick way. For this, memory pools are
being used.
Signed-off-by: Erez Shitrit
Signed-off-by: Alex Vesker
---
drivers/net/mlx5/hws/mlx5dr_buddy.c | 201 +
drivers/net/mlx5/hws/mlx5dr_buddy.h | 22 +
drivers/net/mlx5/hws/mlx5dr_pool.c | 672
drivers/net/mlx5/hws/mlx5dr_pool.h | 152 +++
4 files changed, 1047 insertions(+)
create mode 100644 drivers/net/mlx5/hws/mlx5dr_buddy.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_buddy.h
create mode 100644 drivers/net/mlx5/hws/mlx5dr_pool.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_pool.h
diff --git a/drivers/net/mlx5/hws/mlx5dr_buddy.c
b/drivers/net/mlx5/hws/mlx5dr_buddy.c
new file mode 100644
index 00..9dba95f0b1
--- /dev/null
+++ b/drivers/net/mlx5/hws/mlx5dr_buddy.c
@@ -0,0 +1,201 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2022 NVIDIA Corporation & Affiliates
+ */
+
+#include
+#include
+#include "mlx5dr_internal.h"
+#include "mlx5dr_buddy.h"
+
+static struct rte_bitmap *bitmap_alloc0(int s)
+{
+ struct rte_bitmap *bitmap;
+ uint32_t bmp_size;
+ void *mem;
+
+ bmp_size = rte_bitmap_get_memory_footprint(s);
+ mem = rte_zmalloc("create_bmap", bmp_size, RTE_CACHE_LINE_SIZE);
+ if (!mem) {
+ DR_LOG(ERR, "No mem for bitmap");
+ rte_errno = ENOMEM;
+ return NULL;
+ }
+
+ bitmap = rte_bitmap_init(s, mem, bmp_size);
+ if (!bitmap) {
+ DR_LOG(ERR, "%s Failed to initialize bitmap", __func__);
+ rte_errno = EINVAL;
+ goto err_mem_alloc;
+ }
+
+ return bitmap;
+
+err_mem_alloc:
+ rte_free(mem);
+ return NULL;
+}
+
+static void bitmap_set_bit(struct rte_bitmap *bmp, uint32_t pos)
+{
+ rte_bitmap_set(bmp, pos);
+}
+
+static void bitmap_clear_bit(struct rte_bitmap *bmp, uint32_t pos)
+{
+ rte_bitmap_clear(bmp, pos);
+}
+
+static bool bitmap_test_bit(struct rte_bitmap *bmp, unsigned long n)
+{
+ return !!rte_bitmap_get(bmp, n);
+}
+
+static unsigned long bitmap_ffs(struct rte_bitmap *bmap,
+ unsigned long n, unsigned long m)
+{
+ uint64_t out_slab = 0;
+ uint32_t pos = 0; /* Compilation warn */
+
+ __rte_bitmap_scan_init(bmap);
+ if (!rte_bitmap_scan(bmap, &pos, &out_slab)) {
+ DR_LOG(ERR, "Failed to get slab from bitmap.");
+ return m;
+ }
+ pos = pos + __builtin_ctzll(out_slab);
+
+ if (pos < n) {
+ DR_LOG(ERR, "Unexpected bit (%d < %"PRIx64") from bitmap", pos,
n);
+ return m;
+ }
+ return pos;
+}
+
+static unsigned long mlx5dr_buddy_find_first_bit(struct rte_bitmap *addr,
+uint32_t size)
+{
+ return bitmap_ffs(addr, 0, size);
+}
+
+static int mlx5dr_buddy_init(struct mlx5dr_buddy_mem *buddy, uint32_t
max_order)
+{
+ int i, s;
+
+ buddy->max_order = max_order;
+
+ buddy->bits = simple_calloc(buddy->max_order + 1, sizeof(long *));
+ if (!buddy->bits) {
+ rte_errno = ENOMEM;
+ return -1;
+ }
+
+ buddy->num_free = simple_calloc(buddy->max_order + 1,
sizeof(*buddy->num_free));
+ if (!buddy->num_free) {
+ rte_errno = ENOMEM;
+ goto err_out_free_bits;
+ }
+
+ for (i = 0; i <= (int)buddy->max_order; ++i) {
+ s = 1 << (buddy->max_order - i);
+ buddy->bits[i] = bitmap_alloc0(s);
+ if (!buddy->bits[i])
+ goto err_out_free_num_free;
+ }
+
+ bitmap_set_bit(buddy->bits[buddy->max_order], 0);
+
+ buddy->num_free[buddy->max_order] = 1;
+
+ return 0;
+
+err_out_free_num_free:
+ for (i = 0; i <= (int)buddy->max_order; ++i)
+ rte_free(buddy->bits[i]);
+
+ simple_free(buddy->num_free);
+
+err_out_free_bits:
+ simple_free(buddy->bits);
+ return -1;
+}
+
+struct mlx5dr_buddy_mem *mlx5dr_buddy_create(uint32_t max_order)
+{
+ struct mlx5dr_buddy_mem *buddy;
+
+ buddy = simple_calloc(1, sizeof(*buddy));
+ if (!buddy) {
+ rte_errno = ENOMEM;
+ return NULL;
+ }
+
+ if (mlx5dr_buddy_init(buddy, max_order))
+ goto free_buddy;
+
+ return buddy;
+
+free_buddy:
+ simple_free(buddy);
+ return NULL;
+}
+
+void mlx5dr_buddy_cleanup(struct mlx5dr_buddy_mem *buddy)
+{
+ int i;
+
+ for (i = 0; i <= (int)buddy->max_order; ++i) {
+ rte_free(buddy->bits[i]);
+ }
+
+ simple_free(buddy->num_free);
+ simple_free(bud
[v5 10/18] net/mlx5/hws: Add HWS send layer
HWS configures flows to the HW using a QP, each WQE has
the details of the flow we want to offload. The send layer
allocates the resources needed to send the request to the HW
as well as managing the queues, getting completions and
handling failures.
Signed-off-by: Mark Bloch
Signed-off-by: Alex Vesker
---
drivers/net/mlx5/hws/mlx5dr_send.c | 844 +
drivers/net/mlx5/hws/mlx5dr_send.h | 275 ++
2 files changed, 1119 insertions(+)
create mode 100644 drivers/net/mlx5/hws/mlx5dr_send.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_send.h
diff --git a/drivers/net/mlx5/hws/mlx5dr_send.c
b/drivers/net/mlx5/hws/mlx5dr_send.c
new file mode 100644
index 00..26904a9040
--- /dev/null
+++ b/drivers/net/mlx5/hws/mlx5dr_send.c
@@ -0,0 +1,844 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2022 NVIDIA Corporation & Affiliates
+ */
+
+#include "mlx5dr_internal.h"
+
+struct mlx5dr_send_ring_dep_wqe *
+mlx5dr_send_add_new_dep_wqe(struct mlx5dr_send_engine *queue)
+{
+ struct mlx5dr_send_ring_sq *send_sq = &queue->send_ring->send_sq;
+ unsigned int idx = send_sq->head_dep_idx++ & (queue->num_entries - 1);
+
+ memset(&send_sq->dep_wqe[idx].wqe_data.tag, 0, MLX5DR_MATCH_TAG_SZ);
+
+ return &send_sq->dep_wqe[idx];
+}
+
+void mlx5dr_send_abort_new_dep_wqe(struct mlx5dr_send_engine *queue)
+{
+ queue->send_ring->send_sq.head_dep_idx--;
+}
+
+void mlx5dr_send_all_dep_wqe(struct mlx5dr_send_engine *queue)
+{
+ struct mlx5dr_send_ring_sq *send_sq = &queue->send_ring->send_sq;
+ struct mlx5dr_send_ste_attr ste_attr = {0};
+ struct mlx5dr_send_ring_dep_wqe *dep_wqe;
+
+ ste_attr.send_attr.opmod = MLX5DR_WQE_GTA_OPMOD_STE;
+ ste_attr.send_attr.opcode = MLX5DR_WQE_OPCODE_TBL_ACCESS;
+ ste_attr.send_attr.len = MLX5DR_WQE_SZ_GTA_CTRL +
MLX5DR_WQE_SZ_GTA_DATA;
+ ste_attr.gta_opcode = MLX5DR_WQE_GTA_OP_ACTIVATE;
+
+ /* Fence first from previous depend WQEs */
+ ste_attr.send_attr.fence = 1;
+
+ while (send_sq->head_dep_idx != send_sq->tail_dep_idx) {
+ dep_wqe = &send_sq->dep_wqe[send_sq->tail_dep_idx++ &
(queue->num_entries - 1)];
+
+ /* Notify HW on the last WQE */
+ ste_attr.send_attr.notify_hw = (send_sq->tail_dep_idx ==
send_sq->head_dep_idx);
+ ste_attr.send_attr.user_data = dep_wqe->user_data;
+ ste_attr.send_attr.rule = dep_wqe->rule;
+
+ ste_attr.rtc_0 = dep_wqe->rtc_0;
+ ste_attr.rtc_1 = dep_wqe->rtc_1;
+ ste_attr.retry_rtc_0 = dep_wqe->retry_rtc_0;
+ ste_attr.retry_rtc_1 = dep_wqe->retry_rtc_1;
+ ste_attr.used_id_rtc_0 = &dep_wqe->rule->rtc_0;
+ ste_attr.used_id_rtc_1 = &dep_wqe->rule->rtc_1;
+ ste_attr.wqe_ctrl = &dep_wqe->wqe_ctrl;
+ ste_attr.wqe_data = &dep_wqe->wqe_data;
+
+ mlx5dr_send_ste(queue, &ste_attr);
+
+ /* Fencing is done only on the first WQE */
+ ste_attr.send_attr.fence = 0;
+ }
+}
+
+struct mlx5dr_send_engine_post_ctrl
+mlx5dr_send_engine_post_start(struct mlx5dr_send_engine *queue)
+{
+ struct mlx5dr_send_engine_post_ctrl ctrl;
+
+ ctrl.queue = queue;
+ /* Currently only one send ring is supported */
+ ctrl.send_ring = &queue->send_ring[0];
+ ctrl.num_wqebbs = 0;
+
+ return ctrl;
+}
+
+void mlx5dr_send_engine_post_req_wqe(struct mlx5dr_send_engine_post_ctrl *ctrl,
+char **buf, size_t *len)
+{
+ struct mlx5dr_send_ring_sq *send_sq = &ctrl->send_ring->send_sq;
+ unsigned int idx;
+
+ idx = (send_sq->cur_post + ctrl->num_wqebbs) & send_sq->buf_mask;
+
+ *buf = send_sq->buf + (idx << MLX5_SEND_WQE_SHIFT);
+ *len = MLX5_SEND_WQE_BB;
+
+ if (!ctrl->num_wqebbs) {
+ *buf += sizeof(struct mlx5dr_wqe_ctrl_seg);
+ *len -= sizeof(struct mlx5dr_wqe_ctrl_seg);
+ }
+
+ ctrl->num_wqebbs++;
+}
+
+static void mlx5dr_send_engine_post_ring(struct mlx5dr_send_ring_sq *sq,
+struct mlx5dv_devx_uar *uar,
+struct mlx5dr_wqe_ctrl_seg *wqe_ctrl)
+{
+ rte_compiler_barrier();
+ sq->db[MLX5_SND_DBR] = rte_cpu_to_be_32(sq->cur_post);
+
+ rte_wmb();
+ mlx5dr_uar_write64_relaxed(*((uint64_t *)wqe_ctrl), uar->reg_addr);
+ rte_wmb();
+}
+
+static void
+mlx5dr_send_wqe_set_tag(struct mlx5dr_wqe_gta_data_seg_ste *wqe_data,
+ struct mlx5dr_rule_match_tag *tag,
+ bool is_jumbo)
+{
+ if (is_jumbo) {
+ /* Clear previous possibly dirty control */
+
[v5 11/18] net/mlx5/hws: Add HWS definer layer
Definers are HW objects that are used for matching, rte items
are translated to definers, each definer holds the fields and
bit-masks used for HW flow matching. The definer layer is used
for finding the most efficient definer for each set of items.
In addition to definer creation we also calculate the field
copy (fc) array used for efficient items to WQE conversion.
Signed-off-by: Mark Bloch
Signed-off-by: Alex Vesker
---
drivers/net/mlx5/hws/mlx5dr_definer.c | 1968 +
drivers/net/mlx5/hws/mlx5dr_definer.h | 585
2 files changed, 2553 insertions(+)
create mode 100644 drivers/net/mlx5/hws/mlx5dr_definer.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_definer.h
diff --git a/drivers/net/mlx5/hws/mlx5dr_definer.c
b/drivers/net/mlx5/hws/mlx5dr_definer.c
new file mode 100644
index 00..6b98eb8c96
--- /dev/null
+++ b/drivers/net/mlx5/hws/mlx5dr_definer.c
@@ -0,0 +1,1968 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2022 NVIDIA Corporation & Affiliates
+ */
+
+#include "mlx5dr_internal.h"
+
+#define GTP_PDU_SC 0x85
+#define BAD_PORT 0xBAD
+#define ETH_TYPE_IPV4_VXLAN0x0800
+#define ETH_TYPE_IPV6_VXLAN0x86DD
+#define ETH_VXLAN_DEFAULT_PORT 4789
+
+#define STE_NO_VLAN0x0
+#define STE_SVLAN 0x1
+#define STE_CVLAN 0x2
+#define STE_IPV4 0x1
+#define STE_IPV6 0x2
+#define STE_TCP0x1
+#define STE_UDP0x2
+#define STE_ICMP 0x3
+
+/* Setter function based on bit offset and mask, for 32bit DW*/
+#define _DR_SET_32(p, v, byte_off, bit_off, mask) \
+ do { \
+ u32 _v = v; \
+ *((rte_be32_t *)(p) + ((byte_off) / 4)) = \
+ rte_cpu_to_be_32((rte_be_to_cpu_32(*((u32 *)(p) + \
+ ((byte_off) / 4))) & \
+ (~((mask) << (bit_off | \
+(((_v) & (mask)) << \
+ (bit_off))); \
+ } while (0)
+
+/* Setter function based on bit offset and mask */
+#define DR_SET(p, v, byte_off, bit_off, mask) \
+ do { \
+ if (unlikely((bit_off) < 0)) { \
+ u32 _bit_off = -1 * (bit_off); \
+ u32 second_dw_mask = (mask) & ((1 << _bit_off) - 1); \
+ _DR_SET_32(p, (v) >> _bit_off, byte_off, 0, (mask) >>
_bit_off); \
+ _DR_SET_32(p, (v) & second_dw_mask, (byte_off) +
DW_SIZE, \
+ (bit_off) % BITS_IN_DW, second_dw_mask); \
+ } else { \
+ _DR_SET_32(p, v, byte_off, (bit_off), (mask)); \
+ } \
+ } while (0)
+
+/* Setter function based on byte offset to directly set FULL BE32 value */
+#define DR_SET_BE32(p, v, byte_off, bit_off, mask) \
+ (*((rte_be32_t *)((uint8_t *)(p) + (byte_off))) = (v))
+
+/* Setter function based on byte offset to directly set FULL BE32 value from
ptr */
+#define DR_SET_BE32P(p, v_ptr, byte_off, bit_off, mask) \
+ memcpy((uint8_t *)(p) + (byte_off), v_ptr, 4)
+
+/* Setter function based on byte offset to directly set FULL BE16 value */
+#define DR_SET_BE16(p, v, byte_off, bit_off, mask) \
+ (*((rte_be16_t *)((uint8_t *)(p) + (byte_off))) = (v))
+
+/* Setter function based on byte offset to directly set FULL BE16 value from
ptr */
+#define DR_SET_BE16P(p, v_ptr, byte_off, bit_off, mask) \
+ memcpy((uint8_t *)(p) + (byte_off), v_ptr, 2)
+
+#define DR_CALC_FNAME(field, inner) \
+ ((inner) ? MLX5DR_DEFINER_FNAME_##field##_I : \
+ MLX5DR_DEFINER_FNAME_##field##_O)
+
+#define DR_CALC_SET_HDR(fc, hdr, field) \
+ do { \
+ (fc)->bit_mask = __mlx5_mask(definer_hl, hdr.field); \
+ (fc)->bit_off = __mlx5_dw_bit_off(definer_hl, hdr.field); \
+ (fc)->byte_off = MLX5_BYTE_OFF(definer_hl, hdr.field); \
+ } while (0)
+
+/* Helper to calculate data used by DR_SET */
+#define DR_CALC_SET(fc, hdr, field, is_inner) \
+ do { \
+ if (is_inner) { \
+ DR_CALC_SET_HDR(fc, hdr##_inner, field); \
+ } else { \
+ DR_CALC_SET_HDR(fc, hdr##_outer, field); \
+ } \
+ } while (0)
+
+ #define DR_GET(typ, p, fld) \
+ ((rte_be_to_cpu_32(*((const rte_be32_t *)(p) + \
+ __mlx5_dw_off(typ, fld))) >> __mlx5_dw_bit_off(typ, fld)) & \
+ __mlx5_mask(typ, fld))
+
+struct mlx5dr_definer_sel_ctrl {
+ uint8_t allowed_full_dw; /* Full DW selectors cover all offsets */
+ uint8_t allowed_lim_dw; /* Limited DW selectors cover offset < 64 */
+ uint8_t allowed_bytes; /* Bytes selectors, up to offset 255 */
+ uint8_t used_full_dw;
+ uint8_t used_lim_dw;
+ uint8_t used_bytes;
+ uint8_t full_dw_selector[DW_SELECTORS];
[v5 12/18] net/mlx5/hws: Add HWS context object
Context is the first mlx5dr object created, all sub object:
table, matcher, rule, action are created using the context.
The context holds the capabilities and send queues used for
configuring the offloads to the HW.
Signed-off-by: Alex Vesker
---
drivers/net/mlx5/hws/mlx5dr_context.c | 223 ++
drivers/net/mlx5/hws/mlx5dr_context.h | 40 +
2 files changed, 263 insertions(+)
create mode 100644 drivers/net/mlx5/hws/mlx5dr_context.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_context.h
diff --git a/drivers/net/mlx5/hws/mlx5dr_context.c
b/drivers/net/mlx5/hws/mlx5dr_context.c
new file mode 100644
index 00..ae86694a51
--- /dev/null
+++ b/drivers/net/mlx5/hws/mlx5dr_context.c
@@ -0,0 +1,223 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2022 NVIDIA Corporation & Affiliates
+ */
+
+#include "mlx5dr_internal.h"
+
+static int mlx5dr_context_pools_init(struct mlx5dr_context *ctx)
+{
+ struct mlx5dr_pool_attr pool_attr = {0};
+ uint8_t max_log_sz;
+ int i;
+
+ if (mlx5dr_pat_init_pattern_cache(&ctx->pattern_cache))
+ return rte_errno;
+
+ /* Create an STC pool per FT type */
+ pool_attr.pool_type = MLX5DR_POOL_TYPE_STC;
+ pool_attr.flags = MLX5DR_POOL_FLAGS_FOR_STC_POOL;
+ max_log_sz = RTE_MIN(MLX5DR_POOL_STC_LOG_SZ,
ctx->caps->stc_alloc_log_max);
+ pool_attr.alloc_log_sz = RTE_MAX(max_log_sz,
ctx->caps->stc_alloc_log_gran);
+
+ for (i = 0; i < MLX5DR_TABLE_TYPE_MAX; i++) {
+ pool_attr.table_type = i;
+ ctx->stc_pool[i] = mlx5dr_pool_create(ctx, &pool_attr);
+ if (!ctx->stc_pool[i]) {
+ DR_LOG(ERR, "Failed to allocate STC pool [%d]", i);
+ goto free_stc_pools;
+ }
+ }
+
+ return 0;
+
+free_stc_pools:
+ for (i = 0; i < MLX5DR_TABLE_TYPE_MAX; i++)
+ if (ctx->stc_pool[i])
+ mlx5dr_pool_destroy(ctx->stc_pool[i]);
+
+ mlx5dr_pat_uninit_pattern_cache(ctx->pattern_cache);
+
+ return rte_errno;
+}
+
+static void mlx5dr_context_pools_uninit(struct mlx5dr_context *ctx)
+{
+ int i;
+
+ mlx5dr_pat_uninit_pattern_cache(ctx->pattern_cache);
+
+ for (i = 0; i < MLX5DR_TABLE_TYPE_MAX; i++) {
+ if (ctx->stc_pool[i])
+ mlx5dr_pool_destroy(ctx->stc_pool[i]);
+ }
+}
+
+static int mlx5dr_context_init_pd(struct mlx5dr_context *ctx,
+ struct ibv_pd *pd)
+{
+ struct mlx5dv_pd mlx5_pd = {0};
+ struct mlx5dv_obj obj;
+ int ret;
+
+ if (pd) {
+ ctx->pd = pd;
+ } else {
+ ctx->pd = mlx5_glue->alloc_pd(ctx->ibv_ctx);
+ if (!ctx->pd) {
+ DR_LOG(ERR, "Failed to allocate PD");
+ rte_errno = errno;
+ return rte_errno;
+ }
+ ctx->flags |= MLX5DR_CONTEXT_FLAG_PRIVATE_PD;
+ }
+
+ obj.pd.in = ctx->pd;
+ obj.pd.out = &mlx5_pd;
+
+ ret = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_PD);
+ if (ret)
+ goto free_private_pd;
+
+ ctx->pd_num = mlx5_pd.pdn;
+
+ return 0;
+
+free_private_pd:
+ if (ctx->flags & MLX5DR_CONTEXT_FLAG_PRIVATE_PD)
+ mlx5_glue->dealloc_pd(ctx->pd);
+
+ return ret;
+}
+
+static int mlx5dr_context_uninit_pd(struct mlx5dr_context *ctx)
+{
+ if (ctx->flags & MLX5DR_CONTEXT_FLAG_PRIVATE_PD)
+ return mlx5_glue->dealloc_pd(ctx->pd);
+
+ return 0;
+}
+
+static void mlx5dr_context_check_hws_supp(struct mlx5dr_context *ctx)
+{
+ struct mlx5dr_cmd_query_caps *caps = ctx->caps;
+
+ /* HWS not supported on device / FW */
+ if (!caps->wqe_based_update) {
+ DR_LOG(INFO, "Required HWS WQE based insertion cap not
supported");
+ return;
+ }
+
+ /* Current solution requires all rules to set reparse bit */
+ if ((!caps->nic_ft.reparse || !caps->fdb_ft.reparse) ||
+ !IS_BIT_SET(caps->rtc_reparse_mode, MLX5_IFC_RTC_REPARSE_ALWAYS)) {
+ DR_LOG(INFO, "Required HWS reparse cap not supported");
+ return;
+ }
+
+ /* FW/HW must support 8DW STE */
+ if (!IS_BIT_SET(caps->ste_format, MLX5_IFC_RTC_STE_FORMAT_8DW)) {
+ DR_LOG(INFO, "Required HWS STE format not supported");
+ return;
+ }
+
+ /* Adding rules by hash and by offset are requirements */
+ if (!IS_BIT_SET(caps->rtc_index_mode,
MLX5_IFC_RTC_STE_UPDATE_MODE_BY_HASH) ||
+ !IS_BIT_SET(caps->rtc_index_mode,
MLX5_IFC_RTC_STE_UPDATE_MODE_BY_OFFSET)) {
+
[v5 13/18] net/mlx5/hws: Add HWS table object
HWS table resides under the context object, each context can
have multiple tables with different steering types RX/TX/FDB.
The table is not only a logical object but it is also represented
in the HW, packets can be steered to the table and from there
to other tables.
Signed-off-by: Erez Shitrit
Signed-off-by: Alex Vesker
---
drivers/net/mlx5/hws/mlx5dr_table.c | 248
drivers/net/mlx5/hws/mlx5dr_table.h | 44 +
2 files changed, 292 insertions(+)
create mode 100644 drivers/net/mlx5/hws/mlx5dr_table.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_table.h
diff --git a/drivers/net/mlx5/hws/mlx5dr_table.c
b/drivers/net/mlx5/hws/mlx5dr_table.c
new file mode 100644
index 00..d3f77e4780
--- /dev/null
+++ b/drivers/net/mlx5/hws/mlx5dr_table.c
@@ -0,0 +1,248 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2022 NVIDIA Corporation & Affiliates
+ */
+
+#include "mlx5dr_internal.h"
+
+static void mlx5dr_table_init_next_ft_attr(struct mlx5dr_table *tbl,
+ struct mlx5dr_cmd_ft_create_attr
*ft_attr)
+{
+ ft_attr->type = tbl->fw_ft_type;
+ if (tbl->type == MLX5DR_TABLE_TYPE_FDB)
+ ft_attr->level = tbl->ctx->caps->fdb_ft.max_level - 1;
+ else
+ ft_attr->level = tbl->ctx->caps->nic_ft.max_level - 1;
+ ft_attr->rtc_valid = true;
+}
+
+/* Call this under ctx->ctrl_lock */
+static int
+mlx5dr_table_up_default_fdb_miss_tbl(struct mlx5dr_table *tbl)
+{
+ struct mlx5dr_cmd_ft_create_attr ft_attr = {0};
+ struct mlx5dr_cmd_forward_tbl *default_miss;
+ struct mlx5dr_context *ctx = tbl->ctx;
+ uint8_t tbl_type = tbl->type;
+ uint32_t vport;
+
+ if (tbl->type != MLX5DR_TABLE_TYPE_FDB)
+ return 0;
+
+ if (ctx->common_res[tbl_type].default_miss) {
+ ctx->common_res[tbl_type].default_miss->refcount++;
+ return 0;
+ }
+
+ ft_attr.type = tbl->fw_ft_type;
+ ft_attr.level = tbl->ctx->caps->fdb_ft.max_level; /* The last level */
+ ft_attr.rtc_valid = false;
+
+ assert(ctx->caps->eswitch_manager);
+ vport = ctx->caps->eswitch_manager_vport_number;
+
+ default_miss = mlx5dr_cmd_miss_ft_create(ctx->ibv_ctx, &ft_attr, vport);
+ if (!default_miss) {
+ DR_LOG(ERR, "Failed to default miss table type: 0x%x",
tbl_type);
+ return rte_errno;
+ }
+
+ ctx->common_res[tbl_type].default_miss = default_miss;
+ ctx->common_res[tbl_type].default_miss->refcount++;
+ return 0;
+}
+
+/* Called under pthread_spin_lock(&ctx->ctrl_lock) */
+static void mlx5dr_table_down_default_fdb_miss_tbl(struct mlx5dr_table *tbl)
+{
+ struct mlx5dr_cmd_forward_tbl *default_miss;
+ struct mlx5dr_context *ctx = tbl->ctx;
+ uint8_t tbl_type = tbl->type;
+
+ if (tbl->type != MLX5DR_TABLE_TYPE_FDB)
+ return;
+
+ default_miss = ctx->common_res[tbl_type].default_miss;
+ if (--default_miss->refcount)
+ return;
+
+ mlx5dr_cmd_miss_ft_destroy(default_miss);
+
+ simple_free(default_miss);
+ ctx->common_res[tbl_type].default_miss = NULL;
+}
+
+static int
+mlx5dr_table_connect_to_default_miss_tbl(struct mlx5dr_table *tbl,
+struct mlx5dr_devx_obj *ft)
+{
+ struct mlx5dr_cmd_ft_modify_attr ft_attr = {0};
+ int ret;
+
+ assert(tbl->type == MLX5DR_TABLE_TYPE_FDB);
+
+ mlx5dr_cmd_set_attr_connect_miss_tbl(tbl->ctx,
+tbl->fw_ft_type,
+tbl->type,
+&ft_attr);
+
+ /* Connect to next */
+ ret = mlx5dr_cmd_flow_table_modify(ft, &ft_attr);
+ if (ret) {
+ DR_LOG(ERR, "Failed to connect FT to default FDB FT");
+ return errno;
+ }
+
+ return 0;
+}
+
+struct mlx5dr_devx_obj *
+mlx5dr_table_create_default_ft(struct mlx5dr_table *tbl)
+{
+ struct mlx5dr_cmd_ft_create_attr ft_attr = {0};
+ struct mlx5dr_devx_obj *ft_obj;
+ int ret;
+
+ mlx5dr_table_init_next_ft_attr(tbl, &ft_attr);
+
+ ft_obj = mlx5dr_cmd_flow_table_create(tbl->ctx->ibv_ctx, &ft_attr);
+ if (ft_obj && tbl->type == MLX5DR_TABLE_TYPE_FDB) {
+ /* Take/create ref over the default miss */
+ ret = mlx5dr_table_up_default_fdb_miss_tbl(tbl);
+ if (ret) {
+ DR_LOG(ERR, "Failed to get default fdb miss");
+ goto free_ft_obj;
+ }
+ ret = mlx5dr_table_connect_to_default_miss_tbl(tbl, ft_obj);
+
[v5 14/18] net/mlx5/hws: Add HWS matcher object
HWS matcher resides under the table object, each table can
have multiple chained matcher with different attributes. Each
matcher represents a combination of match and action templates.
Each matcher can contain multiple configurations based on the
templates. Packets are steered from the table to the matcher
and from there to other objects. The matcher allows efficent
HW packet field matching and action execution based on the
configuration done to it.
Signed-off-by: Alex Vesker
---
drivers/common/mlx5/linux/meson.build | 2 +
drivers/net/mlx5/hws/mlx5dr_matcher.c | 919 ++
drivers/net/mlx5/hws/mlx5dr_matcher.h | 76 +++
3 files changed, 997 insertions(+)
create mode 100644 drivers/net/mlx5/hws/mlx5dr_matcher.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_matcher.h
diff --git a/drivers/common/mlx5/linux/meson.build
b/drivers/common/mlx5/linux/meson.build
index e77b46d157..e8b9a07db5 100644
--- a/drivers/common/mlx5/linux/meson.build
+++ b/drivers/common/mlx5/linux/meson.build
@@ -74,6 +74,8 @@ has_member_args = [
'struct ibv_counters_init_attr', 'comp_mask' ],
[ 'HAVE_MLX5DV_DEVX_UAR_OFFSET', 'infiniband/mlx5dv.h',
'struct mlx5dv_devx_uar', 'mmap_off' ],
+[ 'HAVE_MLX5DV_FLOW_MATCHER_FT_TYPE', 'infiniband/mlx5dv.h',
+'struct mlx5dv_flow_matcher_attr', 'ft_type' ],
]
# input array for meson symbol search:
# [ "MACRO to define if found", "header for the search",
diff --git a/drivers/net/mlx5/hws/mlx5dr_matcher.c
b/drivers/net/mlx5/hws/mlx5dr_matcher.c
new file mode 100644
index 00..d1205c42fa
--- /dev/null
+++ b/drivers/net/mlx5/hws/mlx5dr_matcher.c
@@ -0,0 +1,919 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2022 NVIDIA Corporation & Affiliates
+ */
+
+#include "mlx5dr_internal.h"
+
+static bool mlx5dr_matcher_requires_col_tbl(uint8_t log_num_of_rules)
+{
+ /* Collision table concatenation is done only for large rule tables */
+ return log_num_of_rules > MLX5DR_MATCHER_ASSURED_RULES_TH;
+}
+
+static uint8_t mlx5dr_matcher_rules_to_tbl_depth(uint8_t log_num_of_rules)
+{
+ if (mlx5dr_matcher_requires_col_tbl(log_num_of_rules))
+ return MLX5DR_MATCHER_ASSURED_MAIN_TBL_DEPTH;
+
+ /* For small rule tables we use a single deep table to assure insertion
*/
+ return RTE_MIN(log_num_of_rules, MLX5DR_MATCHER_ASSURED_COL_TBL_DEPTH);
+}
+
+static int mlx5dr_matcher_create_end_ft(struct mlx5dr_matcher *matcher)
+{
+ struct mlx5dr_table *tbl = matcher->tbl;
+
+ matcher->end_ft = mlx5dr_table_create_default_ft(tbl);
+ if (!matcher->end_ft) {
+ DR_LOG(ERR, "Failed to create matcher end flow table");
+ return rte_errno;
+ }
+ return 0;
+}
+
+static void mlx5dr_matcher_destroy_end_ft(struct mlx5dr_matcher *matcher)
+{
+ mlx5dr_table_destroy_default_ft(matcher->tbl, matcher->end_ft);
+}
+
+static int mlx5dr_matcher_connect(struct mlx5dr_matcher *matcher)
+{
+ struct mlx5dr_cmd_ft_modify_attr ft_attr = {0};
+ struct mlx5dr_table *tbl = matcher->tbl;
+ struct mlx5dr_matcher *prev = NULL;
+ struct mlx5dr_matcher *next = NULL;
+ struct mlx5dr_matcher *tmp_matcher;
+ struct mlx5dr_devx_obj *ft;
+ int ret;
+
+ /* Find location in matcher list */
+ if (LIST_EMPTY(&tbl->head)) {
+ LIST_INSERT_HEAD(&tbl->head, matcher, next);
+ goto connect;
+ }
+
+ LIST_FOREACH(tmp_matcher, &tbl->head, next) {
+ if (tmp_matcher->attr.priority > matcher->attr.priority) {
+ next = tmp_matcher;
+ break;
+ }
+ prev = tmp_matcher;
+ }
+
+ if (next)
+ LIST_INSERT_BEFORE(next, matcher, next);
+ else
+ LIST_INSERT_AFTER(prev, matcher, next);
+
+connect:
+ ft_attr.modify_fs = MLX5_IFC_MODIFY_FLOW_TABLE_RTC_ID;
+ ft_attr.type = tbl->fw_ft_type;
+
+ /* Connect to next */
+ if (next) {
+ if (next->match_ste.rtc_0)
+ ft_attr.rtc_id_0 = next->match_ste.rtc_0->id;
+ if (next->match_ste.rtc_1)
+ ft_attr.rtc_id_1 = next->match_ste.rtc_1->id;
+
+ ret = mlx5dr_cmd_flow_table_modify(matcher->end_ft, &ft_attr);
+ if (ret) {
+ DR_LOG(ERR, "Failed to connect new matcher to next
RTC");
+ goto remove_from_list;
+ }
+ }
+
+ /* Connect to previous */
+ ft = prev ? prev->end_ft : tbl->ft;
+
+ if (matcher->match_ste.rtc_0)
+ ft_attr.rtc_id_0 = matcher->match_ste.rtc_0->
[v5 15/18] net/mlx5/hws: Add HWS rule object
HWS rule objects reside under the matcher, each rule holds
the configuration for the packet fields to match on and the
set of actions to execute over the packet that has the requested
fields. Rules can be created asynchronously in parallel over multiple
queues to different matchers. Each rule is configured to the HW.
Signed-off-by: Erez Shitrit
Signed-off-by: Alex Vesker
---
drivers/net/mlx5/hws/mlx5dr_rule.c | 528 +
drivers/net/mlx5/hws/mlx5dr_rule.h | 50 +++
2 files changed, 578 insertions(+)
create mode 100644 drivers/net/mlx5/hws/mlx5dr_rule.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_rule.h
diff --git a/drivers/net/mlx5/hws/mlx5dr_rule.c
b/drivers/net/mlx5/hws/mlx5dr_rule.c
new file mode 100644
index 00..b27318e6d4
--- /dev/null
+++ b/drivers/net/mlx5/hws/mlx5dr_rule.c
@@ -0,0 +1,528 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2022 NVIDIA Corporation & Affiliates
+ */
+
+#include "mlx5dr_internal.h"
+
+static void mlx5dr_rule_skip(struct mlx5dr_matcher *matcher,
+const struct rte_flow_item *items,
+bool *skip_rx, bool *skip_tx)
+{
+ struct mlx5dr_match_template *mt = matcher->mt[0];
+ const struct flow_hw_port_info *vport;
+ const struct rte_flow_item_ethdev *v;
+
+ /* Flow_src is the 1st priority */
+ if (matcher->attr.optimize_flow_src) {
+ *skip_tx = matcher->attr.optimize_flow_src ==
MLX5DR_MATCHER_FLOW_SRC_WIRE;
+ *skip_rx = matcher->attr.optimize_flow_src ==
MLX5DR_MATCHER_FLOW_SRC_VPORT;
+ return;
+ }
+
+ /* By default FDB rules are added to both RX and TX */
+ *skip_rx = false;
+ *skip_tx = false;
+
+ if (mt->item_flags & MLX5_FLOW_ITEM_REPRESENTED_PORT) {
+ v = items[mt->vport_item_id].spec;
+ vport = flow_hw_conv_port_id(v->port_id);
+ if (unlikely(!vport)) {
+ DR_LOG(ERR, "Fail to map port ID %d, ignoring",
v->port_id);
+ return;
+ }
+
+ if (!vport->is_wire)
+ /* Match vport ID is not WIRE -> Skip RX */
+ *skip_rx = true;
+ else
+ /* Match vport ID is WIRE -> Skip TX */
+ *skip_tx = true;
+ }
+}
+
+static void mlx5dr_rule_init_dep_wqe(struct mlx5dr_send_ring_dep_wqe *dep_wqe,
+struct mlx5dr_rule *rule,
+const struct rte_flow_item *items,
+void *user_data)
+{
+ struct mlx5dr_matcher *matcher = rule->matcher;
+ struct mlx5dr_table *tbl = matcher->tbl;
+ bool skip_rx, skip_tx;
+
+ dep_wqe->rule = rule;
+ dep_wqe->user_data = user_data;
+
+ switch (tbl->type) {
+ case MLX5DR_TABLE_TYPE_NIC_RX:
+ case MLX5DR_TABLE_TYPE_NIC_TX:
+ dep_wqe->rtc_0 = matcher->match_ste.rtc_0->id;
+ dep_wqe->retry_rtc_0 = matcher->col_matcher ?
+
matcher->col_matcher->match_ste.rtc_0->id : 0;
+ dep_wqe->rtc_1 = 0;
+ dep_wqe->retry_rtc_1 = 0;
+ break;
+
+ case MLX5DR_TABLE_TYPE_FDB:
+ mlx5dr_rule_skip(matcher, items, &skip_rx, &skip_tx);
+
+ if (!skip_rx) {
+ dep_wqe->rtc_0 = matcher->match_ste.rtc_0->id;
+ dep_wqe->retry_rtc_0 = matcher->col_matcher ?
+
matcher->col_matcher->match_ste.rtc_0->id : 0;
+ } else {
+ dep_wqe->rtc_0 = 0;
+ dep_wqe->retry_rtc_0 = 0;
+ }
+
+ if (!skip_tx) {
+ dep_wqe->rtc_1 = matcher->match_ste.rtc_1->id;
+ dep_wqe->retry_rtc_1 = matcher->col_matcher ?
+
matcher->col_matcher->match_ste.rtc_1->id : 0;
+ } else {
+ dep_wqe->rtc_1 = 0;
+ dep_wqe->retry_rtc_1 = 0;
+ }
+
+ break;
+
+ default:
+ assert(false);
+ break;
+ }
+}
+
+static void mlx5dr_rule_gen_comp(struct mlx5dr_send_engine *queue,
+struct mlx5dr_rule *rule,
+bool err,
+void *user_data,
+enum mlx5dr_rule_status rule_status_on_succ)
+{
+ enum rte_flow_op_status comp_status;
+
+ if (!err) {
+ comp_status = RTE_FLOW_OP_SUCCESS;
+ rule->status = rule_status_on_succ;
[v5 16/18] net/mlx5/hws: Add HWS action object
From: Erez Shitrit
Action objects are used for executing different HW actions
over packets. Each action contains the HW resources and parameters
needed for action use over the HW when creating a rule.
Signed-off-by: Erez Shitrit
Signed-off-by: Alex Vesker
---
drivers/net/mlx5/hws/mlx5dr_action.c | 2237 +
drivers/net/mlx5/hws/mlx5dr_action.h | 253 +++
drivers/net/mlx5/hws/mlx5dr_pat_arg.c | 511 ++
drivers/net/mlx5/hws/mlx5dr_pat_arg.h | 83 +
4 files changed, 3084 insertions(+)
create mode 100644 drivers/net/mlx5/hws/mlx5dr_action.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_action.h
create mode 100644 drivers/net/mlx5/hws/mlx5dr_pat_arg.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_pat_arg.h
diff --git a/drivers/net/mlx5/hws/mlx5dr_action.c
b/drivers/net/mlx5/hws/mlx5dr_action.c
new file mode 100644
index 00..ea43383a33
--- /dev/null
+++ b/drivers/net/mlx5/hws/mlx5dr_action.c
@@ -0,0 +1,2237 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2022 NVIDIA Corporation & Affiliates
+ */
+
+#include "mlx5dr_internal.h"
+
+#define WIRE_PORT 0x
+
+#define MLX5DR_ACTION_METER_INIT_COLOR_OFFSET 1
+
+/* This is the maximum allowed action order for each table type:
+ * TX: POP_VLAN, CTR, ASO_METER, AS_CT, PUSH_VLAN, MODIFY, ENCAP, Term
+ * RX: TAG, DECAP, POP_VLAN, CTR, ASO_METER, ASO_CT, PUSH_VLAN, MODIFY,
+ * ENCAP, Term
+ * FDB: DECAP, POP_VLAN, CTR, ASO_METER, ASO_CT, PUSH_VLAN, MODIFY,
+ * ENCAP, Term
+ */
+static const uint32_t
action_order_arr[MLX5DR_TABLE_TYPE_MAX][MLX5DR_ACTION_TYP_MAX] = {
+ [MLX5DR_TABLE_TYPE_NIC_RX] = {
+ BIT(MLX5DR_ACTION_TYP_TAG),
+ BIT(MLX5DR_ACTION_TYP_TNL_L2_TO_L2) |
+ BIT(MLX5DR_ACTION_TYP_TNL_L3_TO_L2),
+ BIT(MLX5DR_ACTION_TYP_POP_VLAN),
+ BIT(MLX5DR_ACTION_TYP_POP_VLAN),
+ BIT(MLX5DR_ACTION_TYP_CTR),
+ BIT(MLX5DR_ACTION_TYP_ASO_METER),
+ BIT(MLX5DR_ACTION_TYP_ASO_CT),
+ BIT(MLX5DR_ACTION_TYP_PUSH_VLAN),
+ BIT(MLX5DR_ACTION_TYP_PUSH_VLAN),
+ BIT(MLX5DR_ACTION_TYP_MODIFY_HDR),
+ BIT(MLX5DR_ACTION_TYP_L2_TO_TNL_L2) |
+ BIT(MLX5DR_ACTION_TYP_L2_TO_TNL_L3),
+ BIT(MLX5DR_ACTION_TYP_FT) |
+ BIT(MLX5DR_ACTION_TYP_MISS) |
+ BIT(MLX5DR_ACTION_TYP_TIR) |
+ BIT(MLX5DR_ACTION_TYP_DROP),
+ BIT(MLX5DR_ACTION_TYP_LAST),
+ },
+ [MLX5DR_TABLE_TYPE_NIC_TX] = {
+ BIT(MLX5DR_ACTION_TYP_POP_VLAN),
+ BIT(MLX5DR_ACTION_TYP_POP_VLAN),
+ BIT(MLX5DR_ACTION_TYP_CTR),
+ BIT(MLX5DR_ACTION_TYP_ASO_METER),
+ BIT(MLX5DR_ACTION_TYP_ASO_CT),
+ BIT(MLX5DR_ACTION_TYP_PUSH_VLAN),
+ BIT(MLX5DR_ACTION_TYP_PUSH_VLAN),
+ BIT(MLX5DR_ACTION_TYP_MODIFY_HDR),
+ BIT(MLX5DR_ACTION_TYP_L2_TO_TNL_L2) |
+ BIT(MLX5DR_ACTION_TYP_L2_TO_TNL_L3),
+ BIT(MLX5DR_ACTION_TYP_FT) |
+ BIT(MLX5DR_ACTION_TYP_MISS) |
+ BIT(MLX5DR_ACTION_TYP_DROP),
+ BIT(MLX5DR_ACTION_TYP_LAST),
+ },
+ [MLX5DR_TABLE_TYPE_FDB] = {
+ BIT(MLX5DR_ACTION_TYP_TNL_L2_TO_L2) |
+ BIT(MLX5DR_ACTION_TYP_TNL_L3_TO_L2),
+ BIT(MLX5DR_ACTION_TYP_POP_VLAN),
+ BIT(MLX5DR_ACTION_TYP_POP_VLAN),
+ BIT(MLX5DR_ACTION_TYP_CTR),
+ BIT(MLX5DR_ACTION_TYP_ASO_METER),
+ BIT(MLX5DR_ACTION_TYP_ASO_CT),
+ BIT(MLX5DR_ACTION_TYP_PUSH_VLAN),
+ BIT(MLX5DR_ACTION_TYP_PUSH_VLAN),
+ BIT(MLX5DR_ACTION_TYP_MODIFY_HDR),
+ BIT(MLX5DR_ACTION_TYP_L2_TO_TNL_L2) |
+ BIT(MLX5DR_ACTION_TYP_L2_TO_TNL_L3),
+ BIT(MLX5DR_ACTION_TYP_FT) |
+ BIT(MLX5DR_ACTION_TYP_MISS) |
+ BIT(MLX5DR_ACTION_TYP_VPORT) |
+ BIT(MLX5DR_ACTION_TYP_DROP),
+ BIT(MLX5DR_ACTION_TYP_LAST),
+ },
+};
+
+static int mlx5dr_action_get_shared_stc_nic(struct mlx5dr_context *ctx,
+ enum mlx5dr_context_shared_stc_type
stc_type,
+ uint8_t tbl_type)
+{
+ struct mlx5dr_cmd_stc_modify_attr stc_attr = {0};
+ struct mlx5dr_action_shared_stc *shared_stc;
+ int ret;
+
+ pthread_spin_lock(&ctx->ctrl_lock);
+ if (ctx->common_res[tbl_type].shared_stc[stc_type]) {
+
rte_atomic32_add(&ctx->common_res[tbl_type].shared_stc[stc_type]->refcount, 1);
+ pthread_spin_unlock(&ctx->ctrl_lock);
+ return 0;
+ }
+
+ shared_stc = simple_calloc(1, sizeof(*shared_stc));
+ if (!shared_stc) {
+ DR_LOG(ERR, &
[v5 18/18] net/mlx5/hws: Enable HWS
Replace stub implenation of HWS with mlx5dr code.
Signed-off-by: Alex Vesker
---
doc/guides/nics/mlx5.rst | 5 +-
doc/guides/rel_notes/release_22_11.rst | 4 +
drivers/common/mlx5/linux/meson.build| 3 +
drivers/net/mlx5/hws/meson.build | 18 +
drivers/net/mlx5/{mlx5_dr.h => hws/mlx5dr.h} | 209 --
drivers/net/mlx5/hws/mlx5dr_internal.h | 93 +
drivers/net/mlx5/meson.build | 7 +-
drivers/net/mlx5/mlx5.c | 6 +-
drivers/net/mlx5/mlx5.h | 7 +-
drivers/net/mlx5/mlx5_devx.c | 2 +-
drivers/net/mlx5/mlx5_dr.c | 383 ---
drivers/net/mlx5/mlx5_flow.c | 2 +
drivers/net/mlx5/mlx5_flow.h | 11 +-
drivers/net/mlx5/mlx5_flow_hw.c | 10 +-
14 files changed, 328 insertions(+), 432 deletions(-)
create mode 100644 drivers/net/mlx5/hws/meson.build
rename drivers/net/mlx5/{mlx5_dr.h => hws/mlx5dr.h} (66%)
create mode 100644 drivers/net/mlx5/hws/mlx5dr_internal.h
delete mode 100644 drivers/net/mlx5/mlx5_dr.c
diff --git a/doc/guides/nics/mlx5.rst b/doc/guides/nics/mlx5.rst
index bb436892a0..303eb17714 100644
--- a/doc/guides/nics/mlx5.rst
+++ b/doc/guides/nics/mlx5.rst
@@ -539,7 +539,10 @@ Limitations
- WQE based high scaling and safer flow insertion/destruction.
- Set ``dv_flow_en`` to 2 in order to enable HW steering.
-- Async queue-based ``rte_flow_q`` APIs supported only.
+- Async queue-based ``rte_flow_async`` APIs supported only.
+- NIC ConnectX-5 and before are not supported.
+- Partial match with item template is not supported.
+- IPv6 5-tuple matching is not supported.
- Match on GRE header supports the following fields:
diff --git a/doc/guides/rel_notes/release_22_11.rst
b/doc/guides/rel_notes/release_22_11.rst
index a3700bbb34..eed7acc838 100644
--- a/doc/guides/rel_notes/release_22_11.rst
+++ b/doc/guides/rel_notes/release_22_11.rst
@@ -237,6 +237,10 @@ New Features
sysfs entries to adjust the minimum and maximum uncore frequency values,
which works on Linux with Intel hardware only.
+* **Updated Nvidia mlx5 driver.**
+
+ * Added fully support for queue based async HW steering to the PMD.
+
* **Rewritten pmdinfo script.**
The ``dpdk-pmdinfo.py`` script was rewritten to produce valid JSON only.
diff --git a/drivers/common/mlx5/linux/meson.build
b/drivers/common/mlx5/linux/meson.build
index e8b9a07db5..2c69c5e546 100644
--- a/drivers/common/mlx5/linux/meson.build
+++ b/drivers/common/mlx5/linux/meson.build
@@ -8,6 +8,7 @@ dlopen_ibverbs = (get_option('ibverbs_link') == 'dlopen')
LIB_GLUE_BASE = 'librte_common_mlx5_glue.so'
LIB_GLUE_VERSION = abi_version
LIB_GLUE = LIB_GLUE_BASE + '.' + LIB_GLUE_VERSION
+MLX5_HAVE_IBV_FLOW_DV_SUPPORT = false
if dlopen_ibverbs
dpdk_conf.set('RTE_IBVERBS_LINK_DLOPEN', 1)
cflags += [
@@ -231,6 +232,8 @@ foreach arg:has_member_args
endforeach
configure_file(output : 'mlx5_autoconf.h', configuration : config)
+MLX5_HAVE_IBV_FLOW_DV_SUPPORT=config.get('HAVE_IBV_FLOW_DV_SUPPORT')
+
# Build Glue Library
if dlopen_ibverbs
dlopen_name = 'mlx5_glue'
diff --git a/drivers/net/mlx5/hws/meson.build b/drivers/net/mlx5/hws/meson.build
new file mode 100644
index 00..f94798dd2d
--- /dev/null
+++ b/drivers/net/mlx5/hws/meson.build
@@ -0,0 +1,18 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2022 NVIDIA Corporation & Affiliates
+
+includes += include_directories('.')
+sources += files(
+ 'mlx5dr_context.c',
+ 'mlx5dr_table.c',
+ 'mlx5dr_matcher.c',
+ 'mlx5dr_rule.c',
+ 'mlx5dr_action.c',
+ 'mlx5dr_buddy.c',
+ 'mlx5dr_pool.c',
+ 'mlx5dr_cmd.c',
+ 'mlx5dr_send.c',
+ 'mlx5dr_definer.c',
+ 'mlx5dr_debug.c',
+ 'mlx5dr_pat_arg.c',
+)
diff --git a/drivers/net/mlx5/mlx5_dr.h b/drivers/net/mlx5/hws/mlx5dr.h
similarity index 66%
rename from drivers/net/mlx5/mlx5_dr.h
rename to drivers/net/mlx5/hws/mlx5dr.h
index d0b2c15652..664dadbcde 100644
--- a/drivers/net/mlx5/mlx5_dr.h
+++ b/drivers/net/mlx5/hws/mlx5dr.h
@@ -1,9 +1,9 @@
/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright (c) 2021 NVIDIA CORPORATION. All rights reserved.
+ * Copyright (c) 2022 NVIDIA Corporation & Affiliates
*/
-#ifndef MLX5_DR_H_
-#define MLX5_DR_H_
+#ifndef MLX5DR_H_
+#define MLX5DR_H_
#include
@@ -11,6 +11,7 @@ struct mlx5dr_context;
struct mlx5dr_table;
struct mlx5dr_matcher;
struct mlx5dr_rule;
+struct ibv_context;
enum mlx5dr_table_type {
MLX5DR_TABLE_TYPE_NIC_RX,
@@ -26,6 +27,27 @@ enum mlx5dr_matcher_resource_mode {
MLX5DR_MATCHER_RESOURCE_MODE_HTABLE,
[v5 17/18] net/mlx5/hws: Add HWS debug layer
From: Hamdan Igbaria
The debug layer is used to generate a debug CSV file
containing details of the context, table, matcher, rules
and other useful debug information.
Signed-off-by: Hamdan Igbaria
Signed-off-by: Alex Vesker
---
drivers/net/mlx5/hws/mlx5dr_debug.c | 462
drivers/net/mlx5/hws/mlx5dr_debug.h | 28 ++
2 files changed, 490 insertions(+)
create mode 100644 drivers/net/mlx5/hws/mlx5dr_debug.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_debug.h
diff --git a/drivers/net/mlx5/hws/mlx5dr_debug.c
b/drivers/net/mlx5/hws/mlx5dr_debug.c
new file mode 100644
index 00..890a761c48
--- /dev/null
+++ b/drivers/net/mlx5/hws/mlx5dr_debug.c
@@ -0,0 +1,462 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2022 NVIDIA Corporation & Affiliates
+ */
+
+#include "mlx5dr_internal.h"
+
+const char *mlx5dr_debug_action_type_str[] = {
+ [MLX5DR_ACTION_TYP_LAST] = "LAST",
+ [MLX5DR_ACTION_TYP_TNL_L2_TO_L2] = "TNL_L2_TO_L2",
+ [MLX5DR_ACTION_TYP_L2_TO_TNL_L2] = "L2_TO_TNL_L2",
+ [MLX5DR_ACTION_TYP_TNL_L3_TO_L2] = "TNL_L3_TO_L2",
+ [MLX5DR_ACTION_TYP_L2_TO_TNL_L3] = "L2_TO_TNL_L3",
+ [MLX5DR_ACTION_TYP_DROP] = "DROP",
+ [MLX5DR_ACTION_TYP_TIR] = "TIR",
+ [MLX5DR_ACTION_TYP_FT] = "FT",
+ [MLX5DR_ACTION_TYP_CTR] = "CTR",
+ [MLX5DR_ACTION_TYP_TAG] = "TAG",
+ [MLX5DR_ACTION_TYP_MODIFY_HDR] = "MODIFY_HDR",
+ [MLX5DR_ACTION_TYP_VPORT] = "VPORT",
+ [MLX5DR_ACTION_TYP_MISS] = "DEFAULT_MISS",
+ [MLX5DR_ACTION_TYP_POP_VLAN] = "POP_VLAN",
+ [MLX5DR_ACTION_TYP_PUSH_VLAN] = "PUSH_VLAN",
+ [MLX5DR_ACTION_TYP_ASO_METER] = "ASO_METER",
+ [MLX5DR_ACTION_TYP_ASO_CT] = "ASO_CT",
+};
+
+static_assert(ARRAY_SIZE(mlx5dr_debug_action_type_str) ==
MLX5DR_ACTION_TYP_MAX,
+ "Missing mlx5dr_debug_action_type_str");
+
+const char *mlx5dr_debug_action_type_to_str(enum mlx5dr_action_type
action_type)
+{
+ return mlx5dr_debug_action_type_str[action_type];
+}
+
+static int
+mlx5dr_debug_dump_matcher_template_definer(FILE *f,
+ struct mlx5dr_match_template *mt)
+{
+ struct mlx5dr_definer *definer = mt->definer;
+ int i, ret;
+
+ ret = fprintf(f, "%d,0x%" PRIx64 ",0x%" PRIx64 ",%d,%d,",
+ MLX5DR_DEBUG_RES_TYPE_MATCHER_TEMPLATE_DEFINER,
+ (uint64_t)(uintptr_t)definer,
+ (uint64_t)(uintptr_t)mt,
+ definer->obj->id,
+ definer->type);
+ if (ret < 0) {
+ rte_errno = EINVAL;
+ return rte_errno;
+ }
+
+ for (i = 0; i < DW_SELECTORS; i++) {
+ ret = fprintf(f, "0x%x%s", definer->dw_selector[i],
+ (i == DW_SELECTORS - 1) ? "," : "-");
+ if (ret < 0) {
+ rte_errno = EINVAL;
+ return rte_errno;
+ }
+ }
+
+ for (i = 0; i < BYTE_SELECTORS; i++) {
+ ret = fprintf(f, "0x%x%s", definer->byte_selector[i],
+ (i == BYTE_SELECTORS - 1) ? "," : "-");
+ if (ret < 0) {
+ rte_errno = EINVAL;
+ return rte_errno;
+ }
+ }
+
+ for (i = 0; i < MLX5DR_JUMBO_TAG_SZ; i++) {
+ ret = fprintf(f, "%02x", definer->mask.jumbo[i]);
+ if (ret < 0) {
+ rte_errno = EINVAL;
+ return rte_errno;
+ }
+ }
+
+ ret = fprintf(f, "\n");
+ if (ret < 0) {
+ rte_errno = EINVAL;
+ return rte_errno;
+ }
+
+ return 0;
+}
+
+static int
+mlx5dr_debug_dump_matcher_match_template(FILE *f, struct mlx5dr_matcher
*matcher)
+{
+ bool is_root = matcher->tbl->level == MLX5DR_ROOT_LEVEL;
+ int i, ret;
+
+ for (i = 0; i < matcher->num_of_mt; i++) {
+ struct mlx5dr_match_template *mt = matcher->mt[i];
+
+ ret = fprintf(f, "%d,0x%" PRIx64 ",0x%" PRIx64 ",%d,%d\n",
+ MLX5DR_DEBUG_RES_TYPE_MATCHER_MATCH_TEMPLATE,
+ (uint64_t)(uintptr_t)mt,
+ (uint64_t)(uintptr_t)matcher,
+ is_root ? 0 : mt->fc_sz,
+ mt->flags);
+ if (ret < 0) {
+ rte_errno = EINVAL;
+ return rte_errno;
+ }
+
+ i
[v6 00/18] net/mlx5: Add HW steering low level support
Mellanox ConnetX devices supports packet matching, packet modification and
redirection. These functionalities are also referred to as flow-steering.
To configure a steering rule, the rule is written to the device owned
memory, this memory is accessed and cached by the device when processing
a packet.
The highlight of this patchset is supporting HW Steering (HWS) which
is the new technology supported in new ConnectX devices, HWS allows
configuring steering rules directly to the HW using special HW queues
with minimal CPU effort.
This patchset is the internal low layer implementation for HWS used by
the mlx5 PMD. The mlx5dr (direct rule) is layer that bridges between the
PMD and the HW by configuring the HW offloads based on the PMD logic
v2:
Fix check patch and cosmetic changes
v3:
-Fix unsupported items
-Fix compilation with mlx5dv dependency
v4:
-Fix compile on Windows
v5:
-Fix compile on old rdma-core or no rdma core
v6:
-Fix meson style and improve configure
-Checkpatch and compilation fixes
-Fix action number issue
Alex Vesker (8):
net/mlx5: Add additional glue functions for HWS
net/mlx5/hws: Add HWS send layer
net/mlx5/hws: Add HWS definer layer
net/mlx5/hws: Add HWS context object
net/mlx5/hws: Add HWS table object
net/mlx5/hws: Add HWS matcher object
net/mlx5/hws: Add HWS rule object
net/mlx5/hws: Enable HWS
Bing Zhao (2):
common/mlx5: query set capability of registers
net/mlx5: provide the available tag registers
Dariusz Sosnowski (1):
net/mlx5: add port to metadata conversion
Erez Shitrit (3):
net/mlx5/hws: Add HWS command layer
net/mlx5/hws: Add HWS pool and buddy
net/mlx5/hws: Add HWS action object
Hamdan Igbaria (1):
net/mlx5/hws: Add HWS debug layer
Suanming Mou (3):
net/mlx5: split flow item translation
net/mlx5: split flow item matcher and value translation
net/mlx5: add hardware steering item translation function
doc/guides/nics/features/default.ini |1 +
doc/guides/nics/features/mlx5.ini|1 +
doc/guides/nics/mlx5.rst |5 +-
doc/guides/rel_notes/release_22_11.rst |4 +
drivers/common/mlx5/linux/meson.build| 11 +-
drivers/common/mlx5/linux/mlx5_glue.c| 121 +-
drivers/common/mlx5/linux/mlx5_glue.h| 17 +
drivers/common/mlx5/mlx5_devx_cmds.c | 30 +
drivers/common/mlx5/mlx5_devx_cmds.h |2 +
drivers/common/mlx5/mlx5_prm.h | 652 -
drivers/net/mlx5/hws/meson.build | 18 +
drivers/net/mlx5/{mlx5_dr.h => hws/mlx5dr.h} | 201 +-
drivers/net/mlx5/hws/mlx5dr_action.c | 2237 +++
drivers/net/mlx5/hws/mlx5dr_action.h | 253 ++
drivers/net/mlx5/hws/mlx5dr_buddy.c | 200 ++
drivers/net/mlx5/hws/mlx5dr_buddy.h | 22 +
drivers/net/mlx5/hws/mlx5dr_cmd.c| 948 +++
drivers/net/mlx5/hws/mlx5dr_cmd.h| 230 ++
drivers/net/mlx5/hws/mlx5dr_context.c| 223 ++
drivers/net/mlx5/hws/mlx5dr_context.h| 40 +
drivers/net/mlx5/hws/mlx5dr_debug.c | 462
drivers/net/mlx5/hws/mlx5dr_debug.h | 28 +
drivers/net/mlx5/hws/mlx5dr_definer.c| 1968 ++
drivers/net/mlx5/hws/mlx5dr_definer.h| 585
drivers/net/mlx5/hws/mlx5dr_internal.h | 93 +
drivers/net/mlx5/hws/mlx5dr_matcher.c| 919 +++
drivers/net/mlx5/hws/mlx5dr_matcher.h| 76 +
drivers/net/mlx5/hws/mlx5dr_pat_arg.c| 511
drivers/net/mlx5/hws/mlx5dr_pat_arg.h| 83 +
drivers/net/mlx5/hws/mlx5dr_pool.c | 672 +
drivers/net/mlx5/hws/mlx5dr_pool.h | 152 ++
drivers/net/mlx5/hws/mlx5dr_rule.c | 528
drivers/net/mlx5/hws/mlx5dr_rule.h | 50 +
drivers/net/mlx5/hws/mlx5dr_send.c | 844 ++
drivers/net/mlx5/hws/mlx5dr_send.h | 275 ++
drivers/net/mlx5/hws/mlx5dr_table.c | 248 ++
drivers/net/mlx5/hws/mlx5dr_table.h | 44 +
drivers/net/mlx5/linux/mlx5_os.c | 12 +-
drivers/net/mlx5/meson.build |7 +-
drivers/net/mlx5/mlx5.c |9 +-
drivers/net/mlx5/mlx5.h |8 +-
drivers/net/mlx5/mlx5_defs.h |2 +
drivers/net/mlx5/mlx5_devx.c |2 +-
drivers/net/mlx5/mlx5_dr.c | 383 ---
drivers/net/mlx5/mlx5_flow.c | 29 +-
drivers/net/mlx5/mlx5_flow.h | 174 +-
drivers/net/mlx5/mlx5_flow_dv.c | 2567 +-
drivers/net/mlx5/mlx5_flow_hw.c | 115 +-
48 files changed, 14368 insertions(+), 1694 deletions(-)
create mode 100644 drivers/net/mlx5/hws/meson.build
rename drivers/net/mlx5/{mlx5_dr.h => hws/mlx5dr.h} (67%)
create mode 100644 drivers/net/mlx5/hws/mlx5dr_action.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_action.h
create mode 100644 drivers/net/mlx5/hws/mlx5dr_b
[v6 01/18] net/mlx5: split flow item translation
From: Suanming Mou
In order to share the item translation code with hardware steering
mode, this commit splits flow item translation code to a dedicate
function.
Signed-off-by: Suanming Mou
---
drivers/net/mlx5/mlx5_flow_dv.c | 1915 ---
1 file changed, 979 insertions(+), 936 deletions(-)
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 4bdcb1815b..0f3ff4db51 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -13076,8 +13076,7 @@ flow_dv_translate_create_conntrack(struct rte_eth_dev
*dev,
}
/**
- * Fill the flow with DV spec, lock free
- * (mutex should be acquired by caller).
+ * Translate the flow item to matcher.
*
* @param[in] dev
* Pointer to rte_eth_dev structure.
@@ -13087,8 +13086,8 @@ flow_dv_translate_create_conntrack(struct rte_eth_dev
*dev,
* Pointer to the flow attributes.
* @param[in] items
* Pointer to the list of items.
- * @param[in] actions
- * Pointer to the list of actions.
+ * @param[in] matcher
+ * Pointer to the flow matcher.
* @param[out] error
* Pointer to the error structure.
*
@@ -13096,650 +13095,656 @@ flow_dv_translate_create_conntrack(struct
rte_eth_dev *dev,
* 0 on success, a negative errno value otherwise and rte_errno is set.
*/
static int
-flow_dv_translate(struct rte_eth_dev *dev,
- struct mlx5_flow *dev_flow,
- const struct rte_flow_attr *attr,
- const struct rte_flow_item items[],
- const struct rte_flow_action actions[],
- struct rte_flow_error *error)
+flow_dv_translate_items(struct rte_eth_dev *dev,
+ struct mlx5_flow *dev_flow,
+ const struct rte_flow_attr *attr,
+ const struct rte_flow_item items[],
+ struct mlx5_flow_dv_matcher *matcher,
+ struct rte_flow_error *error)
{
struct mlx5_priv *priv = dev->data->dev_private;
- struct mlx5_sh_config *dev_conf = &priv->sh->config;
struct rte_flow *flow = dev_flow->flow;
struct mlx5_flow_handle *handle = dev_flow->handle;
struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
- struct mlx5_flow_rss_desc *rss_desc;
+ struct mlx5_flow_rss_desc *rss_desc = &wks->rss_desc;
uint64_t item_flags = 0;
uint64_t last_item = 0;
- uint64_t action_flags = 0;
- struct mlx5_flow_dv_matcher matcher = {
- .mask = {
- .size = sizeof(matcher.mask.buf),
- },
- };
- int actions_n = 0;
- bool actions_end = false;
- union {
- struct mlx5_flow_dv_modify_hdr_resource res;
- uint8_t len[sizeof(struct mlx5_flow_dv_modify_hdr_resource) +
- sizeof(struct mlx5_modification_cmd) *
- (MLX5_MAX_MODIFY_NUM + 1)];
- } mhdr_dummy;
- struct mlx5_flow_dv_modify_hdr_resource *mhdr_res = &mhdr_dummy.res;
- const struct rte_flow_action_count *count = NULL;
- const struct rte_flow_action_age *non_shared_age = NULL;
- union flow_dv_attr flow_attr = { .attr = 0 };
- uint32_t tag_be;
- union mlx5_flow_tbl_key tbl_key;
- uint32_t modify_action_position = UINT32_MAX;
- void *match_mask = matcher.mask.buf;
+ void *match_mask = matcher->mask.buf;
void *match_value = dev_flow->dv.value.buf;
uint8_t next_protocol = 0xff;
- struct rte_vlan_hdr vlan = { 0 };
- struct mlx5_flow_dv_dest_array_resource mdest_res;
- struct mlx5_flow_dv_sample_resource sample_res;
- void *sample_actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS] = {0};
- const struct rte_flow_action_sample *sample = NULL;
- struct mlx5_flow_sub_actions_list *sample_act;
- uint32_t sample_act_pos = UINT32_MAX;
- uint32_t age_act_pos = UINT32_MAX;
- uint32_t num_of_dest = 0;
- int tmp_actions_n = 0;
- uint32_t table;
- int ret = 0;
- const struct mlx5_flow_tunnel *tunnel = NULL;
- struct flow_grp_info grp_info = {
- .external = !!dev_flow->external,
- .transfer = !!attr->transfer,
- .fdb_def_rule = !!priv->fdb_def_rule,
- .skip_scale = dev_flow->skip_scale &
- (1 << MLX5_SCALE_FLOW_GROUP_BIT),
- .std_tbl_fix = true,
- };
+ uint16_t priority = 0;
const struct rte_flow_item *integrity_items[2] = {NULL, NULL};
const struct rte_flow_item *tunnel_item = NULL;
const struct rte_flow_item *gre_item = NULL;
+ int ret = 0;
- if (!wks)
- return rte_flow_error_set(error, ENOMEM,
- RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
- NULL,
- "failed to pu
[v6 04/18] net/mlx5: add port to metadata conversion
From: Dariusz Sosnowski
This patch initial version of functions used to:
- convert between ethdev port_id and internal tag/mask value,
- convert between IB context and internal tag/mask value.
Signed-off-by: Dariusz Sosnowski
---
drivers/net/mlx5/linux/mlx5_os.c | 10 +-
drivers/net/mlx5/mlx5.c | 1 +
drivers/net/mlx5/mlx5_flow.c | 6
drivers/net/mlx5/mlx5_flow.h | 52
drivers/net/mlx5/mlx5_flow_hw.c | 29 ++
5 files changed, 97 insertions(+), 1 deletion(-)
diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index 3e505d8f4c..d1e7bcce57 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -1554,8 +1554,16 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
if (!priv->hrxqs)
goto error;
rte_rwlock_init(&priv->ind_tbls_lock);
- if (priv->sh->config.dv_flow_en == 2)
+ if (priv->sh->config.dv_flow_en == 2) {
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+ if (priv->vport_meta_mask)
+ flow_hw_set_port_info(eth_dev);
return eth_dev;
+#else
+ DRV_LOG(ERR, "DV support is missing for HWS.");
+ goto error;
+#endif
+ }
/* Port representor shares the same max priority with pf port. */
if (!priv->sh->flow_priority_check_flag) {
/* Supported Verbs flow priority number detection. */
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 752b60d769..1d10932619 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -1944,6 +1944,7 @@ mlx5_dev_close(struct rte_eth_dev *dev)
mlx5_flex_item_port_cleanup(dev);
#if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
flow_hw_resource_release(dev);
+ flow_hw_clear_port_info(dev);
#endif
if (priv->rxq_privs != NULL) {
/* XXX race condition if mlx5_rx_burst() is still running. */
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 026d77b01f..72f4374c07 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -33,6 +33,12 @@
#include "mlx5_common_os.h"
#include "rte_pmd_mlx5.h"
+/*
+ * Shared array for quick translation between port_id and vport mask/values
+ * used for HWS rules.
+ */
+struct flow_hw_port_info mlx5_flow_hw_port_infos[RTE_MAX_ETHPORTS];
+
struct tunnel_default_miss_ctx {
uint16_t *queue;
__extension__
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index 3537eb3d66..c0c719dd8b 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -1326,6 +1326,58 @@ struct mlx5_flow_split_info {
uint64_t prefix_layers; /**< Prefix subflow layers. */
};
+struct flow_hw_port_info {
+ uint32_t regc_mask;
+ uint32_t regc_value;
+ uint32_t is_wire:1;
+};
+
+extern struct flow_hw_port_info mlx5_flow_hw_port_infos[RTE_MAX_ETHPORTS];
+
+/*
+ * Get metadata match tag and mask for given rte_eth_dev port.
+ * Used in HWS rule creation.
+ */
+static __rte_always_inline const struct flow_hw_port_info *
+flow_hw_conv_port_id(const uint16_t port_id)
+{
+ struct flow_hw_port_info *port_info;
+
+ if (port_id >= RTE_MAX_ETHPORTS)
+ return NULL;
+ port_info = &mlx5_flow_hw_port_infos[port_id];
+ return !!port_info->regc_mask ? port_info : NULL;
+}
+
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+/*
+ * Get metadata match tag and mask for the uplink port represented
+ * by given IB context. Used in HWS context creation.
+ */
+static __rte_always_inline const struct flow_hw_port_info *
+flow_hw_get_wire_port(struct ibv_context *ibctx)
+{
+ struct ibv_device *ibdev = ibctx->device;
+ uint16_t port_id;
+
+ MLX5_ETH_FOREACH_DEV(port_id, NULL) {
+ const struct mlx5_priv *priv =
+ rte_eth_devices[port_id].data->dev_private;
+
+ if (priv && priv->master) {
+ struct ibv_context *port_ibctx = priv->sh->cdev->ctx;
+
+ if (port_ibctx->device == ibdev)
+ return flow_hw_conv_port_id(port_id);
+ }
+ }
+ return NULL;
+}
+#endif
+
+void flow_hw_set_port_info(struct rte_eth_dev *dev);
+void flow_hw_clear_port_info(struct rte_eth_dev *dev);
+
typedef int (*mlx5_flow_validate_t)(struct rte_eth_dev *dev,
const struct rte_flow_attr *attr,
const struct rte_flow_item items[],
diff --git a/drivers/net/mlx5/mlx5_flow_hw.c b/drivers/net/mlx5/mlx5_flow_hw.c
index b168ff9e7e..765e5164cb 100644
--- a/drivers/net/mlx5/mlx5_flow_hw.c
+++ b/drivers/net/mlx5/mlx5_flow_hw.c
@@ -2211,6 +2211,35 @@ flow_hw_resource_release(struct rte_eth_dev *dev)
priv->nb_queue = 0;
}
+/* Sets vport tag and mask, for given port, used in HWS ru
[v6 02/18] net/mlx5: split flow item matcher and value translation
From: Suanming Mou
As hardware steering mode translates flow matcher and value in two
different stages, split the flow item matcher and value translation
to help reuse the code.
Signed-off-by: Suanming Mou
---
drivers/net/mlx5/mlx5_flow.h| 32 +
drivers/net/mlx5/mlx5_flow_dv.c | 2314 +++
2 files changed, 1185 insertions(+), 1161 deletions(-)
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index 8e97fa188a..7e5ade52cb 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -1267,6 +1267,38 @@ struct mlx5_flow_workspace {
uint32_t skip_matcher_reg:1;
/* Indicates if need to skip matcher register in translate. */
uint32_t mark:1; /* Indicates if flow contains mark action. */
+ uint32_t vport_meta_tag; /* Used for vport index match. */
+};
+
+/* Matcher translate type. */
+enum MLX5_SET_MATCHER {
+ MLX5_SET_MATCHER_SW_V = 1 << 0,
+ MLX5_SET_MATCHER_SW_M = 1 << 1,
+ MLX5_SET_MATCHER_HS_V = 1 << 2,
+ MLX5_SET_MATCHER_HS_M = 1 << 3,
+};
+
+#define MLX5_SET_MATCHER_SW (MLX5_SET_MATCHER_SW_V | MLX5_SET_MATCHER_SW_M)
+#define MLX5_SET_MATCHER_HS (MLX5_SET_MATCHER_HS_V | MLX5_SET_MATCHER_HS_M)
+#define MLX5_SET_MATCHER_V (MLX5_SET_MATCHER_SW_V | MLX5_SET_MATCHER_HS_V)
+#define MLX5_SET_MATCHER_M (MLX5_SET_MATCHER_SW_M | MLX5_SET_MATCHER_HS_M)
+
+/* Flow matcher workspace intermediate data. */
+struct mlx5_dv_matcher_workspace {
+ uint8_t priority; /* Flow priority. */
+ uint64_t last_item; /* Last item in pattern. */
+ uint64_t item_flags; /* Flow item pattern flags. */
+ uint64_t action_flags; /* Flow action flags. */
+ bool external; /* External flow or not. */
+ uint32_t vlan_tag:12; /* Flow item VLAN tag. */
+ uint8_t next_protocol; /* Tunnel next protocol */
+ uint32_t geneve_tlv_option; /* Flow item Geneve TLV option. */
+ uint32_t group; /* Flow group. */
+ uint16_t udp_dport; /* Flow item UDP port. */
+ const struct rte_flow_attr *attr; /* Flow attribute. */
+ struct mlx5_flow_rss_desc *rss_desc; /* RSS descriptor. */
+ const struct rte_flow_item *tunnel_item; /* Flow tunnel item. */
+ const struct rte_flow_item *gre_item; /* Flow GRE item. */
};
struct mlx5_flow_split_info {
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 0f3ff4db51..944db9c3e4 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -63,6 +63,25 @@
#define MLX5DV_FLOW_VLAN_PCP_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_PCP_MASK)
#define MLX5DV_FLOW_VLAN_VID_MASK_BE RTE_BE16(MLX5DV_FLOW_VLAN_VID_MASK)
+#define MLX5_ITEM_VALID(item, key_type) \
+ (((MLX5_SET_MATCHER_SW & (key_type)) && !((item)->spec)) || \
+((MLX5_SET_MATCHER_HS_V == (key_type)) && !((item)->spec)) || \
+((MLX5_SET_MATCHER_HS_M == (key_type)) && !((item)->mask)))
+
+#define MLX5_ITEM_UPDATE(item, key_type, v, m, gm) \
+ do { \
+ if ((key_type) == MLX5_SET_MATCHER_SW_V) { \
+ v = (item)->spec; \
+ m = (item)->mask ? (item)->mask : (gm); \
+ } else if ((key_type) == MLX5_SET_MATCHER_HS_V) { \
+ v = (item)->spec; \
+ m = (v); \
+ } else { \
+ v = (item)->mask ? (item)->mask : (gm); \
+ m = (v); \
+ } \
+ } while (0)
+
union flow_dv_attr {
struct {
uint32_t valid:1;
@@ -8325,70 +8344,61 @@ flow_dv_check_valid_spec(void *match_mask, void
*match_value)
static inline void
flow_dv_set_match_ip_version(uint32_t group,
void *headers_v,
-void *headers_m,
+uint32_t key_type,
uint8_t ip_version)
{
- if (group == 0)
- MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version, 0xf);
+ if (group == 0 && (key_type & MLX5_SET_MATCHER_M))
+ MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, 0xf);
else
- MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version,
+ MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version,
ip_version);
- MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version, ip_version);
MLX5_SET(fte_match_set_lyr_2_4, headers_v, ethertype, 0);
- MLX5_SET(fte_match_set_lyr_2_4, headers_m, ethertype, 0);
}
/**
- * Add Ethernet item to matcher and to the value.
+ * Add Ethernet item to the value.
*
- * @param[in, out] matcher
- * Flow matcher.
* @param[in, out] key
* Flow matcher value.
* @param[in] item
* Flow pattern to translate.
* @param[in] inner
* Item is inner pattern.
+ * @param[in] grpup
+ * Flow matcher group.
+ * @param[in] key_type
+ * Set flow matcher mask or value.
*/
static voi
[v6 03/18] net/mlx5: add hardware steering item translation function
From: Suanming Mou
As hardware steering root table flows still work under FW steering
mode. This commit provides shared item tranlsation code for hardware
steering root table flows.
Signed-off-by: Suanming Mou
---
drivers/net/mlx5/mlx5_flow.c| 10 +--
drivers/net/mlx5/mlx5_flow.h| 52 -
drivers/net/mlx5/mlx5_flow_dv.c | 134
3 files changed, 155 insertions(+), 41 deletions(-)
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 06b465de7a..026d77b01f 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -7108,7 +7108,7 @@ mlx5_flow_create_devx_sq_miss_flow(struct rte_eth_dev
*dev, uint32_t txq)
struct rte_flow_item_port_id port_spec = {
.id = MLX5_PORT_ESW_MGR,
};
- struct mlx5_rte_flow_item_tx_queue txq_spec = {
+ struct mlx5_rte_flow_item_sq txq_spec = {
.queue = txq,
};
struct rte_flow_item pattern[] = {
@@ -7118,7 +7118,7 @@ mlx5_flow_create_devx_sq_miss_flow(struct rte_eth_dev
*dev, uint32_t txq)
},
{
.type = (enum rte_flow_item_type)
- MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE,
+ MLX5_RTE_FLOW_ITEM_TYPE_SQ,
.spec = &txq_spec,
},
{
@@ -7504,16 +7504,16 @@ mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev,
.egress = 1,
.priority = 0,
};
- struct mlx5_rte_flow_item_tx_queue queue_spec = {
+ struct mlx5_rte_flow_item_sq queue_spec = {
.queue = queue,
};
- struct mlx5_rte_flow_item_tx_queue queue_mask = {
+ struct mlx5_rte_flow_item_sq queue_mask = {
.queue = UINT32_MAX,
};
struct rte_flow_item items[] = {
{
.type = (enum rte_flow_item_type)
- MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE,
+ MLX5_RTE_FLOW_ITEM_TYPE_SQ,
.spec = &queue_spec,
.last = NULL,
.mask = &queue_mask,
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index 7e5ade52cb..3537eb3d66 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -28,7 +28,7 @@
enum mlx5_rte_flow_item_type {
MLX5_RTE_FLOW_ITEM_TYPE_END = INT_MIN,
MLX5_RTE_FLOW_ITEM_TYPE_TAG,
- MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE,
+ MLX5_RTE_FLOW_ITEM_TYPE_SQ,
MLX5_RTE_FLOW_ITEM_TYPE_VLAN,
MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL,
};
@@ -95,7 +95,7 @@ struct mlx5_flow_action_copy_mreg {
};
/* Matches on source queue. */
-struct mlx5_rte_flow_item_tx_queue {
+struct mlx5_rte_flow_item_sq {
uint32_t queue;
};
@@ -159,7 +159,7 @@ enum mlx5_feature_name {
#define MLX5_FLOW_LAYER_GENEVE (1u << 26)
/* Queue items. */
-#define MLX5_FLOW_ITEM_TX_QUEUE (1u << 27)
+#define MLX5_FLOW_ITEM_SQ (1u << 27)
/* Pattern tunnel Layer bits (continued). */
#define MLX5_FLOW_LAYER_GTP (1u << 28)
@@ -196,6 +196,9 @@ enum mlx5_feature_name {
#define MLX5_FLOW_ITEM_PORT_REPRESENTOR (UINT64_C(1) << 41)
#define MLX5_FLOW_ITEM_REPRESENTED_PORT (UINT64_C(1) << 42)
+/* Meter color item */
+#define MLX5_FLOW_ITEM_METER_COLOR (UINT64_C(1) << 44)
+
/* Outer Masks. */
#define MLX5_FLOW_LAYER_OUTER_L3 \
(MLX5_FLOW_LAYER_OUTER_L3_IPV4 | MLX5_FLOW_LAYER_OUTER_L3_IPV6)
@@ -1009,6 +1012,18 @@ flow_items_to_tunnel(const struct rte_flow_item items[])
return items[0].spec;
}
+/* HW steering flow attributes. */
+struct mlx5_flow_attr {
+ uint32_t port_id; /* Port index. */
+ uint32_t group; /* Flow group. */
+ uint32_t priority; /* Original Priority. */
+ /* rss level, used by priority adjustment. */
+ uint32_t rss_level;
+ /* Action flags, used by priority adjustment. */
+ uint32_t act_flags;
+ uint32_t tbl_type; /* Flow table type. */
+};
+
/* Flow structure. */
struct rte_flow {
uint32_t dev_handles;
@@ -1769,6 +1784,32 @@ mlx5_translate_tunnel_etypes(uint64_t pattern_flags)
int flow_hw_q_flow_flush(struct rte_eth_dev *dev,
struct rte_flow_error *error);
+
+/*
+ * Convert rte_mtr_color to mlx5 color.
+ *
+ * @param[in] rcol
+ * rte_mtr_color.
+ *
+ * @return
+ * mlx5 color.
+ */
+static inline int
+rte_col_2_mlx5_col(enum rte_color rcol)
+{
+ switch (rcol) {
+ case RTE_COLOR_GREEN:
+ return MLX5_FLOW_COLOR_GREEN;
+ case RTE_COLOR_YELLOW:
+ return MLX5_FLOW_COLOR_YELLOW;
+ case RTE_COLOR_RED:
+ return MLX5_FLOW_COLOR_RED;
+ default:
+ break;
+ }
+ return MLX5_FLOW_COLOR_UNDEFINED;
+}
+
int mlx5_flow_group_to_table(struct rte_eth_dev *dev,
const struct mlx5_f
[v6 06/18] net/mlx5: provide the available tag registers
From: Bing Zhao
The available tags that can be used by the application are fixed
after startup.
A global array is used to store the information and transfer the TAG
item directly from the ID to the REG_C_x.
Signed-off-by: Bing Zhao
---
drivers/net/mlx5/linux/mlx5_os.c | 2 +
drivers/net/mlx5/mlx5.c | 2 +
drivers/net/mlx5/mlx5.h | 1 +
drivers/net/mlx5/mlx5_defs.h | 2 +
drivers/net/mlx5/mlx5_flow.c | 11 +
drivers/net/mlx5/mlx5_flow.h | 27
drivers/net/mlx5/mlx5_flow_hw.c | 76
7 files changed, 121 insertions(+)
diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index d1e7bcce57..12f503474a 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -1558,6 +1558,8 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
#ifdef HAVE_IBV_FLOW_DV_SUPPORT
if (priv->vport_meta_mask)
flow_hw_set_port_info(eth_dev);
+ /* Only HWS requires this information. */
+ flow_hw_init_tags_set(eth_dev);
return eth_dev;
#else
DRV_LOG(ERR, "DV support is missing for HWS.");
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 1d10932619..b39ef1ecbe 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -1945,6 +1945,8 @@ mlx5_dev_close(struct rte_eth_dev *dev)
#if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
flow_hw_resource_release(dev);
flow_hw_clear_port_info(dev);
+ if (priv->sh->config.dv_flow_en == 2)
+ flow_hw_clear_tags_set(dev);
#endif
if (priv->rxq_privs != NULL) {
/* XXX race condition if mlx5_rx_burst() is still running. */
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index c2c3ed81fa..aa328c3bc9 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -1205,6 +1205,7 @@ struct mlx5_dev_ctx_shared {
uint32_t drop_action_check_flag:1; /* Check Flag for drop action. */
uint32_t flow_priority_check_flag:1; /* Check Flag for flow priority. */
uint32_t metadata_regc_check_flag:1; /* Check Flag for metadata REGC. */
+ uint32_t hws_tags:1; /* Check if tags info for HWS initialized. */
uint32_t max_port; /* Maximal IB device port index. */
struct mlx5_bond_info bond; /* Bonding information. */
struct mlx5_common_device *cdev; /* Backend mlx5 device. */
diff --git a/drivers/net/mlx5/mlx5_defs.h b/drivers/net/mlx5/mlx5_defs.h
index 018d3f0f0c..585afb0a98 100644
--- a/drivers/net/mlx5/mlx5_defs.h
+++ b/drivers/net/mlx5/mlx5_defs.h
@@ -139,6 +139,8 @@
#define MLX5_XMETA_MODE_META32 2
/* Provide info on patrial hw miss. Implies MLX5_XMETA_MODE_META16 */
#define MLX5_XMETA_MODE_MISS_INFO 3
+/* Only valid in HWS, 32bits extended META without MARK support in FDB. */
+#define MLX5_XMETA_MODE_META32_HWS 4
/* Tx accurate scheduling on timestamps parameters. */
#define MLX5_TXPP_WAIT_INIT_TS 1000ul /* How long to wait timestamp. */
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 72f4374c07..1543d7f75e 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -39,6 +39,17 @@
*/
struct flow_hw_port_info mlx5_flow_hw_port_infos[RTE_MAX_ETHPORTS];
+/*
+ * A global structure to save the available REG_C_x for tags usage.
+ * The Meter color REG (ASO) and the last available one will be reserved
+ * for PMD internal usage.
+ * Since there is no "port" concept in the driver, it is assumed that the
+ * available tags set will be the minimum intersection.
+ * 3 - in FDB mode / 5 - in legacy mode
+ */
+uint32_t mlx5_flow_hw_avl_tags_init_cnt;
+enum modify_reg mlx5_flow_hw_avl_tags[MLX5_FLOW_HW_TAGS_MAX] = {REG_NON};
+
struct tunnel_default_miss_ctx {
uint16_t *queue;
__extension__
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index c0c719dd8b..98ae7c6bda 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -1334,6 +1334,10 @@ struct flow_hw_port_info {
extern struct flow_hw_port_info mlx5_flow_hw_port_infos[RTE_MAX_ETHPORTS];
+#define MLX5_FLOW_HW_TAGS_MAX 8
+extern uint32_t mlx5_flow_hw_avl_tags_init_cnt;
+extern enum modify_reg mlx5_flow_hw_avl_tags[];
+
/*
* Get metadata match tag and mask for given rte_eth_dev port.
* Used in HWS rule creation.
@@ -1375,9 +1379,32 @@ flow_hw_get_wire_port(struct ibv_context *ibctx)
}
#endif
+/*
+ * Convert metadata or tag to the actual register.
+ * META: Can only be used to match in the FDB in this stage, fixed C_1.
+ * TAG: C_x expect meter color reg and the reserved ones.
+ * TODO: Per port / device, FDB or NIC for Meta matching.
+ */
+static __rte_always_inline int
+flow_hw_get_reg_id(enum rte_flow_item_type type, uint32_t id)
+{
+ switch (type) {
+ case RTE_FLOW_ITEM_TYPE_META:
+ return REG_
[v6 05/18] common/mlx5: query set capability of registers
From: Bing Zhao
In the flow table capabilities, new fields are added to query the
capability to set, add, copy to a REG_C_x.
The set capability are queried and saved for the future usage.
Signed-off-by: Bing Zhao
---
drivers/common/mlx5/mlx5_devx_cmds.c | 30 +++
drivers/common/mlx5/mlx5_devx_cmds.h | 2 ++
drivers/common/mlx5/mlx5_prm.h | 45 +---
3 files changed, 73 insertions(+), 4 deletions(-)
diff --git a/drivers/common/mlx5/mlx5_devx_cmds.c
b/drivers/common/mlx5/mlx5_devx_cmds.c
index 76f0b6724f..9c185366d0 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.c
+++ b/drivers/common/mlx5/mlx5_devx_cmds.c
@@ -1064,6 +1064,24 @@ mlx5_devx_cmd_query_hca_attr(void *ctx,
attr->modify_outer_ip_ecn = MLX5_GET
(flow_table_nic_cap, hcattr,
ft_header_modify_nic_receive.outer_ip_ecn);
+ attr->set_reg_c = 0xff;
+ if (attr->nic_flow_table) {
+#define GET_RX_REG_X_BITS \
+ MLX5_GET(flow_table_nic_cap, hcattr, \
+ft_header_modify_nic_receive.metadata_reg_c_x)
+#define GET_TX_REG_X_BITS \
+ MLX5_GET(flow_table_nic_cap, hcattr, \
+ft_header_modify_nic_transmit.metadata_reg_c_x)
+
+ uint32_t tx_reg, rx_reg;
+
+ tx_reg = GET_TX_REG_X_BITS;
+ rx_reg = GET_RX_REG_X_BITS;
+ attr->set_reg_c &= (rx_reg & tx_reg);
+
+#undef GET_RX_REG_X_BITS
+#undef GET_TX_REG_X_BITS
+ }
attr->pkt_integrity_match = mlx5_devx_query_pkt_integrity_match(hcattr);
attr->inner_ipv4_ihl = MLX5_GET
(flow_table_nic_cap, hcattr,
@@ -1163,6 +1181,18 @@ mlx5_devx_cmd_query_hca_attr(void *ctx,
attr->esw_mgr_vport_id =
MLX5_GET(esw_cap, hcattr, esw_manager_vport_number);
}
+ if (attr->eswitch_manager) {
+ uint32_t esw_reg;
+
+ hcattr = mlx5_devx_get_hca_cap(ctx, in, out, &rc,
+ MLX5_GET_HCA_CAP_OP_MOD_ESW_FLOW_TABLE |
+ MLX5_HCA_CAP_OPMOD_GET_CUR);
+ if (!hcattr)
+ return rc;
+ esw_reg = MLX5_GET(flow_table_esw_cap, hcattr,
+ ft_header_modify_esw_fdb.metadata_reg_c_x);
+ attr->set_reg_c &= esw_reg;
+ }
return 0;
error:
rc = (rc > 0) ? -rc : rc;
diff --git a/drivers/common/mlx5/mlx5_devx_cmds.h
b/drivers/common/mlx5/mlx5_devx_cmds.h
index cceaf3411d..a10aa3331b 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.h
+++ b/drivers/common/mlx5/mlx5_devx_cmds.h
@@ -263,6 +263,8 @@ struct mlx5_hca_attr {
uint32_t crypto_wrapped_import_method:1;
uint16_t esw_mgr_vport_id; /* E-Switch Mgr vport ID . */
uint16_t max_wqe_sz_sq;
+ uint32_t set_reg_c:8;
+ uint32_t nic_flow_table:1;
uint32_t modify_outer_ip_ecn:1;
};
diff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h
index 9c1c93f916..ca4763f53d 100644
--- a/drivers/common/mlx5/mlx5_prm.h
+++ b/drivers/common/mlx5/mlx5_prm.h
@@ -1295,6 +1295,7 @@ enum {
MLX5_GET_HCA_CAP_OP_MOD_QOS_CAP = 0xc << 1,
MLX5_GET_HCA_CAP_OP_MOD_ROCE = 0x4 << 1,
MLX5_GET_HCA_CAP_OP_MOD_NIC_FLOW_TABLE = 0x7 << 1,
+ MLX5_GET_HCA_CAP_OP_MOD_ESW_FLOW_TABLE = 0x8 << 1,
MLX5_SET_HCA_CAP_OP_MOD_ESW = 0x9 << 1,
MLX5_GET_HCA_CAP_OP_MOD_VDPA_EMULATION = 0x13 << 1,
MLX5_GET_HCA_CAP_OP_MOD_CRYPTO = 0x1A << 1,
@@ -1892,6 +1893,7 @@ struct mlx5_ifc_roce_caps_bits {
};
struct mlx5_ifc_ft_fields_support_bits {
+ /* set_action_field_support */
u8 outer_dmac[0x1];
u8 outer_smac[0x1];
u8 outer_ether_type[0x1];
@@ -1919,7 +1921,7 @@ struct mlx5_ifc_ft_fields_support_bits {
u8 outer_gre_key[0x1];
u8 outer_vxlan_vni[0x1];
u8 reserved_at_1a[0x5];
- u8 source_eswitch_port[0x1];
+ u8 source_eswitch_port[0x1]; /* end of DW0 */
u8 inner_dmac[0x1];
u8 inner_smac[0x1];
u8 inner_ether_type[0x1];
@@ -1943,8 +1945,33 @@ struct mlx5_ifc_ft_fields_support_bits {
u8 inner_tcp_sport[0x1];
u8 inner_tcp_dport[0x1];
u8 inner_tcp_flags[0x1];
- u8 reserved_at_37[0x9];
- u8 reserved_at_40[0x40];
+ u8 reserved_at_37[0x9]; /* end of DW1 */
+ u8 reserved_at_40[0x20]; /* end of DW2 */
+ u8 reserved_at_60[0x18];
+ union {
+ struct {
+ u8 metadata_reg_c_7[0x1];
+ u8 metadata_reg_c_6[0x1];
+ u8 metadata_reg_c_5[0x1];
+ u8 metadata_reg_c_4[0x1];
+ u8 metadata_reg_c_3[0x1];
+ u8 metadata_reg_c_2[0x1];
+ u8 metadata_reg_c_1[0x1];
+ u8 metadata_reg_c_0[0x1];
+ };
+ u8 metadata_reg_c_x[0
[v6 09/18] net/mlx5/hws: Add HWS pool and buddy
From: Erez Shitrit
HWS needs to manage different types of device memory in
an efficient and quick way. For this, memory pools are
being used.
Signed-off-by: Erez Shitrit
Signed-off-by: Alex Vesker
---
drivers/net/mlx5/hws/mlx5dr_buddy.c | 200 +
drivers/net/mlx5/hws/mlx5dr_buddy.h | 22 +
drivers/net/mlx5/hws/mlx5dr_pool.c | 672
drivers/net/mlx5/hws/mlx5dr_pool.h | 152 +++
4 files changed, 1046 insertions(+)
create mode 100644 drivers/net/mlx5/hws/mlx5dr_buddy.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_buddy.h
create mode 100644 drivers/net/mlx5/hws/mlx5dr_pool.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_pool.h
diff --git a/drivers/net/mlx5/hws/mlx5dr_buddy.c
b/drivers/net/mlx5/hws/mlx5dr_buddy.c
new file mode 100644
index 00..cde4f54f66
--- /dev/null
+++ b/drivers/net/mlx5/hws/mlx5dr_buddy.c
@@ -0,0 +1,200 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2022 NVIDIA Corporation & Affiliates
+ */
+
+#include
+#include
+#include "mlx5dr_internal.h"
+#include "mlx5dr_buddy.h"
+
+static struct rte_bitmap *bitmap_alloc0(int s)
+{
+ struct rte_bitmap *bitmap;
+ uint32_t bmp_size;
+ void *mem;
+
+ bmp_size = rte_bitmap_get_memory_footprint(s);
+ mem = rte_zmalloc("create_bmap", bmp_size, RTE_CACHE_LINE_SIZE);
+ if (!mem) {
+ DR_LOG(ERR, "No mem for bitmap");
+ rte_errno = ENOMEM;
+ return NULL;
+ }
+
+ bitmap = rte_bitmap_init(s, mem, bmp_size);
+ if (!bitmap) {
+ DR_LOG(ERR, "%s Failed to initialize bitmap", __func__);
+ rte_errno = EINVAL;
+ goto err_mem_alloc;
+ }
+
+ return bitmap;
+
+err_mem_alloc:
+ rte_free(mem);
+ return NULL;
+}
+
+static void bitmap_set_bit(struct rte_bitmap *bmp, uint32_t pos)
+{
+ rte_bitmap_set(bmp, pos);
+}
+
+static void bitmap_clear_bit(struct rte_bitmap *bmp, uint32_t pos)
+{
+ rte_bitmap_clear(bmp, pos);
+}
+
+static bool bitmap_test_bit(struct rte_bitmap *bmp, unsigned long n)
+{
+ return !!rte_bitmap_get(bmp, n);
+}
+
+static unsigned long bitmap_ffs(struct rte_bitmap *bmap,
+ uint64_t n, unsigned long m)
+{
+ uint64_t out_slab = 0;
+ uint32_t pos = 0; /* Compilation warn */
+
+ __rte_bitmap_scan_init(bmap);
+ if (!rte_bitmap_scan(bmap, &pos, &out_slab)) {
+ DR_LOG(ERR, "Failed to get slab from bitmap.");
+ return m;
+ }
+ pos = pos + __builtin_ctzll(out_slab);
+
+ if (pos < n) {
+ DR_LOG(ERR, "Unexpected bit (%d < %"PRIx64") from bitmap", pos,
n);
+ return m;
+ }
+ return pos;
+}
+
+static unsigned long mlx5dr_buddy_find_first_bit(struct rte_bitmap *addr,
+uint32_t size)
+{
+ return bitmap_ffs(addr, 0, size);
+}
+
+static int mlx5dr_buddy_init(struct mlx5dr_buddy_mem *buddy, uint32_t
max_order)
+{
+ int i, s;
+
+ buddy->max_order = max_order;
+
+ buddy->bits = simple_calloc(buddy->max_order + 1, sizeof(long *));
+ if (!buddy->bits) {
+ rte_errno = ENOMEM;
+ return -1;
+ }
+
+ buddy->num_free = simple_calloc(buddy->max_order + 1,
sizeof(*buddy->num_free));
+ if (!buddy->num_free) {
+ rte_errno = ENOMEM;
+ goto err_out_free_bits;
+ }
+
+ for (i = 0; i <= (int)buddy->max_order; ++i) {
+ s = 1 << (buddy->max_order - i);
+ buddy->bits[i] = bitmap_alloc0(s);
+ if (!buddy->bits[i])
+ goto err_out_free_num_free;
+ }
+
+ bitmap_set_bit(buddy->bits[buddy->max_order], 0);
+
+ buddy->num_free[buddy->max_order] = 1;
+
+ return 0;
+
+err_out_free_num_free:
+ for (i = 0; i <= (int)buddy->max_order; ++i)
+ rte_free(buddy->bits[i]);
+
+ simple_free(buddy->num_free);
+
+err_out_free_bits:
+ simple_free(buddy->bits);
+ return -1;
+}
+
+struct mlx5dr_buddy_mem *mlx5dr_buddy_create(uint32_t max_order)
+{
+ struct mlx5dr_buddy_mem *buddy;
+
+ buddy = simple_calloc(1, sizeof(*buddy));
+ if (!buddy) {
+ rte_errno = ENOMEM;
+ return NULL;
+ }
+
+ if (mlx5dr_buddy_init(buddy, max_order))
+ goto free_buddy;
+
+ return buddy;
+
+free_buddy:
+ simple_free(buddy);
+ return NULL;
+}
+
+void mlx5dr_buddy_cleanup(struct mlx5dr_buddy_mem *buddy)
+{
+ int i;
+
+ for (i = 0; i <= (int)buddy->max_order; ++i)
+ rte_free(buddy->bits[i]);
+
+ simple_free(buddy->num_free);
+ simple_free(buddy->bits);
+}
+
+int mlx
[v6 10/18] net/mlx5/hws: Add HWS send layer
HWS configures flows to the HW using a QP, each WQE has
the details of the flow we want to offload. The send layer
allocates the resources needed to send the request to the HW
as well as managing the queues, getting completions and
handling failures.
Signed-off-by: Mark Bloch
Signed-off-by: Alex Vesker
---
drivers/net/mlx5/hws/mlx5dr_send.c | 844 +
drivers/net/mlx5/hws/mlx5dr_send.h | 275 ++
2 files changed, 1119 insertions(+)
create mode 100644 drivers/net/mlx5/hws/mlx5dr_send.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_send.h
diff --git a/drivers/net/mlx5/hws/mlx5dr_send.c
b/drivers/net/mlx5/hws/mlx5dr_send.c
new file mode 100644
index 00..26904a9040
--- /dev/null
+++ b/drivers/net/mlx5/hws/mlx5dr_send.c
@@ -0,0 +1,844 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2022 NVIDIA Corporation & Affiliates
+ */
+
+#include "mlx5dr_internal.h"
+
+struct mlx5dr_send_ring_dep_wqe *
+mlx5dr_send_add_new_dep_wqe(struct mlx5dr_send_engine *queue)
+{
+ struct mlx5dr_send_ring_sq *send_sq = &queue->send_ring->send_sq;
+ unsigned int idx = send_sq->head_dep_idx++ & (queue->num_entries - 1);
+
+ memset(&send_sq->dep_wqe[idx].wqe_data.tag, 0, MLX5DR_MATCH_TAG_SZ);
+
+ return &send_sq->dep_wqe[idx];
+}
+
+void mlx5dr_send_abort_new_dep_wqe(struct mlx5dr_send_engine *queue)
+{
+ queue->send_ring->send_sq.head_dep_idx--;
+}
+
+void mlx5dr_send_all_dep_wqe(struct mlx5dr_send_engine *queue)
+{
+ struct mlx5dr_send_ring_sq *send_sq = &queue->send_ring->send_sq;
+ struct mlx5dr_send_ste_attr ste_attr = {0};
+ struct mlx5dr_send_ring_dep_wqe *dep_wqe;
+
+ ste_attr.send_attr.opmod = MLX5DR_WQE_GTA_OPMOD_STE;
+ ste_attr.send_attr.opcode = MLX5DR_WQE_OPCODE_TBL_ACCESS;
+ ste_attr.send_attr.len = MLX5DR_WQE_SZ_GTA_CTRL +
MLX5DR_WQE_SZ_GTA_DATA;
+ ste_attr.gta_opcode = MLX5DR_WQE_GTA_OP_ACTIVATE;
+
+ /* Fence first from previous depend WQEs */
+ ste_attr.send_attr.fence = 1;
+
+ while (send_sq->head_dep_idx != send_sq->tail_dep_idx) {
+ dep_wqe = &send_sq->dep_wqe[send_sq->tail_dep_idx++ &
(queue->num_entries - 1)];
+
+ /* Notify HW on the last WQE */
+ ste_attr.send_attr.notify_hw = (send_sq->tail_dep_idx ==
send_sq->head_dep_idx);
+ ste_attr.send_attr.user_data = dep_wqe->user_data;
+ ste_attr.send_attr.rule = dep_wqe->rule;
+
+ ste_attr.rtc_0 = dep_wqe->rtc_0;
+ ste_attr.rtc_1 = dep_wqe->rtc_1;
+ ste_attr.retry_rtc_0 = dep_wqe->retry_rtc_0;
+ ste_attr.retry_rtc_1 = dep_wqe->retry_rtc_1;
+ ste_attr.used_id_rtc_0 = &dep_wqe->rule->rtc_0;
+ ste_attr.used_id_rtc_1 = &dep_wqe->rule->rtc_1;
+ ste_attr.wqe_ctrl = &dep_wqe->wqe_ctrl;
+ ste_attr.wqe_data = &dep_wqe->wqe_data;
+
+ mlx5dr_send_ste(queue, &ste_attr);
+
+ /* Fencing is done only on the first WQE */
+ ste_attr.send_attr.fence = 0;
+ }
+}
+
+struct mlx5dr_send_engine_post_ctrl
+mlx5dr_send_engine_post_start(struct mlx5dr_send_engine *queue)
+{
+ struct mlx5dr_send_engine_post_ctrl ctrl;
+
+ ctrl.queue = queue;
+ /* Currently only one send ring is supported */
+ ctrl.send_ring = &queue->send_ring[0];
+ ctrl.num_wqebbs = 0;
+
+ return ctrl;
+}
+
+void mlx5dr_send_engine_post_req_wqe(struct mlx5dr_send_engine_post_ctrl *ctrl,
+char **buf, size_t *len)
+{
+ struct mlx5dr_send_ring_sq *send_sq = &ctrl->send_ring->send_sq;
+ unsigned int idx;
+
+ idx = (send_sq->cur_post + ctrl->num_wqebbs) & send_sq->buf_mask;
+
+ *buf = send_sq->buf + (idx << MLX5_SEND_WQE_SHIFT);
+ *len = MLX5_SEND_WQE_BB;
+
+ if (!ctrl->num_wqebbs) {
+ *buf += sizeof(struct mlx5dr_wqe_ctrl_seg);
+ *len -= sizeof(struct mlx5dr_wqe_ctrl_seg);
+ }
+
+ ctrl->num_wqebbs++;
+}
+
+static void mlx5dr_send_engine_post_ring(struct mlx5dr_send_ring_sq *sq,
+struct mlx5dv_devx_uar *uar,
+struct mlx5dr_wqe_ctrl_seg *wqe_ctrl)
+{
+ rte_compiler_barrier();
+ sq->db[MLX5_SND_DBR] = rte_cpu_to_be_32(sq->cur_post);
+
+ rte_wmb();
+ mlx5dr_uar_write64_relaxed(*((uint64_t *)wqe_ctrl), uar->reg_addr);
+ rte_wmb();
+}
+
+static void
+mlx5dr_send_wqe_set_tag(struct mlx5dr_wqe_gta_data_seg_ste *wqe_data,
+ struct mlx5dr_rule_match_tag *tag,
+ bool is_jumbo)
+{
+ if (is_jumbo) {
+ /* Clear previous possibly dirty control */
+
[v6 07/18] net/mlx5: Add additional glue functions for HWS
Add missing glue support for HWS mlx5dr layer. The new glue
functions are needed for mlx5dv create matcher and action,
which are used as the kernel root table as well as for
capabilities query like device name and ports info.
Signed-off-by: Alex Vesker
---
drivers/common/mlx5/linux/mlx5_glue.c | 121 --
drivers/common/mlx5/linux/mlx5_glue.h | 17
2 files changed, 131 insertions(+), 7 deletions(-)
diff --git a/drivers/common/mlx5/linux/mlx5_glue.c
b/drivers/common/mlx5/linux/mlx5_glue.c
index b954df0784..702eb36b62 100644
--- a/drivers/common/mlx5/linux/mlx5_glue.c
+++ b/drivers/common/mlx5/linux/mlx5_glue.c
@@ -111,6 +111,12 @@ mlx5_glue_query_device_ex(struct ibv_context *context,
return ibv_query_device_ex(context, input, attr);
}
+static const char *
+mlx5_glue_get_device_name(struct ibv_device *device)
+{
+ return ibv_get_device_name(device);
+}
+
static int
mlx5_glue_query_rt_values_ex(struct ibv_context *context,
struct ibv_values_ex *values)
@@ -620,6 +626,20 @@ mlx5_glue_dv_create_qp(struct ibv_context *context,
#endif
}
+static void *
+__mlx5_glue_dv_create_flow_matcher(struct ibv_context *context,
+ struct mlx5dv_flow_matcher_attr *matcher_attr)
+{
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+ return mlx5dv_create_flow_matcher(context, matcher_attr);
+#else
+ (void)context;
+ (void)matcher_attr;
+ errno = ENOTSUP;
+ return NULL;
+#endif
+}
+
static void *
mlx5_glue_dv_create_flow_matcher(struct ibv_context *context,
struct mlx5dv_flow_matcher_attr *matcher_attr,
@@ -633,7 +653,7 @@ mlx5_glue_dv_create_flow_matcher(struct ibv_context
*context,
matcher_attr->match_mask);
#else
(void)tbl;
- return mlx5dv_create_flow_matcher(context, matcher_attr);
+ return __mlx5_glue_dv_create_flow_matcher(context, matcher_attr);
#endif
#else
(void)context;
@@ -644,6 +664,26 @@ mlx5_glue_dv_create_flow_matcher(struct ibv_context
*context,
#endif
}
+static void *
+__mlx5_glue_dv_create_flow(void *matcher,
+ void *match_value,
+ size_t num_actions,
+ void *actions)
+{
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+ return mlx5dv_create_flow(matcher,
+ match_value,
+ num_actions,
+ (struct mlx5dv_flow_action_attr *)actions);
+#else
+ (void)matcher;
+ (void)match_value;
+ (void)num_actions;
+ (void)actions;
+ return NULL;
+#endif
+}
+
static void *
mlx5_glue_dv_create_flow(void *matcher,
void *match_value,
@@ -663,8 +703,8 @@ mlx5_glue_dv_create_flow(void *matcher,
for (i = 0; i < num_actions; i++)
actions_attr[i] =
*((struct mlx5dv_flow_action_attr *)(actions[i]));
- return mlx5dv_create_flow(matcher, match_value,
- num_actions, actions_attr);
+ return __mlx5_glue_dv_create_flow(matcher, match_value,
+ num_actions, actions_attr);
#endif
#else
(void)matcher;
@@ -735,6 +775,26 @@ mlx5_glue_dv_create_flow_action_dest_devx_tir(void *tir)
#endif
}
+static void *
+__mlx5_glue_dv_create_flow_action_modify_header
+ (struct ibv_context *ctx,
+size_t actions_sz,
+uint64_t actions[],
+enum mlx5dv_flow_table_type ft_type)
+{
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+ return mlx5dv_create_flow_action_modify_header
+ (ctx, actions_sz, actions, ft_type);
+#else
+ (void)ctx;
+ (void)ft_type;
+ (void)actions_sz;
+ (void)actions;
+ errno = ENOTSUP;
+ return NULL;
+#endif
+}
+
static void *
mlx5_glue_dv_create_flow_action_modify_header
(struct ibv_context *ctx,
@@ -758,7 +818,7 @@ mlx5_glue_dv_create_flow_action_modify_header
if (!action)
return NULL;
action->type = MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION;
- action->action = mlx5dv_create_flow_action_modify_header
+ action->action = __mlx5_glue_dv_create_flow_action_modify_header
(ctx, actions_sz, actions, ft_type);
return action;
#endif
@@ -774,6 +834,27 @@ mlx5_glue_dv_create_flow_action_modify_header
#endif
}
+static void *
+__mlx5_glue_dv_create_flow_action_packet_reformat
+ (struct ibv_context *ctx,
+size_t data_sz, void *data,
+enum mlx5dv_flow_action_packet_reformat_type reformat_type,
+enum mlx5dv_flow_table_type ft_type)
+{
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+ return mlx5dv_create_flow_action_
[v6 08/18] net/mlx5/hws: Add HWS command layer
From: Erez Shitrit
The command layer is used to communicate with the FW,
query capabilities and allocate FW resources needed for HWS.
Signed-off-by: Erez Shitrit
Signed-off-by: Alex Vesker
---
drivers/common/mlx5/mlx5_prm.h| 607 ++-
drivers/net/mlx5/hws/mlx5dr_cmd.c | 948 ++
drivers/net/mlx5/hws/mlx5dr_cmd.h | 230
3 files changed, 1775 insertions(+), 10 deletions(-)
create mode 100644 drivers/net/mlx5/hws/mlx5dr_cmd.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_cmd.h
diff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h
index ca4763f53d..371942ae50 100644
--- a/drivers/common/mlx5/mlx5_prm.h
+++ b/drivers/common/mlx5/mlx5_prm.h
@@ -289,6 +289,8 @@
/* The alignment needed for CQ buffer. */
#define MLX5_CQE_BUF_ALIGNMENT rte_mem_page_size()
+#define MAX_ACTIONS_DATA_IN_HEADER_MODIFY 512
+
/* Completion mode. */
enum mlx5_completion_mode {
MLX5_COMP_ONLY_ERR = 0x0,
@@ -677,6 +679,10 @@ enum {
MLX5_MODIFICATION_TYPE_SET = 0x1,
MLX5_MODIFICATION_TYPE_ADD = 0x2,
MLX5_MODIFICATION_TYPE_COPY = 0x3,
+ MLX5_MODIFICATION_TYPE_INSERT = 0x4,
+ MLX5_MODIFICATION_TYPE_REMOVE = 0x5,
+ MLX5_MODIFICATION_TYPE_NOP = 0x6,
+ MLX5_MODIFICATION_TYPE_REMOVE_WORDS = 0x7,
};
/* The field of packet to be modified. */
@@ -,6 +1117,10 @@ enum {
MLX5_CMD_OP_QUERY_TIS = 0x915,
MLX5_CMD_OP_CREATE_RQT = 0x916,
MLX5_CMD_OP_MODIFY_RQT = 0x917,
+ MLX5_CMD_OP_CREATE_FLOW_TABLE = 0x930,
+ MLX5_CMD_OP_CREATE_FLOW_GROUP = 0x933,
+ MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY = 0x936,
+ MLX5_CMD_OP_MODIFY_FLOW_TABLE = 0x93c,
MLX5_CMD_OP_ALLOC_FLOW_COUNTER = 0x939,
MLX5_CMD_OP_QUERY_FLOW_COUNTER = 0x93b,
MLX5_CMD_OP_CREATE_GENERAL_OBJECT = 0xa00,
@@ -1299,6 +1309,7 @@ enum {
MLX5_SET_HCA_CAP_OP_MOD_ESW = 0x9 << 1,
MLX5_GET_HCA_CAP_OP_MOD_VDPA_EMULATION = 0x13 << 1,
MLX5_GET_HCA_CAP_OP_MOD_CRYPTO = 0x1A << 1,
+ MLX5_GET_HCA_CAP_OP_MOD_WQE_BASED_FLOW_TABLE = 0x1B << 1,
MLX5_GET_HCA_CAP_OP_MOD_PARSE_GRAPH_NODE_CAP = 0x1C << 1,
MLX5_GET_HCA_CAP_OP_MOD_GENERAL_DEVICE_2 = 0x20 << 1,
};
@@ -1317,6 +1328,14 @@ enum {
(1ULL << MLX5_GENERAL_OBJ_TYPE_GENEVE_TLV_OPT)
#define MLX5_GENERAL_OBJ_TYPES_CAP_CONN_TRACK_OFFLOAD \
(1ULL << MLX5_GENERAL_OBJ_TYPE_CONN_TRACK_OFFLOAD)
+#define MLX5_GENERAL_OBJ_TYPES_CAP_RTC \
+ (1ULL << MLX5_GENERAL_OBJ_TYPE_RTC)
+#define MLX5_GENERAL_OBJ_TYPES_CAP_STC \
+ (1ULL << MLX5_GENERAL_OBJ_TYPE_STC)
+#define MLX5_GENERAL_OBJ_TYPES_CAP_STE \
+ (1ULL << MLX5_GENERAL_OBJ_TYPE_STE)
+#define MLX5_GENERAL_OBJ_TYPES_CAP_DEFINER \
+ (1ULL << MLX5_GENERAL_OBJ_TYPE_DEFINER)
#define MLX5_GENERAL_OBJ_TYPES_CAP_DEK \
(1ULL << MLX5_GENERAL_OBJ_TYPE_DEK)
#define MLX5_GENERAL_OBJ_TYPES_CAP_IMPORT_KEK \
@@ -1373,6 +1392,11 @@ enum {
#define MLX5_HCA_FLEX_VXLAN_GPE_ENABLED (1UL << 7)
#define MLX5_HCA_FLEX_ICMP_ENABLED (1UL << 8)
#define MLX5_HCA_FLEX_ICMPV6_ENABLED (1UL << 9)
+#define MLX5_HCA_FLEX_GTPU_ENABLED (1UL << 11)
+#define MLX5_HCA_FLEX_GTPU_DW_2_ENABLED (1UL << 16)
+#define MLX5_HCA_FLEX_GTPU_FIRST_EXT_DW_0_ENABLED (1UL << 17)
+#define MLX5_HCA_FLEX_GTPU_DW_0_ENABLED (1UL << 18)
+#define MLX5_HCA_FLEX_GTPU_TEID_ENABLED (1UL << 19)
/* The device steering logic format. */
#define MLX5_STEERING_LOGIC_FORMAT_CONNECTX_5 0x0
@@ -1505,7 +1529,8 @@ struct mlx5_ifc_cmd_hca_cap_bits {
u8 wol_u[0x1];
u8 wol_p[0x1];
u8 stat_rate_support[0x10];
- u8 reserved_at_1f0[0xc];
+ u8 reserved_at_1ef[0xb];
+ u8 wqe_based_flow_table_update_cap[0x1];
u8 cqe_version[0x4];
u8 compact_address_vector[0x1];
u8 striding_rq[0x1];
@@ -1681,7 +1706,9 @@ struct mlx5_ifc_cmd_hca_cap_bits {
u8 cqe_compression[0x1];
u8 cqe_compression_timeout[0x10];
u8 cqe_compression_max_num[0x10];
- u8 reserved_at_5e0[0x10];
+ u8 reserved_at_5e0[0x8];
+ u8 flex_parser_id_gtpu_dw_0[0x4];
+ u8 reserved_at_5ec[0x4];
u8 tag_matching[0x1];
u8 rndv_offload_rc[0x1];
u8 rndv_offload_dc[0x1];
@@ -1691,17 +1718,38 @@ struct mlx5_ifc_cmd_hca_cap_bits {
u8 affiliate_nic_vport_criteria[0x8];
u8 native_port_num[0x8];
u8 num_vhca_ports[0x8];
- u8 reserved_at_618[0x6];
+ u8 flex_parser_id_gtpu_teid[0x4];
+ u8 reserved_at_61c[0x2];
u8 sw_owner_id[0x1];
u8 reserved_at_61f[0x6C];
u8 wait_on_data[0x1];
u8 wait_on_time[0x1];
- u8 reserved_at_68d[0xBB];
+ u8 reserved_at_68d[0x37];
+ u8 flex_parser_id_geneve_opt_0[0x
[v6 12/18] net/mlx5/hws: Add HWS context object
Context is the first mlx5dr object created, all sub object:
table, matcher, rule, action are created using the context.
The context holds the capabilities and send queues used for
configuring the offloads to the HW.
Signed-off-by: Alex Vesker
---
drivers/net/mlx5/hws/mlx5dr_context.c | 223 ++
drivers/net/mlx5/hws/mlx5dr_context.h | 40 +
2 files changed, 263 insertions(+)
create mode 100644 drivers/net/mlx5/hws/mlx5dr_context.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_context.h
diff --git a/drivers/net/mlx5/hws/mlx5dr_context.c
b/drivers/net/mlx5/hws/mlx5dr_context.c
new file mode 100644
index 00..ae86694a51
--- /dev/null
+++ b/drivers/net/mlx5/hws/mlx5dr_context.c
@@ -0,0 +1,223 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2022 NVIDIA Corporation & Affiliates
+ */
+
+#include "mlx5dr_internal.h"
+
+static int mlx5dr_context_pools_init(struct mlx5dr_context *ctx)
+{
+ struct mlx5dr_pool_attr pool_attr = {0};
+ uint8_t max_log_sz;
+ int i;
+
+ if (mlx5dr_pat_init_pattern_cache(&ctx->pattern_cache))
+ return rte_errno;
+
+ /* Create an STC pool per FT type */
+ pool_attr.pool_type = MLX5DR_POOL_TYPE_STC;
+ pool_attr.flags = MLX5DR_POOL_FLAGS_FOR_STC_POOL;
+ max_log_sz = RTE_MIN(MLX5DR_POOL_STC_LOG_SZ,
ctx->caps->stc_alloc_log_max);
+ pool_attr.alloc_log_sz = RTE_MAX(max_log_sz,
ctx->caps->stc_alloc_log_gran);
+
+ for (i = 0; i < MLX5DR_TABLE_TYPE_MAX; i++) {
+ pool_attr.table_type = i;
+ ctx->stc_pool[i] = mlx5dr_pool_create(ctx, &pool_attr);
+ if (!ctx->stc_pool[i]) {
+ DR_LOG(ERR, "Failed to allocate STC pool [%d]", i);
+ goto free_stc_pools;
+ }
+ }
+
+ return 0;
+
+free_stc_pools:
+ for (i = 0; i < MLX5DR_TABLE_TYPE_MAX; i++)
+ if (ctx->stc_pool[i])
+ mlx5dr_pool_destroy(ctx->stc_pool[i]);
+
+ mlx5dr_pat_uninit_pattern_cache(ctx->pattern_cache);
+
+ return rte_errno;
+}
+
+static void mlx5dr_context_pools_uninit(struct mlx5dr_context *ctx)
+{
+ int i;
+
+ mlx5dr_pat_uninit_pattern_cache(ctx->pattern_cache);
+
+ for (i = 0; i < MLX5DR_TABLE_TYPE_MAX; i++) {
+ if (ctx->stc_pool[i])
+ mlx5dr_pool_destroy(ctx->stc_pool[i]);
+ }
+}
+
+static int mlx5dr_context_init_pd(struct mlx5dr_context *ctx,
+ struct ibv_pd *pd)
+{
+ struct mlx5dv_pd mlx5_pd = {0};
+ struct mlx5dv_obj obj;
+ int ret;
+
+ if (pd) {
+ ctx->pd = pd;
+ } else {
+ ctx->pd = mlx5_glue->alloc_pd(ctx->ibv_ctx);
+ if (!ctx->pd) {
+ DR_LOG(ERR, "Failed to allocate PD");
+ rte_errno = errno;
+ return rte_errno;
+ }
+ ctx->flags |= MLX5DR_CONTEXT_FLAG_PRIVATE_PD;
+ }
+
+ obj.pd.in = ctx->pd;
+ obj.pd.out = &mlx5_pd;
+
+ ret = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_PD);
+ if (ret)
+ goto free_private_pd;
+
+ ctx->pd_num = mlx5_pd.pdn;
+
+ return 0;
+
+free_private_pd:
+ if (ctx->flags & MLX5DR_CONTEXT_FLAG_PRIVATE_PD)
+ mlx5_glue->dealloc_pd(ctx->pd);
+
+ return ret;
+}
+
+static int mlx5dr_context_uninit_pd(struct mlx5dr_context *ctx)
+{
+ if (ctx->flags & MLX5DR_CONTEXT_FLAG_PRIVATE_PD)
+ return mlx5_glue->dealloc_pd(ctx->pd);
+
+ return 0;
+}
+
+static void mlx5dr_context_check_hws_supp(struct mlx5dr_context *ctx)
+{
+ struct mlx5dr_cmd_query_caps *caps = ctx->caps;
+
+ /* HWS not supported on device / FW */
+ if (!caps->wqe_based_update) {
+ DR_LOG(INFO, "Required HWS WQE based insertion cap not
supported");
+ return;
+ }
+
+ /* Current solution requires all rules to set reparse bit */
+ if ((!caps->nic_ft.reparse || !caps->fdb_ft.reparse) ||
+ !IS_BIT_SET(caps->rtc_reparse_mode, MLX5_IFC_RTC_REPARSE_ALWAYS)) {
+ DR_LOG(INFO, "Required HWS reparse cap not supported");
+ return;
+ }
+
+ /* FW/HW must support 8DW STE */
+ if (!IS_BIT_SET(caps->ste_format, MLX5_IFC_RTC_STE_FORMAT_8DW)) {
+ DR_LOG(INFO, "Required HWS STE format not supported");
+ return;
+ }
+
+ /* Adding rules by hash and by offset are requirements */
+ if (!IS_BIT_SET(caps->rtc_index_mode,
MLX5_IFC_RTC_STE_UPDATE_MODE_BY_HASH) ||
+ !IS_BIT_SET(caps->rtc_index_mode,
MLX5_IFC_RTC_STE_UPDATE_MODE_BY_OFFSET)) {
+
[v6 11/18] net/mlx5/hws: Add HWS definer layer
Definers are HW objects that are used for matching, rte items
are translated to definers, each definer holds the fields and
bit-masks used for HW flow matching. The definer layer is used
for finding the most efficient definer for each set of items.
In addition to definer creation we also calculate the field
copy (fc) array used for efficient items to WQE conversion.
Signed-off-by: Mark Bloch
Signed-off-by: Alex Vesker
---
doc/guides/nics/features/default.ini |1 +
doc/guides/nics/features/mlx5.ini |1 +
drivers/net/mlx5/hws/mlx5dr_definer.c | 1968 +
drivers/net/mlx5/hws/mlx5dr_definer.h | 585
4 files changed, 2555 insertions(+)
create mode 100644 drivers/net/mlx5/hws/mlx5dr_definer.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_definer.h
diff --git a/doc/guides/nics/features/default.ini
b/doc/guides/nics/features/default.ini
index 27f1a70a87..67ba3567c2 100644
--- a/doc/guides/nics/features/default.ini
+++ b/doc/guides/nics/features/default.ini
@@ -140,6 +140,7 @@ udp =
vlan =
vxlan=
vxlan_gpe=
+meter_color =
[rte_flow actions]
age =
diff --git a/doc/guides/nics/features/mlx5.ini
b/doc/guides/nics/features/mlx5.ini
index 8697515385..b129f5787d 100644
--- a/doc/guides/nics/features/mlx5.ini
+++ b/doc/guides/nics/features/mlx5.ini
@@ -84,6 +84,7 @@ vlan = Y
vxlan= Y
vxlan_gpe= Y
represented_port = Y
+meter_color = Y
[rte_flow actions]
age = I
diff --git a/drivers/net/mlx5/hws/mlx5dr_definer.c
b/drivers/net/mlx5/hws/mlx5dr_definer.c
new file mode 100644
index 00..6b98eb8c96
--- /dev/null
+++ b/drivers/net/mlx5/hws/mlx5dr_definer.c
@@ -0,0 +1,1968 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2022 NVIDIA Corporation & Affiliates
+ */
+
+#include "mlx5dr_internal.h"
+
+#define GTP_PDU_SC 0x85
+#define BAD_PORT 0xBAD
+#define ETH_TYPE_IPV4_VXLAN0x0800
+#define ETH_TYPE_IPV6_VXLAN0x86DD
+#define ETH_VXLAN_DEFAULT_PORT 4789
+
+#define STE_NO_VLAN0x0
+#define STE_SVLAN 0x1
+#define STE_CVLAN 0x2
+#define STE_IPV4 0x1
+#define STE_IPV6 0x2
+#define STE_TCP0x1
+#define STE_UDP0x2
+#define STE_ICMP 0x3
+
+/* Setter function based on bit offset and mask, for 32bit DW*/
+#define _DR_SET_32(p, v, byte_off, bit_off, mask) \
+ do { \
+ u32 _v = v; \
+ *((rte_be32_t *)(p) + ((byte_off) / 4)) = \
+ rte_cpu_to_be_32((rte_be_to_cpu_32(*((u32 *)(p) + \
+ ((byte_off) / 4))) & \
+ (~((mask) << (bit_off | \
+(((_v) & (mask)) << \
+ (bit_off))); \
+ } while (0)
+
+/* Setter function based on bit offset and mask */
+#define DR_SET(p, v, byte_off, bit_off, mask) \
+ do { \
+ if (unlikely((bit_off) < 0)) { \
+ u32 _bit_off = -1 * (bit_off); \
+ u32 second_dw_mask = (mask) & ((1 << _bit_off) - 1); \
+ _DR_SET_32(p, (v) >> _bit_off, byte_off, 0, (mask) >>
_bit_off); \
+ _DR_SET_32(p, (v) & second_dw_mask, (byte_off) +
DW_SIZE, \
+ (bit_off) % BITS_IN_DW, second_dw_mask); \
+ } else { \
+ _DR_SET_32(p, v, byte_off, (bit_off), (mask)); \
+ } \
+ } while (0)
+
+/* Setter function based on byte offset to directly set FULL BE32 value */
+#define DR_SET_BE32(p, v, byte_off, bit_off, mask) \
+ (*((rte_be32_t *)((uint8_t *)(p) + (byte_off))) = (v))
+
+/* Setter function based on byte offset to directly set FULL BE32 value from
ptr */
+#define DR_SET_BE32P(p, v_ptr, byte_off, bit_off, mask) \
+ memcpy((uint8_t *)(p) + (byte_off), v_ptr, 4)
+
+/* Setter function based on byte offset to directly set FULL BE16 value */
+#define DR_SET_BE16(p, v, byte_off, bit_off, mask) \
+ (*((rte_be16_t *)((uint8_t *)(p) + (byte_off))) = (v))
+
+/* Setter function based on byte offset to directly set FULL BE16 value from
ptr */
+#define DR_SET_BE16P(p, v_ptr, byte_off, bit_off, mask) \
+ memcpy((uint8_t *)(p) + (byte_off), v_ptr, 2)
+
+#define DR_CALC_FNAME(field, inner) \
+ ((inner) ? MLX5DR_DEFINER_FNAME_##field##_I : \
+ MLX5DR_DEFINER_FNAME_##field##_O)
+
+#define DR_CALC_SET_HDR(fc, hdr, field) \
+ do { \
+ (fc)->bit_mask = __mlx5_mask(definer_hl, hdr.field); \
+ (fc)->bit_off = __mlx5_dw_bit_off(definer_hl, hdr.field); \
+ (fc)->byte_off = MLX5_BYTE_OFF(definer_hl, hdr.field); \
+ } while (0)
+
+/* Helper to calculate data used by DR_SET */
+#define DR_CALC_SET
[v6 13/18] net/mlx5/hws: Add HWS table object
HWS table resides under the context object, each context can
have multiple tables with different steering types RX/TX/FDB.
The table is not only a logical object but it is also represented
in the HW, packets can be steered to the table and from there
to other tables.
Signed-off-by: Erez Shitrit
Signed-off-by: Alex Vesker
---
drivers/net/mlx5/hws/mlx5dr_table.c | 248
drivers/net/mlx5/hws/mlx5dr_table.h | 44 +
2 files changed, 292 insertions(+)
create mode 100644 drivers/net/mlx5/hws/mlx5dr_table.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_table.h
diff --git a/drivers/net/mlx5/hws/mlx5dr_table.c
b/drivers/net/mlx5/hws/mlx5dr_table.c
new file mode 100644
index 00..d3f77e4780
--- /dev/null
+++ b/drivers/net/mlx5/hws/mlx5dr_table.c
@@ -0,0 +1,248 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2022 NVIDIA Corporation & Affiliates
+ */
+
+#include "mlx5dr_internal.h"
+
+static void mlx5dr_table_init_next_ft_attr(struct mlx5dr_table *tbl,
+ struct mlx5dr_cmd_ft_create_attr
*ft_attr)
+{
+ ft_attr->type = tbl->fw_ft_type;
+ if (tbl->type == MLX5DR_TABLE_TYPE_FDB)
+ ft_attr->level = tbl->ctx->caps->fdb_ft.max_level - 1;
+ else
+ ft_attr->level = tbl->ctx->caps->nic_ft.max_level - 1;
+ ft_attr->rtc_valid = true;
+}
+
+/* Call this under ctx->ctrl_lock */
+static int
+mlx5dr_table_up_default_fdb_miss_tbl(struct mlx5dr_table *tbl)
+{
+ struct mlx5dr_cmd_ft_create_attr ft_attr = {0};
+ struct mlx5dr_cmd_forward_tbl *default_miss;
+ struct mlx5dr_context *ctx = tbl->ctx;
+ uint8_t tbl_type = tbl->type;
+ uint32_t vport;
+
+ if (tbl->type != MLX5DR_TABLE_TYPE_FDB)
+ return 0;
+
+ if (ctx->common_res[tbl_type].default_miss) {
+ ctx->common_res[tbl_type].default_miss->refcount++;
+ return 0;
+ }
+
+ ft_attr.type = tbl->fw_ft_type;
+ ft_attr.level = tbl->ctx->caps->fdb_ft.max_level; /* The last level */
+ ft_attr.rtc_valid = false;
+
+ assert(ctx->caps->eswitch_manager);
+ vport = ctx->caps->eswitch_manager_vport_number;
+
+ default_miss = mlx5dr_cmd_miss_ft_create(ctx->ibv_ctx, &ft_attr, vport);
+ if (!default_miss) {
+ DR_LOG(ERR, "Failed to default miss table type: 0x%x",
tbl_type);
+ return rte_errno;
+ }
+
+ ctx->common_res[tbl_type].default_miss = default_miss;
+ ctx->common_res[tbl_type].default_miss->refcount++;
+ return 0;
+}
+
+/* Called under pthread_spin_lock(&ctx->ctrl_lock) */
+static void mlx5dr_table_down_default_fdb_miss_tbl(struct mlx5dr_table *tbl)
+{
+ struct mlx5dr_cmd_forward_tbl *default_miss;
+ struct mlx5dr_context *ctx = tbl->ctx;
+ uint8_t tbl_type = tbl->type;
+
+ if (tbl->type != MLX5DR_TABLE_TYPE_FDB)
+ return;
+
+ default_miss = ctx->common_res[tbl_type].default_miss;
+ if (--default_miss->refcount)
+ return;
+
+ mlx5dr_cmd_miss_ft_destroy(default_miss);
+
+ simple_free(default_miss);
+ ctx->common_res[tbl_type].default_miss = NULL;
+}
+
+static int
+mlx5dr_table_connect_to_default_miss_tbl(struct mlx5dr_table *tbl,
+struct mlx5dr_devx_obj *ft)
+{
+ struct mlx5dr_cmd_ft_modify_attr ft_attr = {0};
+ int ret;
+
+ assert(tbl->type == MLX5DR_TABLE_TYPE_FDB);
+
+ mlx5dr_cmd_set_attr_connect_miss_tbl(tbl->ctx,
+tbl->fw_ft_type,
+tbl->type,
+&ft_attr);
+
+ /* Connect to next */
+ ret = mlx5dr_cmd_flow_table_modify(ft, &ft_attr);
+ if (ret) {
+ DR_LOG(ERR, "Failed to connect FT to default FDB FT");
+ return errno;
+ }
+
+ return 0;
+}
+
+struct mlx5dr_devx_obj *
+mlx5dr_table_create_default_ft(struct mlx5dr_table *tbl)
+{
+ struct mlx5dr_cmd_ft_create_attr ft_attr = {0};
+ struct mlx5dr_devx_obj *ft_obj;
+ int ret;
+
+ mlx5dr_table_init_next_ft_attr(tbl, &ft_attr);
+
+ ft_obj = mlx5dr_cmd_flow_table_create(tbl->ctx->ibv_ctx, &ft_attr);
+ if (ft_obj && tbl->type == MLX5DR_TABLE_TYPE_FDB) {
+ /* Take/create ref over the default miss */
+ ret = mlx5dr_table_up_default_fdb_miss_tbl(tbl);
+ if (ret) {
+ DR_LOG(ERR, "Failed to get default fdb miss");
+ goto free_ft_obj;
+ }
+ ret = mlx5dr_table_connect_to_default_miss_tbl(tbl, ft_obj);
+
[v6 14/18] net/mlx5/hws: Add HWS matcher object
HWS matcher resides under the table object, each table can
have multiple chained matcher with different attributes. Each
matcher represents a combination of match and action templates.
Each matcher can contain multiple configurations based on the
templates. Packets are steered from the table to the matcher
and from there to other objects. The matcher allows efficent
HW packet field matching and action execution based on the
configuration done to it.
Signed-off-by: Alex Vesker
Signed-off-by: Erez Shitrit
---
drivers/common/mlx5/linux/meson.build | 2 +
drivers/net/mlx5/hws/mlx5dr_matcher.c | 919 ++
drivers/net/mlx5/hws/mlx5dr_matcher.h | 76 +++
3 files changed, 997 insertions(+)
create mode 100644 drivers/net/mlx5/hws/mlx5dr_matcher.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_matcher.h
diff --git a/drivers/common/mlx5/linux/meson.build
b/drivers/common/mlx5/linux/meson.build
index b044f95700..e6b32eb84d 100644
--- a/drivers/common/mlx5/linux/meson.build
+++ b/drivers/common/mlx5/linux/meson.build
@@ -74,6 +74,8 @@ has_member_args = [
'struct ibv_counters_init_attr', 'comp_mask' ],
[ 'HAVE_MLX5DV_DEVX_UAR_OFFSET', 'infiniband/mlx5dv.h',
'struct mlx5dv_devx_uar', 'mmap_off' ],
+[ 'HAVE_MLX5DV_FLOW_MATCHER_FT_TYPE', 'infiniband/mlx5dv.h',
+'struct mlx5dv_flow_matcher_attr', 'ft_type' ],
]
# input array for meson symbol search:
# [ "MACRO to define if found", "header for the search",
diff --git a/drivers/net/mlx5/hws/mlx5dr_matcher.c
b/drivers/net/mlx5/hws/mlx5dr_matcher.c
new file mode 100644
index 00..d1205c42fa
--- /dev/null
+++ b/drivers/net/mlx5/hws/mlx5dr_matcher.c
@@ -0,0 +1,919 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2022 NVIDIA Corporation & Affiliates
+ */
+
+#include "mlx5dr_internal.h"
+
+static bool mlx5dr_matcher_requires_col_tbl(uint8_t log_num_of_rules)
+{
+ /* Collision table concatenation is done only for large rule tables */
+ return log_num_of_rules > MLX5DR_MATCHER_ASSURED_RULES_TH;
+}
+
+static uint8_t mlx5dr_matcher_rules_to_tbl_depth(uint8_t log_num_of_rules)
+{
+ if (mlx5dr_matcher_requires_col_tbl(log_num_of_rules))
+ return MLX5DR_MATCHER_ASSURED_MAIN_TBL_DEPTH;
+
+ /* For small rule tables we use a single deep table to assure insertion
*/
+ return RTE_MIN(log_num_of_rules, MLX5DR_MATCHER_ASSURED_COL_TBL_DEPTH);
+}
+
+static int mlx5dr_matcher_create_end_ft(struct mlx5dr_matcher *matcher)
+{
+ struct mlx5dr_table *tbl = matcher->tbl;
+
+ matcher->end_ft = mlx5dr_table_create_default_ft(tbl);
+ if (!matcher->end_ft) {
+ DR_LOG(ERR, "Failed to create matcher end flow table");
+ return rte_errno;
+ }
+ return 0;
+}
+
+static void mlx5dr_matcher_destroy_end_ft(struct mlx5dr_matcher *matcher)
+{
+ mlx5dr_table_destroy_default_ft(matcher->tbl, matcher->end_ft);
+}
+
+static int mlx5dr_matcher_connect(struct mlx5dr_matcher *matcher)
+{
+ struct mlx5dr_cmd_ft_modify_attr ft_attr = {0};
+ struct mlx5dr_table *tbl = matcher->tbl;
+ struct mlx5dr_matcher *prev = NULL;
+ struct mlx5dr_matcher *next = NULL;
+ struct mlx5dr_matcher *tmp_matcher;
+ struct mlx5dr_devx_obj *ft;
+ int ret;
+
+ /* Find location in matcher list */
+ if (LIST_EMPTY(&tbl->head)) {
+ LIST_INSERT_HEAD(&tbl->head, matcher, next);
+ goto connect;
+ }
+
+ LIST_FOREACH(tmp_matcher, &tbl->head, next) {
+ if (tmp_matcher->attr.priority > matcher->attr.priority) {
+ next = tmp_matcher;
+ break;
+ }
+ prev = tmp_matcher;
+ }
+
+ if (next)
+ LIST_INSERT_BEFORE(next, matcher, next);
+ else
+ LIST_INSERT_AFTER(prev, matcher, next);
+
+connect:
+ ft_attr.modify_fs = MLX5_IFC_MODIFY_FLOW_TABLE_RTC_ID;
+ ft_attr.type = tbl->fw_ft_type;
+
+ /* Connect to next */
+ if (next) {
+ if (next->match_ste.rtc_0)
+ ft_attr.rtc_id_0 = next->match_ste.rtc_0->id;
+ if (next->match_ste.rtc_1)
+ ft_attr.rtc_id_1 = next->match_ste.rtc_1->id;
+
+ ret = mlx5dr_cmd_flow_table_modify(matcher->end_ft, &ft_attr);
+ if (ret) {
+ DR_LOG(ERR, "Failed to connect new matcher to next
RTC");
+ goto remove_from_list;
+ }
+ }
+
+ /* Connect to previous */
+ ft = prev ? prev->end_ft : tbl->ft;
+
+ if (matcher->match_ste.rtc_0)
+ ft_attr.rtc_id_0 = matcher-
[v6 16/18] net/mlx5/hws: Add HWS action object
From: Erez Shitrit
Action objects are used for executing different HW actions
over packets. Each action contains the HW resources and parameters
needed for action use over the HW when creating a rule.
Signed-off-by: Erez Shitrit
Signed-off-by: Alex Vesker
---
drivers/net/mlx5/hws/mlx5dr_action.c | 2237 +
drivers/net/mlx5/hws/mlx5dr_action.h | 253 +++
drivers/net/mlx5/hws/mlx5dr_pat_arg.c | 511 ++
drivers/net/mlx5/hws/mlx5dr_pat_arg.h | 83 +
4 files changed, 3084 insertions(+)
create mode 100644 drivers/net/mlx5/hws/mlx5dr_action.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_action.h
create mode 100644 drivers/net/mlx5/hws/mlx5dr_pat_arg.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_pat_arg.h
diff --git a/drivers/net/mlx5/hws/mlx5dr_action.c
b/drivers/net/mlx5/hws/mlx5dr_action.c
new file mode 100644
index 00..755d5d09cf
--- /dev/null
+++ b/drivers/net/mlx5/hws/mlx5dr_action.c
@@ -0,0 +1,2237 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2022 NVIDIA Corporation & Affiliates
+ */
+
+#include "mlx5dr_internal.h"
+
+#define WIRE_PORT 0x
+
+#define MLX5DR_ACTION_METER_INIT_COLOR_OFFSET 1
+
+/* This is the maximum allowed action order for each table type:
+ * TX: POP_VLAN, CTR, ASO_METER, AS_CT, PUSH_VLAN, MODIFY, ENCAP, Term
+ * RX: TAG, DECAP, POP_VLAN, CTR, ASO_METER, ASO_CT, PUSH_VLAN, MODIFY,
+ * ENCAP, Term
+ * FDB: DECAP, POP_VLAN, CTR, ASO_METER, ASO_CT, PUSH_VLAN, MODIFY,
+ * ENCAP, Term
+ */
+static const uint32_t
action_order_arr[MLX5DR_TABLE_TYPE_MAX][MLX5DR_ACTION_TYP_MAX] = {
+ [MLX5DR_TABLE_TYPE_NIC_RX] = {
+ BIT(MLX5DR_ACTION_TYP_TAG),
+ BIT(MLX5DR_ACTION_TYP_TNL_L2_TO_L2) |
+ BIT(MLX5DR_ACTION_TYP_TNL_L3_TO_L2),
+ BIT(MLX5DR_ACTION_TYP_POP_VLAN),
+ BIT(MLX5DR_ACTION_TYP_POP_VLAN),
+ BIT(MLX5DR_ACTION_TYP_CTR),
+ BIT(MLX5DR_ACTION_TYP_ASO_METER),
+ BIT(MLX5DR_ACTION_TYP_ASO_CT),
+ BIT(MLX5DR_ACTION_TYP_PUSH_VLAN),
+ BIT(MLX5DR_ACTION_TYP_PUSH_VLAN),
+ BIT(MLX5DR_ACTION_TYP_MODIFY_HDR),
+ BIT(MLX5DR_ACTION_TYP_L2_TO_TNL_L2) |
+ BIT(MLX5DR_ACTION_TYP_L2_TO_TNL_L3),
+ BIT(MLX5DR_ACTION_TYP_FT) |
+ BIT(MLX5DR_ACTION_TYP_MISS) |
+ BIT(MLX5DR_ACTION_TYP_TIR) |
+ BIT(MLX5DR_ACTION_TYP_DROP),
+ BIT(MLX5DR_ACTION_TYP_LAST),
+ },
+ [MLX5DR_TABLE_TYPE_NIC_TX] = {
+ BIT(MLX5DR_ACTION_TYP_POP_VLAN),
+ BIT(MLX5DR_ACTION_TYP_POP_VLAN),
+ BIT(MLX5DR_ACTION_TYP_CTR),
+ BIT(MLX5DR_ACTION_TYP_ASO_METER),
+ BIT(MLX5DR_ACTION_TYP_ASO_CT),
+ BIT(MLX5DR_ACTION_TYP_PUSH_VLAN),
+ BIT(MLX5DR_ACTION_TYP_PUSH_VLAN),
+ BIT(MLX5DR_ACTION_TYP_MODIFY_HDR),
+ BIT(MLX5DR_ACTION_TYP_L2_TO_TNL_L2) |
+ BIT(MLX5DR_ACTION_TYP_L2_TO_TNL_L3),
+ BIT(MLX5DR_ACTION_TYP_FT) |
+ BIT(MLX5DR_ACTION_TYP_MISS) |
+ BIT(MLX5DR_ACTION_TYP_DROP),
+ BIT(MLX5DR_ACTION_TYP_LAST),
+ },
+ [MLX5DR_TABLE_TYPE_FDB] = {
+ BIT(MLX5DR_ACTION_TYP_TNL_L2_TO_L2) |
+ BIT(MLX5DR_ACTION_TYP_TNL_L3_TO_L2),
+ BIT(MLX5DR_ACTION_TYP_POP_VLAN),
+ BIT(MLX5DR_ACTION_TYP_POP_VLAN),
+ BIT(MLX5DR_ACTION_TYP_CTR),
+ BIT(MLX5DR_ACTION_TYP_ASO_METER),
+ BIT(MLX5DR_ACTION_TYP_ASO_CT),
+ BIT(MLX5DR_ACTION_TYP_PUSH_VLAN),
+ BIT(MLX5DR_ACTION_TYP_PUSH_VLAN),
+ BIT(MLX5DR_ACTION_TYP_MODIFY_HDR),
+ BIT(MLX5DR_ACTION_TYP_L2_TO_TNL_L2) |
+ BIT(MLX5DR_ACTION_TYP_L2_TO_TNL_L3),
+ BIT(MLX5DR_ACTION_TYP_FT) |
+ BIT(MLX5DR_ACTION_TYP_MISS) |
+ BIT(MLX5DR_ACTION_TYP_VPORT) |
+ BIT(MLX5DR_ACTION_TYP_DROP),
+ BIT(MLX5DR_ACTION_TYP_LAST),
+ },
+};
+
+static int mlx5dr_action_get_shared_stc_nic(struct mlx5dr_context *ctx,
+ enum mlx5dr_context_shared_stc_type
stc_type,
+ uint8_t tbl_type)
+{
+ struct mlx5dr_cmd_stc_modify_attr stc_attr = {0};
+ struct mlx5dr_action_shared_stc *shared_stc;
+ int ret;
+
+ pthread_spin_lock(&ctx->ctrl_lock);
+ if (ctx->common_res[tbl_type].shared_stc[stc_type]) {
+
rte_atomic32_add(&ctx->common_res[tbl_type].shared_stc[stc_type]->refcount, 1);
+ pthread_spin_unlock(&ctx->ctrl_lock);
+ return 0;
+ }
+
+ shared_stc = simple_calloc(1, sizeof(*shared_stc));
+ if (!shared_stc) {
+ DR_LOG(ERR, &
[v6 15/18] net/mlx5/hws: Add HWS rule object
HWS rule objects reside under the matcher, each rule holds
the configuration for the packet fields to match on and the
set of actions to execute over the packet that has the requested
fields. Rules can be created asynchronously in parallel over multiple
queues to different matchers. Each rule is configured to the HW.
Signed-off-by: Erez Shitrit
Signed-off-by: Alex Vesker
---
drivers/net/mlx5/hws/mlx5dr_rule.c | 528 +
drivers/net/mlx5/hws/mlx5dr_rule.h | 50 +++
2 files changed, 578 insertions(+)
create mode 100644 drivers/net/mlx5/hws/mlx5dr_rule.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_rule.h
diff --git a/drivers/net/mlx5/hws/mlx5dr_rule.c
b/drivers/net/mlx5/hws/mlx5dr_rule.c
new file mode 100644
index 00..b27318e6d4
--- /dev/null
+++ b/drivers/net/mlx5/hws/mlx5dr_rule.c
@@ -0,0 +1,528 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2022 NVIDIA Corporation & Affiliates
+ */
+
+#include "mlx5dr_internal.h"
+
+static void mlx5dr_rule_skip(struct mlx5dr_matcher *matcher,
+const struct rte_flow_item *items,
+bool *skip_rx, bool *skip_tx)
+{
+ struct mlx5dr_match_template *mt = matcher->mt[0];
+ const struct flow_hw_port_info *vport;
+ const struct rte_flow_item_ethdev *v;
+
+ /* Flow_src is the 1st priority */
+ if (matcher->attr.optimize_flow_src) {
+ *skip_tx = matcher->attr.optimize_flow_src ==
MLX5DR_MATCHER_FLOW_SRC_WIRE;
+ *skip_rx = matcher->attr.optimize_flow_src ==
MLX5DR_MATCHER_FLOW_SRC_VPORT;
+ return;
+ }
+
+ /* By default FDB rules are added to both RX and TX */
+ *skip_rx = false;
+ *skip_tx = false;
+
+ if (mt->item_flags & MLX5_FLOW_ITEM_REPRESENTED_PORT) {
+ v = items[mt->vport_item_id].spec;
+ vport = flow_hw_conv_port_id(v->port_id);
+ if (unlikely(!vport)) {
+ DR_LOG(ERR, "Fail to map port ID %d, ignoring",
v->port_id);
+ return;
+ }
+
+ if (!vport->is_wire)
+ /* Match vport ID is not WIRE -> Skip RX */
+ *skip_rx = true;
+ else
+ /* Match vport ID is WIRE -> Skip TX */
+ *skip_tx = true;
+ }
+}
+
+static void mlx5dr_rule_init_dep_wqe(struct mlx5dr_send_ring_dep_wqe *dep_wqe,
+struct mlx5dr_rule *rule,
+const struct rte_flow_item *items,
+void *user_data)
+{
+ struct mlx5dr_matcher *matcher = rule->matcher;
+ struct mlx5dr_table *tbl = matcher->tbl;
+ bool skip_rx, skip_tx;
+
+ dep_wqe->rule = rule;
+ dep_wqe->user_data = user_data;
+
+ switch (tbl->type) {
+ case MLX5DR_TABLE_TYPE_NIC_RX:
+ case MLX5DR_TABLE_TYPE_NIC_TX:
+ dep_wqe->rtc_0 = matcher->match_ste.rtc_0->id;
+ dep_wqe->retry_rtc_0 = matcher->col_matcher ?
+
matcher->col_matcher->match_ste.rtc_0->id : 0;
+ dep_wqe->rtc_1 = 0;
+ dep_wqe->retry_rtc_1 = 0;
+ break;
+
+ case MLX5DR_TABLE_TYPE_FDB:
+ mlx5dr_rule_skip(matcher, items, &skip_rx, &skip_tx);
+
+ if (!skip_rx) {
+ dep_wqe->rtc_0 = matcher->match_ste.rtc_0->id;
+ dep_wqe->retry_rtc_0 = matcher->col_matcher ?
+
matcher->col_matcher->match_ste.rtc_0->id : 0;
+ } else {
+ dep_wqe->rtc_0 = 0;
+ dep_wqe->retry_rtc_0 = 0;
+ }
+
+ if (!skip_tx) {
+ dep_wqe->rtc_1 = matcher->match_ste.rtc_1->id;
+ dep_wqe->retry_rtc_1 = matcher->col_matcher ?
+
matcher->col_matcher->match_ste.rtc_1->id : 0;
+ } else {
+ dep_wqe->rtc_1 = 0;
+ dep_wqe->retry_rtc_1 = 0;
+ }
+
+ break;
+
+ default:
+ assert(false);
+ break;
+ }
+}
+
+static void mlx5dr_rule_gen_comp(struct mlx5dr_send_engine *queue,
+struct mlx5dr_rule *rule,
+bool err,
+void *user_data,
+enum mlx5dr_rule_status rule_status_on_succ)
+{
+ enum rte_flow_op_status comp_status;
+
+ if (!err) {
+ comp_status = RTE_FLOW_OP_SUCCESS;
+ rule->status = rule_status_on_succ;
[v6 17/18] net/mlx5/hws: Add HWS debug layer
From: Hamdan Igbaria
The debug layer is used to generate a debug CSV file
containing details of the context, table, matcher, rules
and other useful debug information.
Signed-off-by: Hamdan Igbaria
Signed-off-by: Alex Vesker
---
drivers/net/mlx5/hws/mlx5dr_debug.c | 462
drivers/net/mlx5/hws/mlx5dr_debug.h | 28 ++
2 files changed, 490 insertions(+)
create mode 100644 drivers/net/mlx5/hws/mlx5dr_debug.c
create mode 100644 drivers/net/mlx5/hws/mlx5dr_debug.h
diff --git a/drivers/net/mlx5/hws/mlx5dr_debug.c
b/drivers/net/mlx5/hws/mlx5dr_debug.c
new file mode 100644
index 00..890a761c48
--- /dev/null
+++ b/drivers/net/mlx5/hws/mlx5dr_debug.c
@@ -0,0 +1,462 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2022 NVIDIA Corporation & Affiliates
+ */
+
+#include "mlx5dr_internal.h"
+
+const char *mlx5dr_debug_action_type_str[] = {
+ [MLX5DR_ACTION_TYP_LAST] = "LAST",
+ [MLX5DR_ACTION_TYP_TNL_L2_TO_L2] = "TNL_L2_TO_L2",
+ [MLX5DR_ACTION_TYP_L2_TO_TNL_L2] = "L2_TO_TNL_L2",
+ [MLX5DR_ACTION_TYP_TNL_L3_TO_L2] = "TNL_L3_TO_L2",
+ [MLX5DR_ACTION_TYP_L2_TO_TNL_L3] = "L2_TO_TNL_L3",
+ [MLX5DR_ACTION_TYP_DROP] = "DROP",
+ [MLX5DR_ACTION_TYP_TIR] = "TIR",
+ [MLX5DR_ACTION_TYP_FT] = "FT",
+ [MLX5DR_ACTION_TYP_CTR] = "CTR",
+ [MLX5DR_ACTION_TYP_TAG] = "TAG",
+ [MLX5DR_ACTION_TYP_MODIFY_HDR] = "MODIFY_HDR",
+ [MLX5DR_ACTION_TYP_VPORT] = "VPORT",
+ [MLX5DR_ACTION_TYP_MISS] = "DEFAULT_MISS",
+ [MLX5DR_ACTION_TYP_POP_VLAN] = "POP_VLAN",
+ [MLX5DR_ACTION_TYP_PUSH_VLAN] = "PUSH_VLAN",
+ [MLX5DR_ACTION_TYP_ASO_METER] = "ASO_METER",
+ [MLX5DR_ACTION_TYP_ASO_CT] = "ASO_CT",
+};
+
+static_assert(ARRAY_SIZE(mlx5dr_debug_action_type_str) ==
MLX5DR_ACTION_TYP_MAX,
+ "Missing mlx5dr_debug_action_type_str");
+
+const char *mlx5dr_debug_action_type_to_str(enum mlx5dr_action_type
action_type)
+{
+ return mlx5dr_debug_action_type_str[action_type];
+}
+
+static int
+mlx5dr_debug_dump_matcher_template_definer(FILE *f,
+ struct mlx5dr_match_template *mt)
+{
+ struct mlx5dr_definer *definer = mt->definer;
+ int i, ret;
+
+ ret = fprintf(f, "%d,0x%" PRIx64 ",0x%" PRIx64 ",%d,%d,",
+ MLX5DR_DEBUG_RES_TYPE_MATCHER_TEMPLATE_DEFINER,
+ (uint64_t)(uintptr_t)definer,
+ (uint64_t)(uintptr_t)mt,
+ definer->obj->id,
+ definer->type);
+ if (ret < 0) {
+ rte_errno = EINVAL;
+ return rte_errno;
+ }
+
+ for (i = 0; i < DW_SELECTORS; i++) {
+ ret = fprintf(f, "0x%x%s", definer->dw_selector[i],
+ (i == DW_SELECTORS - 1) ? "," : "-");
+ if (ret < 0) {
+ rte_errno = EINVAL;
+ return rte_errno;
+ }
+ }
+
+ for (i = 0; i < BYTE_SELECTORS; i++) {
+ ret = fprintf(f, "0x%x%s", definer->byte_selector[i],
+ (i == BYTE_SELECTORS - 1) ? "," : "-");
+ if (ret < 0) {
+ rte_errno = EINVAL;
+ return rte_errno;
+ }
+ }
+
+ for (i = 0; i < MLX5DR_JUMBO_TAG_SZ; i++) {
+ ret = fprintf(f, "%02x", definer->mask.jumbo[i]);
+ if (ret < 0) {
+ rte_errno = EINVAL;
+ return rte_errno;
+ }
+ }
+
+ ret = fprintf(f, "\n");
+ if (ret < 0) {
+ rte_errno = EINVAL;
+ return rte_errno;
+ }
+
+ return 0;
+}
+
+static int
+mlx5dr_debug_dump_matcher_match_template(FILE *f, struct mlx5dr_matcher
*matcher)
+{
+ bool is_root = matcher->tbl->level == MLX5DR_ROOT_LEVEL;
+ int i, ret;
+
+ for (i = 0; i < matcher->num_of_mt; i++) {
+ struct mlx5dr_match_template *mt = matcher->mt[i];
+
+ ret = fprintf(f, "%d,0x%" PRIx64 ",0x%" PRIx64 ",%d,%d\n",
+ MLX5DR_DEBUG_RES_TYPE_MATCHER_MATCH_TEMPLATE,
+ (uint64_t)(uintptr_t)mt,
+ (uint64_t)(uintptr_t)matcher,
+ is_root ? 0 : mt->fc_sz,
+ mt->flags);
+ if (ret < 0) {
+ rte_errno = EINVAL;
+ return rte_errno;
+ }
+
+ i
[v6 18/18] net/mlx5/hws: Enable HWS
Replace stub implenation of HWS with mlx5dr code.
Signed-off-by: Alex Vesker
---
doc/guides/nics/mlx5.rst | 5 +-
doc/guides/rel_notes/release_22_11.rst | 4 +
drivers/common/mlx5/linux/meson.build| 9 +-
drivers/net/mlx5/hws/meson.build | 18 +
drivers/net/mlx5/{mlx5_dr.h => hws/mlx5dr.h} | 201 --
drivers/net/mlx5/hws/mlx5dr_internal.h | 93 +
drivers/net/mlx5/meson.build | 7 +-
drivers/net/mlx5/mlx5.c | 6 +-
drivers/net/mlx5/mlx5.h | 7 +-
drivers/net/mlx5/mlx5_devx.c | 2 +-
drivers/net/mlx5/mlx5_dr.c | 383 ---
drivers/net/mlx5/mlx5_flow.c | 2 +
drivers/net/mlx5/mlx5_flow.h | 11 +-
drivers/net/mlx5/mlx5_flow_hw.c | 10 +-
14 files changed, 326 insertions(+), 432 deletions(-)
create mode 100644 drivers/net/mlx5/hws/meson.build
rename drivers/net/mlx5/{mlx5_dr.h => hws/mlx5dr.h} (67%)
create mode 100644 drivers/net/mlx5/hws/mlx5dr_internal.h
delete mode 100644 drivers/net/mlx5/mlx5_dr.c
diff --git a/doc/guides/nics/mlx5.rst b/doc/guides/nics/mlx5.rst
index bb436892a0..303eb17714 100644
--- a/doc/guides/nics/mlx5.rst
+++ b/doc/guides/nics/mlx5.rst
@@ -539,7 +539,10 @@ Limitations
- WQE based high scaling and safer flow insertion/destruction.
- Set ``dv_flow_en`` to 2 in order to enable HW steering.
-- Async queue-based ``rte_flow_q`` APIs supported only.
+- Async queue-based ``rte_flow_async`` APIs supported only.
+- NIC ConnectX-5 and before are not supported.
+- Partial match with item template is not supported.
+- IPv6 5-tuple matching is not supported.
- Match on GRE header supports the following fields:
diff --git a/doc/guides/rel_notes/release_22_11.rst
b/doc/guides/rel_notes/release_22_11.rst
index 1c3daf141d..cdc5837f1d 100644
--- a/doc/guides/rel_notes/release_22_11.rst
+++ b/doc/guides/rel_notes/release_22_11.rst
@@ -249,6 +249,10 @@ New Features
sysfs entries to adjust the minimum and maximum uncore frequency values,
which works on Linux with Intel hardware only.
+* **Updated Nvidia mlx5 driver.**
+
+ * Added fully support for queue based async HW steering to the PMD.
+
* **Rewritten pmdinfo script.**
The ``dpdk-pmdinfo.py`` script was rewritten to produce valid JSON only.
diff --git a/drivers/common/mlx5/linux/meson.build
b/drivers/common/mlx5/linux/meson.build
index e6b32eb84d..84e2a1ad8c 100644
--- a/drivers/common/mlx5/linux/meson.build
+++ b/drivers/common/mlx5/linux/meson.build
@@ -8,6 +8,7 @@ dlopen_ibverbs = (get_option('ibverbs_link') == 'dlopen')
LIB_GLUE_BASE = 'librte_common_mlx5_glue.so'
LIB_GLUE_VERSION = abi_version
LIB_GLUE = LIB_GLUE_BASE + '.' + LIB_GLUE_VERSION
+mlx5_config = configuration_data()
if dlopen_ibverbs
dpdk_conf.set('RTE_IBVERBS_LINK_DLOPEN', 1)
cflags += [
@@ -223,15 +224,15 @@ if libmtcr_ul_found
[ 'HAVE_MLX5_MSTFLINT', 'mstflint/mtcr.h', 'mopen' ],
]
endif
-config = configuration_data()
+
foreach arg:has_sym_args
-config.set(arg[0], cc.has_header_symbol(arg[1], arg[2], dependencies:
libs))
+mlx5_config.set(arg[0], cc.has_header_symbol(arg[1], arg[2], dependencies:
libs))
endforeach
foreach arg:has_member_args
file_prefix = '#include <' + arg[1] + '>'
-config.set(arg[0], cc.has_member(arg[2], arg[3], prefix : file_prefix,
dependencies: libs))
+mlx5_config.set(arg[0], cc.has_member(arg[2], arg[3], prefix :
file_prefix, dependencies: libs))
endforeach
-configure_file(output : 'mlx5_autoconf.h', configuration : config)
+configure_file(output : 'mlx5_autoconf.h', configuration : mlx5_config)
# Build Glue Library
if dlopen_ibverbs
diff --git a/drivers/net/mlx5/hws/meson.build b/drivers/net/mlx5/hws/meson.build
new file mode 100644
index 00..d2bb864fd2
--- /dev/null
+++ b/drivers/net/mlx5/hws/meson.build
@@ -0,0 +1,18 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2022 NVIDIA Corporation & Affiliates
+
+includes += include_directories('.')
+sources += files(
+'mlx5dr_context.c',
+'mlx5dr_table.c',
+'mlx5dr_matcher.c',
+'mlx5dr_rule.c',
+'mlx5dr_action.c',
+'mlx5dr_buddy.c',
+'mlx5dr_pool.c',
+'mlx5dr_cmd.c',
+'mlx5dr_send.c',
+'mlx5dr_definer.c',
+'mlx5dr_debug.c',
+'mlx5dr_pat_arg.c',
+)
diff --git a/drivers/net/mlx5/mlx5_dr.h b/drivers/net/mlx5/hws/mlx5dr.h
similarity index 67%
rename from drivers/net/mlx5/mlx5_dr.h
rename to drivers/net/mlx5/hws/mlx5dr.h
index d0b2c15652..f8de27c615 100644
--- a/drivers/net/mlx5/mlx5_dr.h
++
RE: [PATCH v3 1/2] common/mlx5: fix build disabling
> -Original Message-
> From: Thomas Monjalon
> Sent: Sunday, October 30, 2022 1:08 PM
> To: dev@dpdk.org
> Cc: Alex Vesker ; ferruh.yi...@amd.com;
> andrew.rybche...@oktetlabs.ru; Raslan Darawsheh ;
> david.march...@redhat.com; Matan Azrad ; Fan
> Zhang ; Ashish Gupta
> ; Slava Ovsiienko ;
> Ori Kam
> Subject: [PATCH v3 1/2] common/mlx5: fix build disabling
>
> If the dependency common/mlx5 is explicitly disabled, but net/mlx5 is not
> explicitly disabled, Meson will read the full recipe of net/mlx5 and will fail
> when accessing a variable from common/mlx5:
> drivers/net/mlx5/meson.build:76:4: ERROR: Unknown variable
> "mlx5_config".
>
> The solution is to stop parsing net/mlx5 if common/mlx5 is disabled.
> The deps array must be defined before stopping, in order to automatically
> disable the build of net/mlx5 and print the reason.
>
> The same protection is applied to other mlx5 drivers, so it will allow using
> the
> variable mlx5_config in future.
>
> Fixes: 22681deead3e ("net/mlx5/hws: enable hardware steering")
>
> Reported-by: Andrew Rybchenko
> Signed-off-by: Thomas Monjalon
> Tested-by: Andrew Rybchenko
> Reviewed-by: David Marchand
> Acked-by: Matan Azrad
> ---
> drivers/compress/mlx5/meson.build | 5 +
> drivers/crypto/mlx5/meson.build | 5 +
> drivers/net/mlx5/meson.build | 5 +
> drivers/regex/mlx5/meson.build| 5 +
> drivers/vdpa/mlx5/meson.build | 5 +
> 5 files changed, 25 insertions(+)
>
> diff --git a/drivers/compress/mlx5/meson.build
> b/drivers/compress/mlx5/meson.build
> index 7aac329986..49ce3aff46 100644
> --- a/drivers/compress/mlx5/meson.build
> +++ b/drivers/compress/mlx5/meson.build
> @@ -9,6 +9,11 @@ endif
>
> fmt_name = 'mlx5_compress'
> deps += ['common_mlx5', 'eal', 'compressdev']
> +if not ('mlx5' in common_drivers)
> +# avoid referencing undefined variables from common/mlx5
> +subdir_done()
> +endif
> +
> sources = files(
> 'mlx5_compress.c',
> )
> diff --git a/drivers/crypto/mlx5/meson.build
> b/drivers/crypto/mlx5/meson.build index 9d9c9c00bc..7521c4c671 100644
> --- a/drivers/crypto/mlx5/meson.build
> +++ b/drivers/crypto/mlx5/meson.build
> @@ -9,6 +9,11 @@ endif
>
> fmt_name = 'mlx5_crypto'
> deps += ['common_mlx5', 'eal', 'cryptodev']
> +if not ('mlx5' in common_drivers)
> +# avoid referencing undefined variables from common/mlx5
> +subdir_done()
> +endif
> +
> sources = files(
> 'mlx5_crypto.c',
> 'mlx5_crypto_dek.c',
> diff --git a/drivers/net/mlx5/meson.build b/drivers/net/mlx5/meson.build
> index ff84448186..fa15158039 100644
> --- a/drivers/net/mlx5/meson.build
> +++ b/drivers/net/mlx5/meson.build
> @@ -9,6 +9,11 @@ if not (is_linux or is_windows) endif
>
> deps += ['hash', 'common_mlx5']
> +if not ('mlx5' in common_drivers)
> +# avoid referencing undefined variables from common/mlx5
> +subdir_done()
> +endif
> +
> headers = files('rte_pmd_mlx5.h')
> sources = files(
> 'mlx5.c',
> diff --git a/drivers/regex/mlx5/meson.build
> b/drivers/regex/mlx5/meson.build index e553dcb83d..70edc5b6da 100644
> --- a/drivers/regex/mlx5/meson.build
> +++ b/drivers/regex/mlx5/meson.build
> @@ -8,6 +8,11 @@ if not is_linux
> endif
>
> deps += ['common_mlx5', 'eal', 'regexdev']
> +if not ('mlx5' in common_drivers)
> +# avoid referencing undefined variables from common/mlx5
> +subdir_done()
> +endif
> +
> sources = files(
> 'mlx5_regex.c',
> 'mlx5_rxp.c',
> diff --git a/drivers/vdpa/mlx5/meson.build
> b/drivers/vdpa/mlx5/meson.build index 9d8dbb1a82..54a4eac6f4 100644
> --- a/drivers/vdpa/mlx5/meson.build
> +++ b/drivers/vdpa/mlx5/meson.build
> @@ -8,6 +8,11 @@ if not is_linux
> endif
>
> deps += ['hash', 'common_mlx5', 'vhost', 'pci', 'eal', 'sched']
> +if not ('mlx5' in common_drivers)
> +# avoid referencing undefined variables from common/mlx5
> +subdir_done()
> +endif
> +
> sources = files(
> 'mlx5_vdpa.c',
> 'mlx5_vdpa_mem.c',
> --
> 2.36.1
Acked-by: Alex Vesker
RE: [PATCH v3 2/2] common/mlx5: move build config initialization and check
> -Original Message-
> From: Thomas Monjalon
> Sent: Sunday, October 30, 2022 1:08 PM
> To: dev@dpdk.org
> Cc: Alex Vesker ; ferruh.yi...@amd.com;
> andrew.rybche...@oktetlabs.ru; Raslan Darawsheh ;
> david.march...@redhat.com; Matan Azrad ; Slava
> Ovsiienko
> Subject: [PATCH v3 2/2] common/mlx5: move build config initialization and
> check
>
> The variable mlx5_config may be used by other mlx5 drivers and should be
> always initialized.
> By moving its initialization (with configuration file generation), it is made
> consistent for Linux and Windows builds.
>
> And the check of mlx5_config in net/mlx5 is moved at the top of
> net/mlx5/hws/meson.build so HWS requirements are in the right context.
>
> Signed-off-by: Thomas Monjalon
> Tested-by: Andrew Rybchenko
> Reviewed-by: David Marchand
> Acked-by: Matan Azrad
> ---
> drivers/common/mlx5/linux/meson.build | 3 +--
> drivers/common/mlx5/meson.build | 2 ++
> drivers/common/mlx5/windows/meson.build | 4
> drivers/net/mlx5/hws/meson.build| 4
> drivers/net/mlx5/meson.build| 4 +---
> 5 files changed, 8 insertions(+), 9 deletions(-)
>
> diff --git a/drivers/common/mlx5/linux/meson.build
> b/drivers/common/mlx5/linux/meson.build
> index 84e2a1ad8c..7e1575efc8 100644
> --- a/drivers/common/mlx5/linux/meson.build
> +++ b/drivers/common/mlx5/linux/meson.build
> @@ -8,7 +8,6 @@ dlopen_ibverbs = (get_option('ibverbs_link') == 'dlopen')
> LIB_GLUE_BASE = 'librte_common_mlx5_glue.so'
> LIB_GLUE_VERSION = abi_version
> LIB_GLUE = LIB_GLUE_BASE + '.' + LIB_GLUE_VERSION -mlx5_config =
> configuration_data() if dlopen_ibverbs
> dpdk_conf.set('RTE_IBVERBS_LINK_DLOPEN', 1)
> cflags += [
> @@ -232,7 +231,6 @@ foreach arg:has_member_args
> file_prefix = '#include <' + arg[1] + '>'
> mlx5_config.set(arg[0], cc.has_member(arg[2], arg[3], prefix :
> file_prefix,
> dependencies: libs)) endforeach -configure_file(output : 'mlx5_autoconf.h',
> configuration : mlx5_config)
>
> # Build Glue Library
> if dlopen_ibverbs
> @@ -243,6 +241,7 @@ if dlopen_ibverbs
> dlopen_install_dir = [ eal_pmd_path + '-glue' ]
> dlopen_includes = [global_inc]
> dlopen_includes +=
> include_directories('../../../../lib/eal/include/generic')
> +dlopen_includes += include_directories('..')
> shared_lib = shared_library(
> dlopen_lib_name,
> dlopen_sources,
> diff --git a/drivers/common/mlx5/meson.build
> b/drivers/common/mlx5/meson.build index 6ddbde7e8f..60ccd95cbc
> 100644
> --- a/drivers/common/mlx5/meson.build
> +++ b/drivers/common/mlx5/meson.build
> @@ -37,4 +37,6 @@ else
> cflags += [ '-UPEDANTIC' ]
> endif
>
> +mlx5_config = configuration_data()
> subdir(exec_env)
> +configure_file(output: 'mlx5_autoconf.h', configuration: mlx5_config)
> diff --git a/drivers/common/mlx5/windows/meson.build
> b/drivers/common/mlx5/windows/meson.build
> index edbbaa9ae1..cc486014a8 100644
> --- a/drivers/common/mlx5/windows/meson.build
> +++ b/drivers/common/mlx5/windows/meson.build
> @@ -39,7 +39,3 @@ if get_option('buildtype').contains('debug')
> else
> cflags += [ '-UPEDANTIC' ]
> endif
> -
> -# Generate an empty mlx5_autoconf.h file for compatibility with Linux -
> config = configuration_data() -configure_file(output : 'mlx5_autoconf.h',
> configuration : config) diff --git a/drivers/net/mlx5/hws/meson.build
> b/drivers/net/mlx5/hws/meson.build
> index d2bb864fd2..38776d5163 100644
> --- a/drivers/net/mlx5/hws/meson.build
> +++ b/drivers/net/mlx5/hws/meson.build
> @@ -1,6 +1,10 @@
> # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2022 NVIDIA
> Corporation & Affiliates
>
> +if not (is_linux and mlx5_config.get('HAVE_IBV_FLOW_DV_SUPPORT', false))
> +subdir_done()
> +endif
> +
> includes += include_directories('.')
> sources += files(
> 'mlx5dr_context.c',
> diff --git a/drivers/net/mlx5/meson.build b/drivers/net/mlx5/meson.build
> index fa15158039..f1aab18f82 100644
> --- a/drivers/net/mlx5/meson.build
> +++ b/drivers/net/mlx5/meson.build
> @@ -78,6 +78,4 @@ testpmd_sources += files('mlx5_testpmd.c')
>
> subdir(exec_env)
>
> -if (is_linux and mlx5_config.get('HAVE_IBV_FLOW_DV_SUPPORT', false))
> -subdir('hws')
> -endif
> +subdir('hws')
> --
> 2.36.1
Acked-by: Alex Vesker
[PATCH] net/mlx5/hws: remove deprecated rte_atomic
The use of rte_atomic functions is deprecated and is not
required in HWS code. HWS refcounts are used only during
control and always under lock.
Fixes: f8c8a6d8440d ("net/mlx5/hws: add action object")
Signed-off-by: Alex Vesker
---
drivers/net/mlx5/hws/mlx5dr_action.c | 8 +++-
drivers/net/mlx5/hws/mlx5dr_action.h | 2 +-
drivers/net/mlx5/hws/mlx5dr_pat_arg.c | 8 +++-
drivers/net/mlx5/hws/mlx5dr_pat_arg.h | 2 +-
4 files changed, 8 insertions(+), 12 deletions(-)
diff --git a/drivers/net/mlx5/hws/mlx5dr_action.c
b/drivers/net/mlx5/hws/mlx5dr_action.c
index 755d5d09cf..a9e12aa1f5 100644
--- a/drivers/net/mlx5/hws/mlx5dr_action.c
+++ b/drivers/net/mlx5/hws/mlx5dr_action.c
@@ -83,7 +83,7 @@ static int mlx5dr_action_get_shared_stc_nic(struct
mlx5dr_context *ctx,
pthread_spin_lock(&ctx->ctrl_lock);
if (ctx->common_res[tbl_type].shared_stc[stc_type]) {
-
rte_atomic32_add(&ctx->common_res[tbl_type].shared_stc[stc_type]->refcount, 1);
+ ctx->common_res[tbl_type].shared_stc[stc_type]->refcount++;
pthread_spin_unlock(&ctx->ctrl_lock);
return 0;
}
@@ -123,9 +123,7 @@ static int mlx5dr_action_get_shared_stc_nic(struct
mlx5dr_context *ctx,
}
ctx->common_res[tbl_type].shared_stc[stc_type] = shared_stc;
-
-
rte_atomic32_init(&ctx->common_res[tbl_type].shared_stc[stc_type]->refcount);
-
rte_atomic32_set(&ctx->common_res[tbl_type].shared_stc[stc_type]->refcount, 1);
+ ctx->common_res[tbl_type].shared_stc[stc_type]->refcount = 1;
pthread_spin_unlock(&ctx->ctrl_lock);
@@ -145,7 +143,7 @@ static void mlx5dr_action_put_shared_stc_nic(struct
mlx5dr_context *ctx,
struct mlx5dr_action_shared_stc *shared_stc;
pthread_spin_lock(&ctx->ctrl_lock);
- if
(!rte_atomic32_dec_and_test(&ctx->common_res[tbl_type].shared_stc[stc_type]->refcount))
{
+ if (--ctx->common_res[tbl_type].shared_stc[stc_type]->refcount) {
pthread_spin_unlock(&ctx->ctrl_lock);
return;
}
diff --git a/drivers/net/mlx5/hws/mlx5dr_action.h
b/drivers/net/mlx5/hws/mlx5dr_action.h
index f14d91f994..3b31ffc90e 100644
--- a/drivers/net/mlx5/hws/mlx5dr_action.h
+++ b/drivers/net/mlx5/hws/mlx5dr_action.h
@@ -70,7 +70,7 @@ struct mlx5dr_action_default_stc {
struct mlx5dr_action_shared_stc {
struct mlx5dr_pool_chunk remove_header;
- rte_atomic32_t refcount;
+ uint32_t refcount;
};
struct mlx5dr_actions_apply_data {
diff --git a/drivers/net/mlx5/hws/mlx5dr_pat_arg.c
b/drivers/net/mlx5/hws/mlx5dr_pat_arg.c
index 46fdc8ce68..df451f1ae0 100644
--- a/drivers/net/mlx5/hws/mlx5dr_pat_arg.c
+++ b/drivers/net/mlx5/hws/mlx5dr_pat_arg.c
@@ -128,7 +128,7 @@ mlx5dr_pat_get_existing_cached_pattern(struct
mlx5dr_pattern_cache *cache,
/* LRU: move it to be first in the list */
LIST_REMOVE(cached_pattern, next);
LIST_INSERT_HEAD(&cache->head, cached_pattern, next);
- rte_atomic32_add(&cached_pattern->refcount, 1);
+ cached_pattern->refcount++;
}
return cached_pattern;
@@ -179,9 +179,7 @@ mlx5dr_pat_add_pattern_to_cache(struct mlx5dr_pattern_cache
*cache,
num_of_actions * MLX5DR_MODIFY_ACTION_SIZE);
LIST_INSERT_HEAD(&cache->head, cached_pattern, next);
-
- rte_atomic32_init(&cached_pattern->refcount);
- rte_atomic32_set(&cached_pattern->refcount, 1);
+ cached_pattern->refcount = 1;
return cached_pattern;
@@ -212,7 +210,7 @@ mlx5dr_pat_put_pattern(struct mlx5dr_pattern_cache *cache,
goto out;
}
- if (!rte_atomic32_dec_and_test(&cached_pattern->refcount))
+ if (--cached_pattern->refcount)
goto out;
mlx5dr_pat_remove_pattern(cached_pattern);
diff --git a/drivers/net/mlx5/hws/mlx5dr_pat_arg.h
b/drivers/net/mlx5/hws/mlx5dr_pat_arg.h
index 8a4670427f..d9353e9a3e 100644
--- a/drivers/net/mlx5/hws/mlx5dr_pat_arg.h
+++ b/drivers/net/mlx5/hws/mlx5dr_pat_arg.h
@@ -35,7 +35,7 @@ struct mlx5dr_pat_cached_pattern {
uint8_t *data;
uint16_t num_of_actions;
} mh_data;
- rte_atomic32_t refcount;
+ uint32_t refcount;
LIST_ENTRY(mlx5dr_pat_cached_pattern) next;
};
--
2.18.1
[PATCH] net/mlx5/hws: fix memory leak on general pool db init
On elemend db init we allocated the element_manager
which was unused and not freed.
Fixes: b4dd7bcb0dcbe ("net/mlx5/hws: add pool and buddy")
Signed-off-by: Alex Vesker
Reviewed-by: Erez Shitrit
---
drivers/net/mlx5/hws/mlx5dr_pool.c | 7 ---
1 file changed, 7 deletions(-)
diff --git a/drivers/net/mlx5/hws/mlx5dr_pool.c
b/drivers/net/mlx5/hws/mlx5dr_pool.c
index fdbd3d438d..af6a5c743b 100644
--- a/drivers/net/mlx5/hws/mlx5dr_pool.c
+++ b/drivers/net/mlx5/hws/mlx5dr_pool.c
@@ -464,13 +464,6 @@ static void mlx5dr_pool_general_element_db_uninit(struct
mlx5dr_pool *pool)
*/
static int mlx5dr_pool_general_element_db_init(struct mlx5dr_pool *pool)
{
- pool->db.element_manager = simple_calloc(1,
sizeof(*pool->db.element_manager));
- if (!pool->db.element_manager) {
- DR_LOG(ERR, "No mem for general elemnt_manager");
- rte_errno = ENOMEM;
- return rte_errno;
- }
-
pool->p_db_uninit = &mlx5dr_pool_general_element_db_uninit;
pool->p_get_chunk = &mlx5dr_pool_general_element_db_get_chunk;
pool->p_put_chunk = &mlx5dr_pool_general_element_db_put_chunk;
--
2.18.1
[v1 00/16] net/mlx5/hws: support range and partial hash matching
ConnectX and BlueField devices which support HWS are capable of performing advanced match, range-match and hashing operations on packet headers. This patch-set introduces support for (1) range matching - allows combining range and exact match over provided item fields using spec, last and mask. A useful example for Range and exact match is matching exact IP and port range. This series also introduces (2) partial hash, which allows using the same matcher and performing partial hashing in case of multiple match templates are provided. The matcher create code finds the union and intersection fields to split the hash and match in a way the HW can support. This increases the match templates combinations possible although it may limit the maximum PPS. Since (1) and (2) are not fully support on current HW the implementation is done on top of a HW like FW API using WQEs. Alex Vesker (16): net/mlx5/hws: support synchronous drain net/mlx5/hws: matcher remove AT and MT limitation net/mlx5/hws: support GTA WQE write using FW command net/mlx5/hws: add capability query for gen wqe command net/mlx5/hws: align RTC create command with PRM format net/mlx5/hws: add send FW match STE using gen WQE net/mlx5/hws: add send FW range STE WQE net/mlx5/hws: move matcher size check to function net/mlx5/hws: support range match net/mlx5/hws: redesign definer create net/mlx5/hws: support partial hash net/mlx5/hws: add range definer creation support net/mlx5/hws: add FW WQE rule creation logic net/mlx5/hws: add debug dump support for range and hash net/mlx5/hws: rename pattern cache object net/mlx5/hws: cache definer for reuse drivers/common/mlx5/mlx5_prm.h | 53 +- drivers/net/mlx5/hws/mlx5dr.h | 6 +- drivers/net/mlx5/hws/mlx5dr_cmd.c | 72 ++- drivers/net/mlx5/hws/mlx5dr_cmd.h | 27 +- drivers/net/mlx5/hws/mlx5dr_context.c | 12 +- drivers/net/mlx5/hws/mlx5dr_context.h | 1 + drivers/net/mlx5/hws/mlx5dr_debug.c| 41 +- drivers/net/mlx5/hws/mlx5dr_debug.h| 4 +- drivers/net/mlx5/hws/mlx5dr_definer.c | 750 + drivers/net/mlx5/hws/mlx5dr_definer.h | 44 +- drivers/net/mlx5/hws/mlx5dr_internal.h | 2 +- drivers/net/mlx5/hws/mlx5dr_matcher.c | 279 ++--- drivers/net/mlx5/hws/mlx5dr_matcher.h | 43 +- drivers/net/mlx5/hws/mlx5dr_pat_arg.c | 49 +- drivers/net/mlx5/hws/mlx5dr_pat_arg.h | 6 +- drivers/net/mlx5/hws/mlx5dr_rule.c | 193 ++- drivers/net/mlx5/hws/mlx5dr_rule.h | 2 + drivers/net/mlx5/hws/mlx5dr_send.c | 169 +- drivers/net/mlx5/hws/mlx5dr_send.h | 24 +- drivers/net/mlx5/mlx5_flow_hw.c| 2 +- 20 files changed, 1475 insertions(+), 304 deletions(-) -- 2.18.1
[v1 06/16] net/mlx5/hws: add send FW match STE using gen WQE
Send STE WQE function wraps the send WQE command to support WQE
build and FDB abstraction. Sending using FW is different from
sending from HW since FW returns the completion immediately which
requires us to retry on failure and prepare the completion as
part of the send process.
Signed-off-by: Alex Vesker
---
drivers/net/mlx5/hws/mlx5dr_send.c | 134 +
drivers/net/mlx5/hws/mlx5dr_send.h | 7 +-
2 files changed, 140 insertions(+), 1 deletion(-)
diff --git a/drivers/net/mlx5/hws/mlx5dr_send.c
b/drivers/net/mlx5/hws/mlx5dr_send.c
index a507e5f626..a9958df4f2 100644
--- a/drivers/net/mlx5/hws/mlx5dr_send.c
+++ b/drivers/net/mlx5/hws/mlx5dr_send.c
@@ -235,6 +235,140 @@ void mlx5dr_send_ste(struct mlx5dr_send_engine *queue,
send_attr->fence = fence;
}
+static
+int mlx5dr_send_wqe_fw(struct ibv_context *ibv_ctx,
+ uint32_t pd_num,
+ struct mlx5dr_send_engine_post_attr *send_attr,
+ struct mlx5dr_wqe_gta_ctrl_seg *send_wqe_ctrl,
+ void *send_wqe_match_data,
+ void *send_wqe_match_tag,
+ bool is_jumbo,
+ uint8_t gta_opcode)
+{
+ bool has_match = send_wqe_match_data || send_wqe_match_tag;
+ struct mlx5dr_wqe_gta_data_seg_ste gta_wqe_data0 = {0};
+ struct mlx5dr_wqe_gta_ctrl_seg gta_wqe_ctrl = {0};
+ struct mlx5dr_cmd_generate_wqe_attr attr = {0};
+ struct mlx5dr_wqe_ctrl_seg wqe_ctrl = {0};
+ struct mlx5_cqe64 cqe;
+ uint32_t flags = 0;
+ int ret;
+
+ /* Set WQE control */
+ wqe_ctrl.opmod_idx_opcode =
+ rte_cpu_to_be_32((send_attr->opmod << 24) | send_attr->opcode);
+ wqe_ctrl.qpn_ds =
+ rte_cpu_to_be_32((send_attr->len + sizeof(struct
mlx5dr_wqe_ctrl_seg)) / 16);
+ flags |= send_attr->notify_hw ? MLX5_WQE_CTRL_CQ_UPDATE : 0;
+ wqe_ctrl.flags = rte_cpu_to_be_32(flags);
+ wqe_ctrl.imm = rte_cpu_to_be_32(send_attr->id);
+
+ /* Set GTA WQE CTRL */
+ memcpy(gta_wqe_ctrl.stc_ix, send_wqe_ctrl->stc_ix,
sizeof(send_wqe_ctrl->stc_ix));
+ gta_wqe_ctrl.op_dirix = htobe32(gta_opcode << 28);
+
+ /* Set GTA match WQE DATA */
+ if (has_match) {
+ if (send_wqe_match_data)
+ memcpy(>a_wqe_data0, send_wqe_match_data,
sizeof(gta_wqe_data0));
+ else
+ mlx5dr_send_wqe_set_tag(>a_wqe_data0,
send_wqe_match_tag, is_jumbo);
+
+ gta_wqe_data0.rsvd1_definer =
htobe32(send_attr->match_definer_id << 8);
+ attr.gta_data_0 = (uint8_t *)>a_wqe_data0;
+ }
+
+ attr.pdn = pd_num;
+ attr.wqe_ctrl = (uint8_t *)&wqe_ctrl;
+ attr.gta_ctrl = (uint8_t *)>a_wqe_ctrl;
+
+send_wqe:
+ ret = mlx5dr_cmd_generate_wqe(ibv_ctx, &attr, &cqe);
+ if (ret) {
+ DR_LOG(ERR, "Failed to write WQE using command");
+ return ret;
+ }
+
+ if ((mlx5dv_get_cqe_opcode(&cqe) == MLX5_CQE_REQ) &&
+ (rte_be_to_cpu_32(cqe.byte_cnt) >> 31 == 0)) {
+ *send_attr->used_id = send_attr->id;
+ return 0;
+ }
+
+ /* Retry if rule failed */
+ if (send_attr->retry_id) {
+ wqe_ctrl.imm = rte_cpu_to_be_32(send_attr->retry_id);
+ send_attr->id = send_attr->retry_id;
+ send_attr->retry_id = 0;
+ goto send_wqe;
+ }
+
+ return -1;
+}
+
+void mlx5dr_send_stes_fw(struct mlx5dr_send_engine *queue,
+struct mlx5dr_send_ste_attr *ste_attr)
+{
+ struct mlx5dr_send_engine_post_attr *send_attr = &ste_attr->send_attr;
+ struct mlx5dr_rule *rule = send_attr->rule;
+ struct ibv_context *ibv_ctx;
+ struct mlx5dr_context *ctx;
+ uint16_t queue_id;
+ uint32_t pdn;
+ int ret;
+
+ ctx = rule->matcher->tbl->ctx;
+ queue_id = queue - ctx->send_queue;
+ ibv_ctx = ctx->ibv_ctx;
+ pdn = ctx->pd_num;
+
+ /* Writing through FW can't HW fence, therefore we drain the queue */
+ if (send_attr->fence)
+ mlx5dr_send_queue_action(ctx,
+queue_id,
+MLX5DR_SEND_QUEUE_ACTION_DRAIN_SYNC);
+
+ if (ste_attr->rtc_1) {
+ send_attr->id = ste_attr->rtc_1;
+ send_attr->used_id = ste_attr->used_id_rtc_1;
+ send_attr->retry_id = ste_attr->retry_rtc_1;
+ ret = mlx5dr_send_wqe_fw(ibv_ctx, pdn, send_attr,
+ste_attr->wqe_ctrl,
+ste_attr->wqe_data,
+
[v1 02/16] net/mlx5/hws: matcher remove AT and MT limitation
The action and match templates were stored on the matcher in
a fixed size array to reduce cache misses and reuse template
calculations. This approuch introduced two issues:
-limitation of fixed array
-definer is bindind to match template and cannot be used with
union definer since the layout is fixed
Signed-off-by: Alex Vesker
---
drivers/net/mlx5/hws/mlx5dr_debug.c | 4 +-
drivers/net/mlx5/hws/mlx5dr_matcher.c | 109 --
drivers/net/mlx5/hws/mlx5dr_matcher.h | 8 +-
drivers/net/mlx5/hws/mlx5dr_rule.c| 15 ++--
4 files changed, 78 insertions(+), 58 deletions(-)
diff --git a/drivers/net/mlx5/hws/mlx5dr_debug.c
b/drivers/net/mlx5/hws/mlx5dr_debug.c
index 0815327b18..9199ec16e0 100644
--- a/drivers/net/mlx5/hws/mlx5dr_debug.c
+++ b/drivers/net/mlx5/hws/mlx5dr_debug.c
@@ -92,7 +92,7 @@ mlx5dr_debug_dump_matcher_match_template(FILE *f, struct
mlx5dr_matcher *matcher
int i, ret;
for (i = 0; i < matcher->num_of_mt; i++) {
- struct mlx5dr_match_template *mt = matcher->mt[i];
+ struct mlx5dr_match_template *mt = &matcher->mt[i];
ret = fprintf(f, "%d,0x%" PRIx64 ",0x%" PRIx64 ",%d,%d\n",
MLX5DR_DEBUG_RES_TYPE_MATCHER_MATCH_TEMPLATE,
@@ -123,7 +123,7 @@ mlx5dr_debug_dump_matcher_action_template(FILE *f, struct
mlx5dr_matcher *matche
int i, j, ret;
for (i = 0; i < matcher->num_of_at; i++) {
- struct mlx5dr_action_template *at = matcher->at[i];
+ struct mlx5dr_action_template *at = &matcher->at[i];
ret = fprintf(f, "%d,0x%" PRIx64 ",0x%" PRIx64 ",%d,%d,%d",
MLX5DR_DEBUG_RES_TYPE_MATCHER_ACTION_TEMPLATE,
diff --git a/drivers/net/mlx5/hws/mlx5dr_matcher.c
b/drivers/net/mlx5/hws/mlx5dr_matcher.c
index d509a2f0e1..913bb9d447 100644
--- a/drivers/net/mlx5/hws/mlx5dr_matcher.c
+++ b/drivers/net/mlx5/hws/mlx5dr_matcher.c
@@ -430,8 +430,8 @@ static int mlx5dr_matcher_create_rtc(struct mlx5dr_matcher
*matcher,
/* The usual Hash Table */
rtc_attr.update_index_mode =
MLX5_IFC_RTC_STE_UPDATE_MODE_BY_HASH;
/* The first match template is used since all share the
same definer */
- rtc_attr.definer_id =
mlx5dr_definer_get_id(matcher->mt[0]->definer);
- rtc_attr.is_jumbo =
mlx5dr_definer_is_jumbo(matcher->mt[0]->definer);
+ rtc_attr.definer_id =
mlx5dr_definer_get_id(matcher->mt->definer);
+ rtc_attr.is_jumbo =
mlx5dr_definer_is_jumbo(matcher->mt->definer);
} else if (attr->insert_mode == MLX5DR_MATCHER_INSERT_BY_INDEX)
{
rtc_attr.update_index_mode =
MLX5_IFC_RTC_STE_UPDATE_MODE_BY_OFFSET;
rtc_attr.num_hash_definer = 1;
@@ -439,10 +439,8 @@ static int mlx5dr_matcher_create_rtc(struct mlx5dr_matcher
*matcher,
if (attr->distribute_mode ==
MLX5DR_MATCHER_DISTRIBUTE_BY_HASH) {
/* Hash Split Table */
rtc_attr.access_index_mode =
MLX5_IFC_RTC_STE_ACCESS_MODE_BY_HASH;
- rtc_attr.definer_id =
-
mlx5dr_definer_get_id(matcher->mt[0]->definer);
- rtc_attr.is_jumbo =
-
mlx5dr_definer_is_jumbo(matcher->mt[0]->definer);
+ rtc_attr.definer_id =
mlx5dr_definer_get_id(matcher->mt->definer);
+ rtc_attr.is_jumbo =
mlx5dr_definer_is_jumbo(matcher->mt->definer);
} else if (attr->distribute_mode ==
MLX5DR_MATCHER_DISTRIBUTE_BY_LINEAR) {
/* Linear Lookup Table */
rtc_attr.access_index_mode =
MLX5_IFC_RTC_STE_ACCESS_MODE_LINEAR;
@@ -579,7 +577,7 @@ static void mlx5dr_matcher_set_pool_attr(struct
mlx5dr_pool_attr *attr,
static int mlx5dr_matcher_bind_at(struct mlx5dr_matcher *matcher)
{
- bool is_jumbo = mlx5dr_definer_is_jumbo(matcher->mt[0]->definer);
+ bool is_jumbo = mlx5dr_definer_is_jumbo(matcher->mt->definer);
struct mlx5dr_cmd_stc_modify_attr stc_attr = {0};
struct mlx5dr_table *tbl = matcher->tbl;
struct mlx5dr_pool_attr pool_attr = {0};
@@ -589,7 +587,7 @@ static int mlx5dr_matcher_bind_at(struct mlx5dr_matcher
*matcher)
bool valid;
for (i = 0; i < matcher->num_of_at; i++) {
- struct mlx5dr_action_template *at = matcher->at[i];
+ struct mlx5dr_action_template *at = &matcher->at[i];
/* Check if action combinabtion is valid */
[v1 07/16] net/mlx5/hws: add send FW range STE WQE
FW WQE supports complex rules, constructed from 2 STEs,
for example:
Hash(DefinerA)
SteMatch(DefinerB)
SteRange(DefinerC)
DefinerA is a subset of DefinerB
This complex rule is written using a single FW command which
has a single WQE control, STE match data0 and STE range data1.
FW manages STEs/ICM and coherency between deletion and creation.
It is possible to also pass the definer value as part of the
STE, this is not supported with current HW.
Signed-off-by: Alex Vesker
---
drivers/net/mlx5/hws/mlx5dr_send.c | 19 +++
drivers/net/mlx5/hws/mlx5dr_send.h | 3 +++
2 files changed, 22 insertions(+)
diff --git a/drivers/net/mlx5/hws/mlx5dr_send.c
b/drivers/net/mlx5/hws/mlx5dr_send.c
index a9958df4f2..51aaf5c8e2 100644
--- a/drivers/net/mlx5/hws/mlx5dr_send.c
+++ b/drivers/net/mlx5/hws/mlx5dr_send.c
@@ -242,11 +242,15 @@ int mlx5dr_send_wqe_fw(struct ibv_context *ibv_ctx,
struct mlx5dr_wqe_gta_ctrl_seg *send_wqe_ctrl,
void *send_wqe_match_data,
void *send_wqe_match_tag,
+ void *send_wqe_range_data,
+ void *send_wqe_range_tag,
bool is_jumbo,
uint8_t gta_opcode)
{
+ bool has_range = send_wqe_range_data || send_wqe_range_tag;
bool has_match = send_wqe_match_data || send_wqe_match_tag;
struct mlx5dr_wqe_gta_data_seg_ste gta_wqe_data0 = {0};
+ struct mlx5dr_wqe_gta_data_seg_ste gta_wqe_data1 = {0};
struct mlx5dr_wqe_gta_ctrl_seg gta_wqe_ctrl = {0};
struct mlx5dr_cmd_generate_wqe_attr attr = {0};
struct mlx5dr_wqe_ctrl_seg wqe_ctrl = {0};
@@ -278,6 +282,17 @@ int mlx5dr_send_wqe_fw(struct ibv_context *ibv_ctx,
attr.gta_data_0 = (uint8_t *)>a_wqe_data0;
}
+ /* Set GTA range WQE DATA */
+ if (has_range) {
+ if (send_wqe_range_data)
+ memcpy(>a_wqe_data1, send_wqe_range_data,
sizeof(gta_wqe_data1));
+ else
+ mlx5dr_send_wqe_set_tag(>a_wqe_data1,
send_wqe_range_tag, false);
+
+ gta_wqe_data1.rsvd1_definer =
htobe32(send_attr->range_definer_id << 8);
+ attr.gta_data_1 = (uint8_t *)>a_wqe_data1;
+ }
+
attr.pdn = pd_num;
attr.wqe_ctrl = (uint8_t *)&wqe_ctrl;
attr.gta_ctrl = (uint8_t *)>a_wqe_ctrl;
@@ -336,6 +351,8 @@ void mlx5dr_send_stes_fw(struct mlx5dr_send_engine *queue,
ste_attr->wqe_ctrl,
ste_attr->wqe_data,
ste_attr->wqe_tag,
+ste_attr->range_wqe_data,
+ste_attr->range_wqe_tag,
ste_attr->wqe_tag_is_jumbo,
ste_attr->gta_opcode);
if (ret)
@@ -350,6 +367,8 @@ void mlx5dr_send_stes_fw(struct mlx5dr_send_engine *queue,
ste_attr->wqe_ctrl,
ste_attr->wqe_data,
ste_attr->wqe_tag,
+ste_attr->range_wqe_data,
+ste_attr->range_wqe_tag,
ste_attr->wqe_tag_is_jumbo,
ste_attr->gta_opcode);
if (ret)
diff --git a/drivers/net/mlx5/hws/mlx5dr_send.h
b/drivers/net/mlx5/hws/mlx5dr_send.h
index 1e845b1c7a..47bb66b3c7 100644
--- a/drivers/net/mlx5/hws/mlx5dr_send.h
+++ b/drivers/net/mlx5/hws/mlx5dr_send.h
@@ -161,6 +161,7 @@ struct mlx5dr_send_engine_post_attr {
uint8_t notify_hw;
uint8_t fence;
uint8_t match_definer_id;
+ uint8_t range_definer_id;
size_t len;
struct mlx5dr_rule *rule;
uint32_t id;
@@ -182,8 +183,10 @@ struct mlx5dr_send_ste_attr {
uint32_t direct_index;
struct mlx5dr_send_engine_post_attr send_attr;
struct mlx5dr_rule_match_tag *wqe_tag;
+ struct mlx5dr_rule_match_tag *range_wqe_tag;
struct mlx5dr_wqe_gta_ctrl_seg *wqe_ctrl;
struct mlx5dr_wqe_gta_data_seg_ste *wqe_data;
+ struct mlx5dr_wqe_gta_data_seg_ste *range_wqe_data;
};
/**
--
2.18.1
[v1 03/16] net/mlx5/hws: support GTA WQE write using FW command
The generate WQE command is used as an interface to writing GTA
WQEs with fields that are not supported in current HW, for example
extended match definer.
Signed-off-by: Alex Vesker
---
drivers/common/mlx5/mlx5_prm.h| 27 +-
drivers/net/mlx5/hws/mlx5dr_cmd.c | 47 +++
drivers/net/mlx5/hws/mlx5dr_cmd.h | 13 +
3 files changed, 86 insertions(+), 1 deletion(-)
diff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h
index 9294f65e24..d4d8ddcd2a 100644
--- a/drivers/common/mlx5/mlx5_prm.h
+++ b/drivers/common/mlx5/mlx5_prm.h
@@ -1141,6 +1141,7 @@ enum {
MLX5_CMD_QUERY_REGEX_REGISTERS = 0xb07,
MLX5_CMD_OP_ACCESS_REGISTER_USER = 0xb0c,
MLX5_CMD_OP_ALLOW_OTHER_VHCA_ACCESS = 0xb16,
+ MLX5_CMD_OP_GENERATE_WQE = 0xb17,
};
enum {
@@ -2159,7 +2160,8 @@ struct mlx5_ifc_cmd_hca_cap_2_bits {
u8 format_select_dw_gtpu_dw_1[0x8];
u8 format_select_dw_gtpu_dw_2[0x8];
u8 format_select_dw_gtpu_first_ext_dw_0[0x8];
- u8 reserved_at_2a0[0x560];
+ u8 generate_wqe_type[0x20];
+ u8 reserved_at_2c0[0x540];
};
struct mlx5_ifc_esw_cap_bits {
@@ -3529,6 +3531,29 @@ struct mlx5_ifc_create_alias_obj_in_bits {
struct mlx5_ifc_alias_context_bits alias_ctx;
};
+struct mlx5_ifc_generate_wqe_in_bits {
+ u8 opcode[0x10];
+ u8 uid[0x10];
+ u8 reserved_at_20[0x10];
+ u8 op_mode[0x10];
+ u8 reserved_at_40[0x40];
+ u8 reserved_at_80[0x8];
+ u8 pdn[0x18];
+ u8 reserved_at_a0[0x160];
+ u8 wqe_ctrl[0x80];
+ u8 wqe_gta_ctrl[0x180];
+ u8 wqe_gta_data_0[0x200];
+ u8 wqe_gta_data_1[0x200];
+};
+
+struct mlx5_ifc_generate_wqe_out_bits {
+ u8 status[0x8];
+ u8 reserved_at_8[0x18];
+ u8 syndrome[0x20];
+ u8 reserved_at_40[0x1c0];
+ u8 cqe_data[0x200];
+};
+
enum {
MLX5_CRYPTO_KEY_SIZE_128b = 0x0,
MLX5_CRYPTO_KEY_SIZE_256b = 0x1,
diff --git a/drivers/net/mlx5/hws/mlx5dr_cmd.c
b/drivers/net/mlx5/hws/mlx5dr_cmd.c
index 32378673cf..c648eacd03 100644
--- a/drivers/net/mlx5/hws/mlx5dr_cmd.c
+++ b/drivers/net/mlx5/hws/mlx5dr_cmd.c
@@ -795,6 +795,53 @@ mlx5dr_cmd_alias_obj_create(struct ibv_context *ctx,
return devx_obj;
}
+int mlx5dr_cmd_generate_wqe(struct ibv_context *ctx,
+ struct mlx5dr_cmd_generate_wqe_attr *attr,
+ struct mlx5_cqe64 *ret_cqe)
+{
+ uint32_t out[MLX5_ST_SZ_DW(generate_wqe_out)] = {0};
+ uint32_t in[MLX5_ST_SZ_DW(generate_wqe_in)] = {0};
+ uint8_t status;
+ void *ptr;
+ int ret;
+
+ MLX5_SET(generate_wqe_in, in, opcode, MLX5_CMD_OP_GENERATE_WQE);
+ MLX5_SET(generate_wqe_in, in, pdn, attr->pdn);
+
+ ptr = MLX5_ADDR_OF(generate_wqe_in, in, wqe_ctrl);
+ memcpy(ptr, attr->wqe_ctrl, MLX5_FLD_SZ_BYTES(generate_wqe_in,
wqe_ctrl));
+
+ ptr = MLX5_ADDR_OF(generate_wqe_in, in, wqe_gta_ctrl);
+ memcpy(ptr, attr->gta_ctrl, MLX5_FLD_SZ_BYTES(generate_wqe_in,
wqe_gta_ctrl));
+
+ ptr = MLX5_ADDR_OF(generate_wqe_in, in, wqe_gta_data_0);
+ memcpy(ptr, attr->gta_data_0, MLX5_FLD_SZ_BYTES(generate_wqe_in,
wqe_gta_data_0));
+
+ if (attr->gta_data_1) {
+ ptr = MLX5_ADDR_OF(generate_wqe_in, in, wqe_gta_data_1);
+ memcpy(ptr, attr->gta_data_1,
MLX5_FLD_SZ_BYTES(generate_wqe_in, wqe_gta_data_1));
+ }
+
+ ret = mlx5_glue->devx_general_cmd(ctx, in, sizeof(in), out,
sizeof(out));
+ if (ret) {
+ DR_LOG(ERR, "Failed to write GTA WQE using FW");
+ rte_errno = errno;
+ return rte_errno;
+ }
+
+ status = MLX5_GET(generate_wqe_out, out, status);
+ if (status) {
+ DR_LOG(ERR, "Invalid FW CQE status %d", status);
+ rte_errno = EINVAL;
+ return rte_errno;
+ }
+
+ ptr = MLX5_ADDR_OF(generate_wqe_out, out, cqe_data);
+ memcpy(ret_cqe, ptr, sizeof(*ret_cqe));
+
+ return 0;
+}
+
int mlx5dr_cmd_query_caps(struct ibv_context *ctx,
struct mlx5dr_cmd_query_caps *caps)
{
diff --git a/drivers/net/mlx5/hws/mlx5dr_cmd.h
b/drivers/net/mlx5/hws/mlx5dr_cmd.h
index 468557ba16..3689d09897 100644
--- a/drivers/net/mlx5/hws/mlx5dr_cmd.h
+++ b/drivers/net/mlx5/hws/mlx5dr_cmd.h
@@ -153,6 +153,14 @@ struct mlx5dr_cmd_query_vport_caps {
uint32_t metadata_c_mask;
};
+struct mlx5dr_cmd_generate_wqe_attr {
+ uint8_t *wqe_ctrl;
+ uint8_t *gta_ctrl;
+ uint8_t *gta_data_0;
+ uint8_t *gta_data_1;
+ uint32_t pdn;
+};
+
struct mlx5dr_cmd_query_caps {
uint32_t wire_regc;
uint32_t wire_regc_mask;
@@ -212,6 +220,11 @@ int
mlx5dr_cmd_stc_modify(struct mlx5dr_devx_obj *devx_obj,
struct mlx5dr_cmd_stc_modify_attr *stc_attr);
+int
+mlx5dr
[v1 10/16] net/mlx5/hws: redesign definer create
Until now definer creation and deletion get and put
functions were used, the get function would calculate the
definer field copy (fc), header layout (hl) and definer
layout internally without taking into account other match
templates used over the same matcher.
This logic had to be split to allow sharing the hl over
multiple definers. First calculate the shared hl than
create definers based on the definer shared layout.
Once all definers use the same layout it is possible to
hash over the shared fields since the location is the same
across all of the definers.
Signed-off-by: Alex Vesker
---
drivers/net/mlx5/hws/mlx5dr_definer.c | 301 -
drivers/net/mlx5/hws/mlx5dr_definer.h | 11 +-
drivers/net/mlx5/hws/mlx5dr_internal.h | 2 +-
drivers/net/mlx5/hws/mlx5dr_matcher.c | 61 ++---
drivers/net/mlx5/hws/mlx5dr_matcher.h | 16 +-
5 files changed, 230 insertions(+), 161 deletions(-)
diff --git a/drivers/net/mlx5/hws/mlx5dr_definer.c
b/drivers/net/mlx5/hws/mlx5dr_definer.c
index c268f94ad3..9560f8a0af 100644
--- a/drivers/net/mlx5/hws/mlx5dr_definer.c
+++ b/drivers/net/mlx5/hws/mlx5dr_definer.c
@@ -104,7 +104,6 @@ struct mlx5dr_definer_conv_data {
struct mlx5dr_definer_fc *fc;
uint8_t relaxed;
uint8_t tunnel;
- uint8_t *hl;
};
/* Xmacro used to create generic item setter from items */
@@ -1504,6 +1503,36 @@ mlx5dr_definer_conv_item_meter_color(struct
mlx5dr_definer_conv_data *cd,
return 0;
}
+static int
+mlx5dr_definer_mt_set_fc(struct mlx5dr_match_template *mt,
+struct mlx5dr_definer_fc *fc,
+uint8_t *hl)
+{
+ uint32_t fc_sz = 0;
+ int i;
+
+ for (i = 0; i < MLX5DR_DEFINER_FNAME_MAX; i++)
+ if (fc[i].tag_set)
+ fc_sz++;
+
+ mt->fc = simple_calloc(fc_sz, sizeof(*mt->fc));
+ if (!mt->fc) {
+ rte_errno = ENOMEM;
+ return rte_errno;
+ }
+
+ for (i = 0; i < MLX5DR_DEFINER_FNAME_MAX; i++) {
+ if (!fc[i].tag_set)
+ continue;
+
+ fc[i].fname = i;
+ memcpy(&mt->fc[mt->fc_sz++], &fc[i], sizeof(*mt->fc));
+ DR_SET(hl, -1, fc[i].byte_off, fc[i].bit_off, fc[i].bit_mask);
+ }
+
+ return 0;
+}
+
static int
mlx5dr_definer_check_item_range_supp(struct rte_flow_item *item)
{
@@ -1535,12 +1564,9 @@ mlx5dr_definer_conv_items_to_hl(struct mlx5dr_context
*ctx,
struct mlx5dr_definer_conv_data cd = {0};
struct rte_flow_item *items = mt->items;
uint64_t item_flags = 0;
- uint32_t total = 0;
- int i, j;
- int ret;
+ int i, ret;
cd.fc = fc;
- cd.hl = hl;
cd.caps = ctx->caps;
cd.relaxed = mt->flags & MLX5DR_MATCH_TEMPLATE_FLAG_RELAXED_MATCH;
@@ -1660,29 +1686,11 @@ mlx5dr_definer_conv_items_to_hl(struct mlx5dr_context
*ctx,
mt->item_flags = item_flags;
- /* Fill in headers layout and calculate total number of fields */
- for (i = 0; i < MLX5DR_DEFINER_FNAME_MAX; i++) {
- if (fc[i].tag_set) {
- total++;
- DR_SET(hl, -1, fc[i].byte_off, fc[i].bit_off,
fc[i].bit_mask);
- }
- }
-
- mt->fc_sz = total;
- mt->fc = simple_calloc(total, sizeof(*mt->fc));
- if (!mt->fc) {
- DR_LOG(ERR, "Failed to allocate field copy array");
- rte_errno = ENOMEM;
- return rte_errno;
- }
-
- j = 0;
- for (i = 0; i < MLX5DR_DEFINER_FNAME_MAX; i++) {
- if (fc[i].tag_set) {
- memcpy(&mt->fc[j], &fc[i], sizeof(*mt->fc));
- mt->fc[j].fname = i;
- j++;
- }
+ /* Fill in headers layout and allocate fc array on mt */
+ ret = mlx5dr_definer_mt_set_fc(mt, fc, hl);
+ if (ret) {
+ DR_LOG(ERR, "Failed to set field copy to match template");
+ return ret;
}
return 0;
@@ -1837,8 +1845,8 @@ mlx5dr_definer_best_hl_fit_recu(struct
mlx5dr_definer_sel_ctrl *ctrl,
}
static void
-mlx5dr_definer_apply_sel_ctrl(struct mlx5dr_definer_sel_ctrl *ctrl,
- struct mlx5dr_definer *definer)
+mlx5dr_definer_copy_sel_ctrl(struct mlx5dr_definer_sel_ctrl *ctrl,
+struct mlx5dr_definer *definer)
{
memcpy(definer->byte_selector, ctrl->byte_selector,
ctrl->allowed_bytes);
memcpy(definer->dw_selector, ctrl->full_dw_selector,
ctrl->allowed_full_dw);
@@ -1848,7 +1856,7 @@ mlx5dr_definer_apply_sel_ctrl(struct
mlx5dr_definer_sel_ctrl *ctrl,
static int
mlx5dr_definer_find_best_hl_fit(struct mlx5dr_context *ctx,
- struct mlx5dr_match_
[v1 08/16] net/mlx5/hws: move matcher size check to function
This check can be later on reused for other places, it
will look better in a function
Signed-off-by: Alex Vesker
---
drivers/net/mlx5/hws/mlx5dr_matcher.c | 43 +--
1 file changed, 27 insertions(+), 16 deletions(-)
diff --git a/drivers/net/mlx5/hws/mlx5dr_matcher.c
b/drivers/net/mlx5/hws/mlx5dr_matcher.c
index 101a12d361..b8db0a27ae 100644
--- a/drivers/net/mlx5/hws/mlx5dr_matcher.c
+++ b/drivers/net/mlx5/hws/mlx5dr_matcher.c
@@ -565,6 +565,32 @@ static void mlx5dr_matcher_destroy_rtc(struct
mlx5dr_matcher *matcher,
mlx5dr_pool_chunk_free(ste_pool, ste);
}
+static int
+mlx5dr_matcher_check_attr_sz(struct mlx5dr_cmd_query_caps *caps,
+struct mlx5dr_matcher_attr *attr)
+{
+ if (attr->table.sz_col_log > caps->rtc_log_depth_max) {
+ DR_LOG(ERR, "Matcher depth exceeds limit %d",
caps->rtc_log_depth_max);
+ goto not_supported;
+ }
+
+ if (attr->table.sz_col_log + attr->table.sz_row_log >
caps->ste_alloc_log_max) {
+ DR_LOG(ERR, "Total matcher size exceeds limit %d",
caps->ste_alloc_log_max);
+ goto not_supported;
+ }
+
+ if (attr->table.sz_col_log + attr->table.sz_row_log <
caps->ste_alloc_log_gran) {
+ DR_LOG(ERR, "Total matcher size below limit %d",
caps->ste_alloc_log_gran);
+ goto not_supported;
+ }
+
+ return 0;
+
+not_supported:
+ rte_errno = EOPNOTSUPP;
+ return rte_errno;
+}
+
static void mlx5dr_matcher_set_pool_attr(struct mlx5dr_pool_attr *attr,
struct mlx5dr_matcher *matcher)
{
@@ -840,22 +866,7 @@ mlx5dr_matcher_process_attr(struct mlx5dr_cmd_query_caps
*caps,
attr->insert_mode == MLX5DR_MATCHER_INSERT_BY_HASH)
attr->table.sz_col_log =
mlx5dr_matcher_rules_to_tbl_depth(attr->rule.num_log);
- if (attr->table.sz_col_log > caps->rtc_log_depth_max) {
- DR_LOG(ERR, "Matcher depth exceeds limit %d",
caps->rtc_log_depth_max);
- goto not_supported;
- }
-
- if (attr->table.sz_col_log + attr->table.sz_row_log >
caps->ste_alloc_log_max) {
- DR_LOG(ERR, "Total matcher size exceeds limit %d",
caps->ste_alloc_log_max);
- goto not_supported;
- }
-
- if (attr->table.sz_col_log + attr->table.sz_row_log <
caps->ste_alloc_log_gran) {
- DR_LOG(ERR, "Total matcher size below limit %d",
caps->ste_alloc_log_gran);
- goto not_supported;
- }
-
- return 0;
+ return mlx5dr_matcher_check_attr_sz(caps, attr);
not_supported:
rte_errno = EOPNOTSUPP;
--
2.18.1
[v1 14/16] net/mlx5/hws: add debug dump support for range and hash
Add support for dumping range and hash definers objects.
Hash definer is a per matcher object describing the fields
used for hashing. Range definer is per match template object
describing the fields used for range matching.
Both are optional based on the given match templates.
Signed-off-by: Alex Vesker
---
drivers/net/mlx5/hws/mlx5dr_debug.c | 37 -
drivers/net/mlx5/hws/mlx5dr_debug.h | 4 +++-
2 files changed, 29 insertions(+), 12 deletions(-)
diff --git a/drivers/net/mlx5/hws/mlx5dr_debug.c
b/drivers/net/mlx5/hws/mlx5dr_debug.c
index 9199ec16e0..b1d271eebe 100644
--- a/drivers/net/mlx5/hws/mlx5dr_debug.c
+++ b/drivers/net/mlx5/hws/mlx5dr_debug.c
@@ -34,15 +34,19 @@ const char *mlx5dr_debug_action_type_to_str(enum
mlx5dr_action_type action_type)
static int
mlx5dr_debug_dump_matcher_template_definer(FILE *f,
- struct mlx5dr_match_template *mt)
+ void *parent_obj,
+ struct mlx5dr_definer *definer,
+ enum mlx5dr_debug_res_type type)
{
- struct mlx5dr_definer *definer = mt->definer;
int i, ret;
+ if (!definer)
+ return 0;
+
ret = fprintf(f, "%d,0x%" PRIx64 ",0x%" PRIx64 ",%d,%d,",
- MLX5DR_DEBUG_RES_TYPE_MATCHER_TEMPLATE_DEFINER,
+ type,
(uint64_t)(uintptr_t)definer,
- (uint64_t)(uintptr_t)mt,
+ (uint64_t)(uintptr_t)parent_obj,
definer->obj->id,
definer->type);
if (ret < 0) {
@@ -89,29 +93,40 @@ static int
mlx5dr_debug_dump_matcher_match_template(FILE *f, struct mlx5dr_matcher
*matcher)
{
bool is_root = matcher->tbl->level == MLX5DR_ROOT_LEVEL;
+ enum mlx5dr_debug_res_type res_type;
int i, ret;
for (i = 0; i < matcher->num_of_mt; i++) {
struct mlx5dr_match_template *mt = &matcher->mt[i];
- ret = fprintf(f, "%d,0x%" PRIx64 ",0x%" PRIx64 ",%d,%d\n",
+ ret = fprintf(f, "%d,0x%" PRIx64 ",0x%" PRIx64 ",%d,%d,%d\n",
MLX5DR_DEBUG_RES_TYPE_MATCHER_MATCH_TEMPLATE,
(uint64_t)(uintptr_t)mt,
(uint64_t)(uintptr_t)matcher,
is_root ? 0 : mt->fc_sz,
- mt->flags);
+ mt->flags,
+ is_root ? 0 : mt->fcr_sz);
if (ret < 0) {
rte_errno = EINVAL;
return rte_errno;
}
- if (!is_root) {
- ret = mlx5dr_debug_dump_matcher_template_definer(f, mt);
- if (ret)
- return ret;
- }
+ res_type = MLX5DR_DEBUG_RES_TYPE_MATCHER_TEMPLATE_MATCH_DEFINER;
+ ret = mlx5dr_debug_dump_matcher_template_definer(f, mt,
mt->definer, res_type);
+ if (ret)
+ return ret;
+
+ res_type = MLX5DR_DEBUG_RES_TYPE_MATCHER_TEMPLATE_RANGE_DEFINER;
+ ret = mlx5dr_debug_dump_matcher_template_definer(f, mt,
mt->range_definer, res_type);
+ if (ret)
+ return ret;
}
+ res_type = MLX5DR_DEBUG_RES_TYPE_MATCHER_TEMPLATE_HASH_DEFINER;
+ ret = mlx5dr_debug_dump_matcher_template_definer(f, matcher,
matcher->hash_definer, res_type);
+ if (ret)
+ return ret;
+
return 0;
}
diff --git a/drivers/net/mlx5/hws/mlx5dr_debug.h
b/drivers/net/mlx5/hws/mlx5dr_debug.h
index cf00170f7d..2c29ca295c 100644
--- a/drivers/net/mlx5/hws/mlx5dr_debug.h
+++ b/drivers/net/mlx5/hws/mlx5dr_debug.h
@@ -19,8 +19,10 @@ enum mlx5dr_debug_res_type {
MLX5DR_DEBUG_RES_TYPE_MATCHER = 4200,
MLX5DR_DEBUG_RES_TYPE_MATCHER_ATTR = 4201,
MLX5DR_DEBUG_RES_TYPE_MATCHER_MATCH_TEMPLATE = 4202,
+ MLX5DR_DEBUG_RES_TYPE_MATCHER_TEMPLATE_MATCH_DEFINER = 4203,
MLX5DR_DEBUG_RES_TYPE_MATCHER_ACTION_TEMPLATE = 4204,
- MLX5DR_DEBUG_RES_TYPE_MATCHER_TEMPLATE_DEFINER = 4203,
+ MLX5DR_DEBUG_RES_TYPE_MATCHER_TEMPLATE_HASH_DEFINER = 4205,
+ MLX5DR_DEBUG_RES_TYPE_MATCHER_TEMPLATE_RANGE_DEFINER = 4206,
};
const char *mlx5dr_debug_action_type_to_str(enum mlx5dr_action_type
action_type);
--
2.18.1
[v1 11/16] net/mlx5/hws: support partial hash
Hash definers allow performing hashing over a subset of
the fields which are used for matching. This allows combining
match templates which were considered invalid until now.
During matcher creation mlx5dr code will process the match
templates and check if such hash definer is needed based
on the definers bitmasks intersection.
Since current HW GTA implementation doesn't allow specifying
match and hash definers rule insertion is done using the FW
GTA WQE command.
Signed-off-by: Alex Vesker
---
drivers/common/mlx5/mlx5_prm.h| 4 +
drivers/net/mlx5/hws/mlx5dr_definer.c | 105 ++
drivers/net/mlx5/hws/mlx5dr_matcher.c | 66 +++-
drivers/net/mlx5/hws/mlx5dr_matcher.h | 10 ++-
4 files changed, 181 insertions(+), 4 deletions(-)
diff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h
index cf46296afb..cca2fb6af7 100644
--- a/drivers/common/mlx5/mlx5_prm.h
+++ b/drivers/common/mlx5/mlx5_prm.h
@@ -2112,6 +2112,10 @@ enum mlx5_ifc_cross_vhca_allowed_objects_types {
MLX5_CROSS_VHCA_ALLOWED_OBJS_RTC = 1 << 0xa,
};
+enum {
+ MLX5_GENERATE_WQE_TYPE_FLOW_UPDATE = 1 << 1,
+};
+
/*
* HCA Capabilities 2
*/
diff --git a/drivers/net/mlx5/hws/mlx5dr_definer.c
b/drivers/net/mlx5/hws/mlx5dr_definer.c
index 9560f8a0af..260e6c5d1d 100644
--- a/drivers/net/mlx5/hws/mlx5dr_definer.c
+++ b/drivers/net/mlx5/hws/mlx5dr_definer.c
@@ -1928,6 +1928,27 @@ int mlx5dr_definer_get_id(struct mlx5dr_definer *definer)
return definer->obj->id;
}
+static int
+mlx5dr_definer_compare(struct mlx5dr_definer *definer_a,
+ struct mlx5dr_definer *definer_b)
+{
+ int i;
+
+ for (i = 0; i < BYTE_SELECTORS; i++)
+ if (definer_a->byte_selector[i] != definer_b->byte_selector[i])
+ return 1;
+
+ for (i = 0; i < DW_SELECTORS; i++)
+ if (definer_a->dw_selector[i] != definer_b->dw_selector[i])
+ return 1;
+
+ for (i = 0; i < MLX5DR_JUMBO_TAG_SZ; i++)
+ if (definer_a->mask.jumbo[i] != definer_b->mask.jumbo[i])
+ return 1;
+
+ return 0;
+}
+
static int
mlx5dr_definer_calc_layout(struct mlx5dr_matcher *matcher,
struct mlx5dr_definer *match_definer)
@@ -2070,6 +2091,80 @@ mlx5dr_definer_matcher_match_uninit(struct
mlx5dr_matcher *matcher)
mlx5dr_definer_free(matcher->mt[i].definer);
}
+static int
+mlx5dr_definer_matcher_hash_init(struct mlx5dr_context *ctx,
+struct mlx5dr_matcher *matcher)
+{
+ struct mlx5dr_cmd_definer_create_attr def_attr = {0};
+ struct mlx5dr_match_template *mt = matcher->mt;
+ struct ibv_context *ibv_ctx = ctx->ibv_ctx;
+ uint8_t *bit_mask;
+ int i, j;
+
+ for (i = 1; i < matcher->num_of_mt; i++)
+ if (mlx5dr_definer_compare(mt[i].definer, mt[i - 1].definer))
+ matcher->flags |= MLX5DR_MATCHER_FLAGS_HASH_DEFINER;
+
+ if (!(matcher->flags & MLX5DR_MATCHER_FLAGS_HASH_DEFINER))
+ return 0;
+
+ /* Insert by index requires all MT using the same definer */
+ if (matcher->attr.insert_mode == MLX5DR_MATCHER_INSERT_BY_INDEX) {
+ DR_LOG(ERR, "Insert by index not supported with MT
combination");
+ rte_errno = EOPNOTSUPP;
+ return rte_errno;
+ }
+
+ matcher->hash_definer = simple_calloc(1,
sizeof(*matcher->hash_definer));
+ if (!matcher->hash_definer) {
+ DR_LOG(ERR, "Failed to allocate memory for hash definer");
+ rte_errno = ENOMEM;
+ return rte_errno;
+ }
+
+ /* Calculate intersection between all match templates bitmasks.
+* We will use mt[0] as reference and intersect it with mt[1..n].
+* From this we will get:
+* hash_definer.selectors = mt[0].selecotrs
+* hash_definer.mask = mt[0].mask & mt[0].mask & ... & mt[n].mask
+*/
+
+ /* Use first definer which should also contain intersection fields */
+ memcpy(matcher->hash_definer, mt->definer, sizeof(struct
mlx5dr_definer));
+
+ /* Calculate intersection between first to all match templates bitmasks
*/
+ for (i = 1; i < matcher->num_of_mt; i++) {
+ bit_mask = (uint8_t *)&mt[i].definer->mask;
+ for (j = 0; j < MLX5DR_JUMBO_TAG_SZ; j++)
+ ((uint8_t *)&matcher->hash_definer->mask)[j] &=
bit_mask[j];
+ }
+
+ def_attr.match_mask = matcher->hash_definer->mask.jumbo;
+ def_attr.dw_selector = matcher->hash_definer->dw_selector;
+ def_attr.byte_selector = matcher->hash_definer->byte_selector;
+ matcher->hash_definer->obj = mlx5dr_cmd_d
[v1 16/16] net/mlx5/hws: cache definer for reuse
Definers are a limited resource in the system per GVMI, to
avoid failure we try to improve bt checking if it is possible
to reuse the definers in some cases. Added a cache on the context
for this purpose.
Signed-off-by: Alex Vesker
---
drivers/net/mlx5/hws/mlx5dr_context.c | 12 ++-
drivers/net/mlx5/hws/mlx5dr_context.h | 1 +
drivers/net/mlx5/hws/mlx5dr_definer.c | 122 ++
drivers/net/mlx5/hws/mlx5dr_definer.h | 14 +++
4 files changed, 130 insertions(+), 19 deletions(-)
diff --git a/drivers/net/mlx5/hws/mlx5dr_context.c
b/drivers/net/mlx5/hws/mlx5dr_context.c
index 6627337d9e..08a5ee92a5 100644
--- a/drivers/net/mlx5/hws/mlx5dr_context.c
+++ b/drivers/net/mlx5/hws/mlx5dr_context.c
@@ -13,6 +13,9 @@ static int mlx5dr_context_pools_init(struct mlx5dr_context
*ctx)
if (mlx5dr_pat_init_pattern_cache(&ctx->pattern_cache))
return rte_errno;
+ if (mlx5dr_definer_init_cache(&ctx->definer_cache))
+ goto uninit_pat_cache;
+
/* Create an STC pool per FT type */
pool_attr.pool_type = MLX5DR_POOL_TYPE_STC;
pool_attr.flags = MLX5DR_POOL_FLAGS_FOR_STC_POOL;
@@ -35,8 +38,10 @@ static int mlx5dr_context_pools_init(struct mlx5dr_context
*ctx)
if (ctx->stc_pool[i])
mlx5dr_pool_destroy(ctx->stc_pool[i]);
- mlx5dr_pat_uninit_pattern_cache(ctx->pattern_cache);
+ mlx5dr_definer_uninit_cache(ctx->definer_cache);
+uninit_pat_cache:
+ mlx5dr_pat_uninit_pattern_cache(ctx->pattern_cache);
return rte_errno;
}
@@ -44,12 +49,13 @@ static void mlx5dr_context_pools_uninit(struct
mlx5dr_context *ctx)
{
int i;
- mlx5dr_pat_uninit_pattern_cache(ctx->pattern_cache);
-
for (i = 0; i < MLX5DR_TABLE_TYPE_MAX; i++) {
if (ctx->stc_pool[i])
mlx5dr_pool_destroy(ctx->stc_pool[i]);
}
+
+ mlx5dr_definer_uninit_cache(ctx->definer_cache);
+ mlx5dr_pat_uninit_pattern_cache(ctx->pattern_cache);
}
static int mlx5dr_context_init_pd(struct mlx5dr_context *ctx,
diff --git a/drivers/net/mlx5/hws/mlx5dr_context.h
b/drivers/net/mlx5/hws/mlx5dr_context.h
index a38d9484b3..0ba8d0c92e 100644
--- a/drivers/net/mlx5/hws/mlx5dr_context.h
+++ b/drivers/net/mlx5/hws/mlx5dr_context.h
@@ -39,6 +39,7 @@ struct mlx5dr_context {
struct mlx5dr_context_common_res common_res[MLX5DR_TABLE_TYPE_MAX];
struct mlx5dr_context_shared_gvmi_res gvmi_res[MLX5DR_TABLE_TYPE_MAX];
struct mlx5dr_pattern_cache *pattern_cache;
+ struct mlx5dr_definer_cache *definer_cache;
pthread_spinlock_t ctrl_lock;
enum mlx5dr_context_flags flags;
struct mlx5dr_send_engine *send_queue;
diff --git a/drivers/net/mlx5/hws/mlx5dr_definer.c
b/drivers/net/mlx5/hws/mlx5dr_definer.c
index cf84fbea71..b91f98ee8f 100644
--- a/drivers/net/mlx5/hws/mlx5dr_definer.c
+++ b/drivers/net/mlx5/hws/mlx5dr_definer.c
@@ -2061,6 +2061,7 @@ mlx5dr_definer_compare(struct mlx5dr_definer *definer_a,
{
int i;
+ /* Future: Optimize by comparing selectors with valid mask only */
for (i = 0; i < BYTE_SELECTORS; i++)
if (definer_a->byte_selector[i] != definer_b->byte_selector[i])
return 1;
@@ -2133,15 +2134,106 @@ mlx5dr_definer_calc_layout(struct mlx5dr_matcher
*matcher,
return rte_errno;
}
+int mlx5dr_definer_init_cache(struct mlx5dr_definer_cache **cache)
+{
+ struct mlx5dr_definer_cache *new_cache;
+
+ new_cache = simple_calloc(1, sizeof(*new_cache));
+ if (!new_cache) {
+ rte_errno = ENOMEM;
+ return rte_errno;
+ }
+ LIST_INIT(&new_cache->head);
+ *cache = new_cache;
+
+ return 0;
+}
+
+void mlx5dr_definer_uninit_cache(struct mlx5dr_definer_cache *cache)
+{
+ simple_free(cache);
+}
+
+static struct mlx5dr_devx_obj *
+mlx5dr_definer_get_obj(struct mlx5dr_context *ctx,
+ struct mlx5dr_definer *definer)
+{
+ struct mlx5dr_definer_cache *cache = ctx->definer_cache;
+ struct mlx5dr_cmd_definer_create_attr def_attr = {0};
+ struct mlx5dr_definer_cache_item *cached_definer;
+ struct mlx5dr_devx_obj *obj;
+
+ /* Search definer cache for requested definer */
+ LIST_FOREACH(cached_definer, &cache->head, next) {
+ if (mlx5dr_definer_compare(&cached_definer->definer, definer))
+ continue;
+
+ /* Reuse definer and set LRU (move to be first in the list) */
+ LIST_REMOVE(cached_definer, next);
+ LIST_INSERT_HEAD(&cache->head, cached_definer, next);
+ cached_definer->refcount++;
+ return cached_definer->definer.obj;
+ }
+
+ /* Allocate and create definer based on the bitmask tag */
+ d
[v1 09/16] net/mlx5/hws: support range match
Support range matching over selected items and range
is not supported over all the items. The range match
is done using:
item->last.field - maximum value
item->mask.field - bitmask
item->spec.field - minimum value
When items are processed if item last and mask fields are
non zero range matching will be done over these fields.
There are two field setter, field copy (fc) and field copy
range (fcr).
Signed-off-by: Alex Vesker
---
drivers/net/mlx5/hws/mlx5dr_definer.c | 73 +--
drivers/net/mlx5/hws/mlx5dr_definer.h | 5 +-
2 files changed, 72 insertions(+), 6 deletions(-)
diff --git a/drivers/net/mlx5/hws/mlx5dr_definer.c
b/drivers/net/mlx5/hws/mlx5dr_definer.c
index 6b98eb8c96..c268f94ad3 100644
--- a/drivers/net/mlx5/hws/mlx5dr_definer.c
+++ b/drivers/net/mlx5/hws/mlx5dr_definer.c
@@ -123,6 +123,7 @@ struct mlx5dr_definer_conv_data {
X(SET, ipv4_next_proto,v->next_proto_id,
rte_ipv4_hdr) \
X(SET, ipv4_version, STE_IPV4,
rte_ipv4_hdr) \
X(SET_BE16, ipv4_frag, v->fragment_offset,
rte_ipv4_hdr) \
+ X(SET_BE16, ipv4_len, v->total_length,
rte_ipv4_hdr) \
X(SET_BE16, ipv6_payload_len, v->hdr.payload_len,
rte_flow_item_ipv6) \
X(SET, ipv6_proto, v->hdr.proto,
rte_flow_item_ipv6) \
X(SET, ipv6_hop_limits,v->hdr.hop_limits,
rte_flow_item_ipv6) \
@@ -516,6 +517,7 @@ mlx5dr_definer_conv_item_ipv4(struct
mlx5dr_definer_conv_data *cd,
int item_idx)
{
const struct rte_ipv4_hdr *m = item->mask;
+ const struct rte_ipv4_hdr *l = item->last;
struct mlx5dr_definer_fc *fc;
bool inner = cd->tunnel;
@@ -533,8 +535,8 @@ mlx5dr_definer_conv_item_ipv4(struct
mlx5dr_definer_conv_data *cd,
if (!m)
return 0;
- if (m->total_length || m->packet_id ||
- m->hdr_checksum) {
+ if (m->packet_id || m->hdr_checksum ||
+ (l && (l->next_proto_id || l->type_of_service))) {
rte_errno = ENOTSUP;
return rte_errno;
}
@@ -553,9 +555,18 @@ mlx5dr_definer_conv_item_ipv4(struct
mlx5dr_definer_conv_data *cd,
DR_CALC_SET(fc, eth_l3, protocol_next_header, inner);
}
+ if (m->total_length) {
+ fc = &cd->fc[DR_CALC_FNAME(IP_LEN, inner)];
+ fc->item_idx = item_idx;
+ fc->is_range = l && l->total_length;
+ fc->tag_set = &mlx5dr_definer_ipv4_len_set;
+ DR_CALC_SET(fc, eth_l3, ipv4_total_length, inner);
+ }
+
if (m->dst_addr) {
fc = &cd->fc[DR_CALC_FNAME(IPV4_DST, inner)];
fc->item_idx = item_idx;
+ fc->is_range = l && l->dst_addr;
fc->tag_set = &mlx5dr_definer_ipv4_dst_addr_set;
DR_CALC_SET(fc, ipv4_src_dest, destination_address, inner);
}
@@ -563,6 +574,7 @@ mlx5dr_definer_conv_item_ipv4(struct
mlx5dr_definer_conv_data *cd,
if (m->src_addr) {
fc = &cd->fc[DR_CALC_FNAME(IPV4_SRC, inner)];
fc->item_idx = item_idx;
+ fc->is_range = l && l->src_addr;
fc->tag_set = &mlx5dr_definer_ipv4_src_addr_set;
DR_CALC_SET(fc, ipv4_src_dest, source_address, inner);
}
@@ -570,6 +582,7 @@ mlx5dr_definer_conv_item_ipv4(struct
mlx5dr_definer_conv_data *cd,
if (m->ihl) {
fc = &cd->fc[DR_CALC_FNAME(IPV4_IHL, inner)];
fc->item_idx = item_idx;
+ fc->is_range = l && l->ihl;
fc->tag_set = &mlx5dr_definer_ipv4_ihl_set;
DR_CALC_SET(fc, eth_l3, ihl, inner);
}
@@ -577,6 +590,7 @@ mlx5dr_definer_conv_item_ipv4(struct
mlx5dr_definer_conv_data *cd,
if (m->time_to_live) {
fc = &cd->fc[DR_CALC_FNAME(IP_TTL, inner)];
fc->item_idx = item_idx;
+ fc->is_range = l && l->time_to_live;
fc->tag_set = &mlx5dr_definer_ipv4_time_to_live_set;
DR_CALC_SET(fc, eth_l3, time_to_live_hop_limit, inner);
}
@@ -597,6 +611,7 @@ mlx5dr_definer_conv_item_ipv6(struct
mlx5dr_definer_conv_data *cd,
int item_idx)
{
const struct rte_flow_item_ipv6 *m = item->mask;
+ const struct rte_flow_item_ipv6 *l = item->last;
struct mlx5dr_definer_fc *fc;
bool inner = cd->tunnel;
@@ -616,7 +631,10 @@ mlx5dr_definer_conv_item_ipv6(struct
mlx5dr_definer_conv_data *cd,
if (m->has_hop_ext || m->has_route_ext || m->h
[v1 12/16] net/mlx5/hws: add range definer creation support
Calculate and create an additional definer used for range
check during matcher creation. In such case two definers will
be created one for specific matching and a range definer.
Since range HW GTA WQE doesn't support the needed range
support rule insertion rule insertion is done using the FW
GTA WQE command.
Signed-off-by: Alex Vesker
---
drivers/net/mlx5/hws/mlx5dr_definer.c | 255 +++---
drivers/net/mlx5/hws/mlx5dr_definer.h | 16 +-
drivers/net/mlx5/hws/mlx5dr_matcher.c | 27 ++-
drivers/net/mlx5/hws/mlx5dr_matcher.h | 17 +-
4 files changed, 281 insertions(+), 34 deletions(-)
diff --git a/drivers/net/mlx5/hws/mlx5dr_definer.c
b/drivers/net/mlx5/hws/mlx5dr_definer.c
index 260e6c5d1d..cf84fbea71 100644
--- a/drivers/net/mlx5/hws/mlx5dr_definer.c
+++ b/drivers/net/mlx5/hws/mlx5dr_definer.c
@@ -1508,26 +1508,33 @@ mlx5dr_definer_mt_set_fc(struct mlx5dr_match_template
*mt,
struct mlx5dr_definer_fc *fc,
uint8_t *hl)
{
- uint32_t fc_sz = 0;
+ uint32_t fc_sz = 0, fcr_sz = 0;
int i;
for (i = 0; i < MLX5DR_DEFINER_FNAME_MAX; i++)
if (fc[i].tag_set)
- fc_sz++;
+ fc[i].is_range ? fcr_sz++ : fc_sz++;
- mt->fc = simple_calloc(fc_sz, sizeof(*mt->fc));
+ mt->fc = simple_calloc(fc_sz + fcr_sz, sizeof(*mt->fc));
if (!mt->fc) {
rte_errno = ENOMEM;
return rte_errno;
}
+ mt->fcr = mt->fc + fc_sz;
+
for (i = 0; i < MLX5DR_DEFINER_FNAME_MAX; i++) {
if (!fc[i].tag_set)
continue;
fc[i].fname = i;
- memcpy(&mt->fc[mt->fc_sz++], &fc[i], sizeof(*mt->fc));
- DR_SET(hl, -1, fc[i].byte_off, fc[i].bit_off, fc[i].bit_mask);
+
+ if (fc[i].is_range) {
+ memcpy(&mt->fcr[mt->fcr_sz++], &fc[i],
sizeof(*mt->fcr));
+ } else {
+ memcpy(&mt->fc[mt->fc_sz++], &fc[i], sizeof(*mt->fc));
+ DR_SET(hl, -1, fc[i].byte_off, fc[i].bit_off,
fc[i].bit_mask);
+ }
}
return 0;
@@ -1686,7 +1693,7 @@ mlx5dr_definer_conv_items_to_hl(struct mlx5dr_context
*ctx,
mt->item_flags = item_flags;
- /* Fill in headers layout and allocate fc array on mt */
+ /* Fill in headers layout and allocate fc & fcr array on mt */
ret = mlx5dr_definer_mt_set_fc(mt, fc, hl);
if (ret) {
DR_LOG(ERR, "Failed to set field copy to match template");
@@ -1855,9 +1862,92 @@ mlx5dr_definer_copy_sel_ctrl(struct
mlx5dr_definer_sel_ctrl *ctrl,
}
static int
-mlx5dr_definer_find_best_hl_fit(struct mlx5dr_context *ctx,
- struct mlx5dr_definer *definer,
- uint8_t *hl)
+mlx5dr_definer_find_best_range_fit(struct mlx5dr_definer *definer,
+ struct mlx5dr_matcher *matcher)
+{
+ uint8_t tag_byte_offset[MLX5DR_DEFINER_FNAME_MAX] = {0};
+ uint8_t field_select[MLX5DR_DEFINER_FNAME_MAX] = {0};
+ struct mlx5dr_definer_sel_ctrl ctrl = {0};
+ uint32_t byte_offset, algn_byte_off;
+ struct mlx5dr_definer_fc *fcr;
+ bool require_dw;
+ int idx, i, j;
+
+ /* Try to create a range definer */
+ ctrl.allowed_full_dw = DW_SELECTORS_RANGE;
+ ctrl.allowed_bytes = BYTE_SELECTORS_RANGE;
+
+ /* Multiple fields cannot share the same DW for range match.
+* The HW doesn't recognize each field but compares the full dw.
+* For example definer DW consists of FieldA_FieldB
+* FieldA: Mask 0x range 0x1 to 0x2
+* FieldB: Mask 0x range 0x3 to 0x4
+* STE DW range will be 0x00010003 - 0x00020004
+* This will cause invalid match for FieldB if FieldA=1 and FieldB=8
+* Since 0x10003 < 0x10008 < 0x20004
+*/
+ for (i = 0; i < matcher->num_of_mt; i++) {
+ for (j = 0; j < matcher->mt[i].fcr_sz; j++) {
+ fcr = &matcher->mt[i].fcr[j];
+
+ /* Found - Reuse previous mt binding */
+ if (field_select[fcr->fname]) {
+ fcr->byte_off = tag_byte_offset[fcr->fname];
+ continue;
+ }
+
+ /* Not found */
+ require_dw = fcr->byte_off >= (64 * DW_SIZE);
+ if (require_dw || ctrl.used_bytes ==
ctrl.allowed_bytes) {
+ /* Try to cover using DW selector */
+ if (ctrl.used_full_dw == ctrl.allowed_full_dw)
+ goto not_supporte
[v1 13/16] net/mlx5/hws: add FW WQE rule creation logic
FW WQE and HW WQE are done in a similar way but not to
jeopardize the performance rule creation is done over
the new FW rule creation function. The deletion function
is shared between both flows.
Signed-off-by: Alex Vesker
---
drivers/net/mlx5/hws/mlx5dr_rule.c | 180 +++--
drivers/net/mlx5/hws/mlx5dr_rule.h | 2 +
drivers/net/mlx5/hws/mlx5dr_send.h | 9 +-
3 files changed, 180 insertions(+), 11 deletions(-)
diff --git a/drivers/net/mlx5/hws/mlx5dr_rule.c
b/drivers/net/mlx5/hws/mlx5dr_rule.c
index f5a0c46315..9d5e5b11a5 100644
--- a/drivers/net/mlx5/hws/mlx5dr_rule.c
+++ b/drivers/net/mlx5/hws/mlx5dr_rule.c
@@ -112,6 +112,62 @@ static void mlx5dr_rule_gen_comp(struct mlx5dr_send_engine
*queue,
mlx5dr_send_engine_gen_comp(queue, user_data, comp_status);
}
+static void
+mlx5dr_rule_save_delete_info(struct mlx5dr_rule *rule,
+struct mlx5dr_send_ste_attr *ste_attr)
+{
+ if (unlikely(mlx5dr_matcher_req_fw_wqe(rule->matcher))) {
+ uint8_t *src_tag;
+
+ /* Save match definer id and tag for delete */
+ rule->tag_ptr = simple_calloc(2, sizeof(*rule->tag_ptr));
+ assert(rule->tag_ptr);
+
+ src_tag = (uint8_t *)ste_attr->wqe_data->tag;
+ memcpy(rule->tag_ptr[0].match, src_tag, MLX5DR_MATCH_TAG_SZ);
+ rule->tag_ptr[1].reserved[0] =
ste_attr->send_attr.match_definer_id;
+
+ /* Save range definer id and tag for delete */
+ if (ste_attr->range_wqe_data) {
+ src_tag = (uint8_t *)ste_attr->range_wqe_data->tag;
+ memcpy(rule->tag_ptr[1].match, src_tag,
MLX5DR_MATCH_TAG_SZ);
+ rule->tag_ptr[1].reserved[1] =
ste_attr->send_attr.range_definer_id;
+ }
+ return;
+ }
+
+ if (ste_attr->wqe_tag_is_jumbo)
+ memcpy(rule->tag.jumbo, ste_attr->wqe_data->jumbo,
MLX5DR_JUMBO_TAG_SZ);
+ else
+ memcpy(rule->tag.match, ste_attr->wqe_data->tag,
MLX5DR_MATCH_TAG_SZ);
+}
+
+static void
+mlx5dr_rule_clear_delete_info(struct mlx5dr_rule *rule)
+{
+ if (unlikely(mlx5dr_matcher_req_fw_wqe(rule->matcher)))
+ simple_free(rule->tag_ptr);
+}
+
+static void
+mlx5dr_rule_load_delete_info(struct mlx5dr_rule *rule,
+struct mlx5dr_send_ste_attr *ste_attr)
+{
+ if (unlikely(mlx5dr_matcher_req_fw_wqe(rule->matcher))) {
+ /* Load match definer id and tag for delete */
+ ste_attr->wqe_tag = &rule->tag_ptr[0];
+ ste_attr->send_attr.match_definer_id =
rule->tag_ptr[1].reserved[0];
+
+ /* Load range definer id and tag for delete */
+ if (rule->matcher->flags & MLX5DR_MATCHER_FLAGS_RANGE_DEFINER) {
+ ste_attr->range_wqe_tag = &rule->tag_ptr[1];
+ ste_attr->send_attr.range_definer_id =
rule->tag_ptr[1].reserved[1];
+ }
+ } else {
+ ste_attr->wqe_tag = &rule->tag;
+ }
+}
+
static int mlx5dr_rule_alloc_action_ste(struct mlx5dr_rule *rule,
struct mlx5dr_rule_attr *attr)
{
@@ -180,6 +236,97 @@ static void mlx5dr_rule_create_init(struct mlx5dr_rule
*rule,
apply->require_dep = 0;
}
+static int mlx5dr_rule_create_hws_fw_wqe(struct mlx5dr_rule *rule,
+struct mlx5dr_rule_attr *attr,
+uint8_t mt_idx,
+const struct rte_flow_item items[],
+uint8_t at_idx,
+struct mlx5dr_rule_action
rule_actions[])
+{
+ struct mlx5dr_action_template *at = &rule->matcher->at[at_idx];
+ struct mlx5dr_match_template *mt = &rule->matcher->mt[mt_idx];
+ struct mlx5dr_send_ring_dep_wqe range_wqe = {{0}};
+ struct mlx5dr_send_ring_dep_wqe match_wqe = {{0}};
+ bool is_range = mlx5dr_matcher_mt_is_range(mt);
+ bool is_jumbo = mlx5dr_matcher_mt_is_jumbo(mt);
+ struct mlx5dr_matcher *matcher = rule->matcher;
+ struct mlx5dr_context *ctx = matcher->tbl->ctx;
+ struct mlx5dr_send_ste_attr ste_attr = {0};
+ struct mlx5dr_actions_apply_data apply;
+ struct mlx5dr_send_engine *queue;
+
+ queue = &ctx->send_queue[attr->queue_id];
+ if (unlikely(mlx5dr_send_engine_err(queue))) {
+ rte_errno = EIO;
+ return rte_errno;
+ }
+
+ mlx5dr_rule_create_init(rule, &ste_attr, &apply);
+ mlx5dr_rule_init_dep_wqe(&match_wqe, rule, items, mt, attr->user_data);
+ mlx5dr_rule_init_dep_wqe(&range_wqe, rule, items,
[v1 15/16] net/mlx5/hws: rename pattern cache object
To have the same name convention for future caches, use
cache and cache item naming.
Signed-off-by: Alex Vesker
---
drivers/net/mlx5/hws/mlx5dr_pat_arg.c | 22 +++---
drivers/net/mlx5/hws/mlx5dr_pat_arg.h | 6 +++---
2 files changed, 14 insertions(+), 14 deletions(-)
diff --git a/drivers/net/mlx5/hws/mlx5dr_pat_arg.c
b/drivers/net/mlx5/hws/mlx5dr_pat_arg.c
index 152025d302..6ed04dac6d 100644
--- a/drivers/net/mlx5/hws/mlx5dr_pat_arg.c
+++ b/drivers/net/mlx5/hws/mlx5dr_pat_arg.c
@@ -94,13 +94,13 @@ static bool mlx5dr_pat_compare_pattern(enum
mlx5dr_action_type cur_type,
return true;
}
-static struct mlx5dr_pat_cached_pattern *
+static struct mlx5dr_pattern_cache_item *
mlx5dr_pat_find_cached_pattern(struct mlx5dr_pattern_cache *cache,
struct mlx5dr_action *action,
uint16_t num_of_actions,
__be64 *actions)
{
- struct mlx5dr_pat_cached_pattern *cached_pat;
+ struct mlx5dr_pattern_cache_item *cached_pat;
LIST_FOREACH(cached_pat, &cache->head, next) {
if (mlx5dr_pat_compare_pattern(cached_pat->type,
@@ -115,13 +115,13 @@ mlx5dr_pat_find_cached_pattern(struct
mlx5dr_pattern_cache *cache,
return NULL;
}
-static struct mlx5dr_pat_cached_pattern *
+static struct mlx5dr_pattern_cache_item *
mlx5dr_pat_get_existing_cached_pattern(struct mlx5dr_pattern_cache *cache,
struct mlx5dr_action *action,
uint16_t num_of_actions,
__be64 *actions)
{
- struct mlx5dr_pat_cached_pattern *cached_pattern;
+ struct mlx5dr_pattern_cache_item *cached_pattern;
cached_pattern = mlx5dr_pat_find_cached_pattern(cache, action,
num_of_actions, actions);
if (cached_pattern) {
@@ -134,11 +134,11 @@ mlx5dr_pat_get_existing_cached_pattern(struct
mlx5dr_pattern_cache *cache,
return cached_pattern;
}
-static struct mlx5dr_pat_cached_pattern *
+static struct mlx5dr_pattern_cache_item *
mlx5dr_pat_get_cached_pattern_by_action(struct mlx5dr_pattern_cache *cache,
struct mlx5dr_action *action)
{
- struct mlx5dr_pat_cached_pattern *cached_pattern;
+ struct mlx5dr_pattern_cache_item *cached_pattern;
LIST_FOREACH(cached_pattern, &cache->head, next) {
if (cached_pattern->mh_data.pattern_obj->id ==
action->modify_header.pattern_obj->id)
@@ -148,14 +148,14 @@ mlx5dr_pat_get_cached_pattern_by_action(struct
mlx5dr_pattern_cache *cache,
return NULL;
}
-static struct mlx5dr_pat_cached_pattern *
+static struct mlx5dr_pattern_cache_item *
mlx5dr_pat_add_pattern_to_cache(struct mlx5dr_pattern_cache *cache,
struct mlx5dr_devx_obj *pattern_obj,
enum mlx5dr_action_type type,
uint16_t num_of_actions,
__be64 *actions)
{
- struct mlx5dr_pat_cached_pattern *cached_pattern;
+ struct mlx5dr_pattern_cache_item *cached_pattern;
cached_pattern = simple_calloc(1, sizeof(*cached_pattern));
if (!cached_pattern) {
@@ -189,7 +189,7 @@ mlx5dr_pat_add_pattern_to_cache(struct mlx5dr_pattern_cache
*cache,
}
static void
-mlx5dr_pat_remove_pattern(struct mlx5dr_pat_cached_pattern *cached_pattern)
+mlx5dr_pat_remove_pattern(struct mlx5dr_pattern_cache_item *cached_pattern)
{
LIST_REMOVE(cached_pattern, next);
simple_free(cached_pattern->mh_data.data);
@@ -200,7 +200,7 @@ static void
mlx5dr_pat_put_pattern(struct mlx5dr_pattern_cache *cache,
struct mlx5dr_action *action)
{
- struct mlx5dr_pat_cached_pattern *cached_pattern;
+ struct mlx5dr_pattern_cache_item *cached_pattern;
pthread_spin_lock(&cache->lock);
cached_pattern = mlx5dr_pat_get_cached_pattern_by_action(cache, action);
@@ -225,7 +225,7 @@ static int mlx5dr_pat_get_pattern(struct mlx5dr_context
*ctx,
size_t pattern_sz,
__be64 *pattern)
{
- struct mlx5dr_pat_cached_pattern *cached_pattern;
+ struct mlx5dr_pattern_cache_item *cached_pattern;
int ret = 0;
pthread_spin_lock(&ctx->pattern_cache->lock);
diff --git a/drivers/net/mlx5/hws/mlx5dr_pat_arg.h
b/drivers/net/mlx5/hws/mlx5dr_pat_arg.h
index d9353e9a3e..92db6d6aee 100644
--- a/drivers/net/mlx5/hws/mlx5dr_pat_arg.h
+++ b/drivers/net/mlx5/hws/mlx5dr_pat_arg.h
@@ -24,10 +24,10 @@ enum {
struct mlx5dr_pattern_cache {
/* Protect pattern list */
pthread_spinlock_t lock;
- LIST_HEAD(pattern_head, mlx5dr_pat_cached_pattern) head;
+ LIST_HEAD(pattern_head, mlx5dr_pattern_cache_item) head;
};
-struct mlx5dr_pat_cached_pattern {
+struct mlx5dr_pattern_
[v1 01/16] net/mlx5/hws: support synchronous drain
Until now we supported asynchronous drain, triggering the queue
to start the drain, now we added support for synchronous which
assures all the work was processed on the queue.
This is useful when working over a FW command and HW queue in parallel
sending arguments over the HW queue and match over the FW command
which requires synchronization.
This also fixes an issue with shared arguments send that require more than
one WQE.
Signed-off-by: Erez Shitrit
Signed-off-by: Alex Vesker
---
drivers/net/mlx5/hws/mlx5dr.h | 6 --
drivers/net/mlx5/hws/mlx5dr_pat_arg.c | 27 ---
drivers/net/mlx5/hws/mlx5dr_send.c| 16 ++--
drivers/net/mlx5/hws/mlx5dr_send.h| 5 +
drivers/net/mlx5/mlx5_flow_hw.c | 2 +-
5 files changed, 28 insertions(+), 28 deletions(-)
diff --git a/drivers/net/mlx5/hws/mlx5dr.h b/drivers/net/mlx5/hws/mlx5dr.h
index b3b2bf34f2..2b02884dc3 100644
--- a/drivers/net/mlx5/hws/mlx5dr.h
+++ b/drivers/net/mlx5/hws/mlx5dr.h
@@ -86,8 +86,10 @@ enum mlx5dr_match_template_flags {
};
enum mlx5dr_send_queue_actions {
- /* Start executing all pending queued rules and write to HW */
- MLX5DR_SEND_QUEUE_ACTION_DRAIN = 1 << 0,
+ /* Start executing all pending queued rules */
+ MLX5DR_SEND_QUEUE_ACTION_DRAIN_ASYNC = 1 << 0,
+ /* Start executing all pending queued rules wait till completion */
+ MLX5DR_SEND_QUEUE_ACTION_DRAIN_SYNC = 1 << 1,
};
struct mlx5dr_context_attr {
diff --git a/drivers/net/mlx5/hws/mlx5dr_pat_arg.c
b/drivers/net/mlx5/hws/mlx5dr_pat_arg.c
index df451f1ae0..152025d302 100644
--- a/drivers/net/mlx5/hws/mlx5dr_pat_arg.c
+++ b/drivers/net/mlx5/hws/mlx5dr_pat_arg.c
@@ -306,27 +306,6 @@ void mlx5dr_arg_decapl3_write(struct mlx5dr_send_engine
*queue,
mlx5dr_send_engine_post_end(&ctrl, &send_attr);
}
-static int
-mlx5dr_arg_poll_for_comp(struct mlx5dr_context *ctx, uint16_t queue_id)
-{
- struct rte_flow_op_result comp[1];
- int ret;
-
- while (true) {
- ret = mlx5dr_send_queue_poll(ctx, queue_id, comp, 1);
- if (ret) {
- if (ret < 0) {
- DR_LOG(ERR, "Failed mlx5dr_send_queue_poll");
- } else if (comp[0].status == RTE_FLOW_OP_ERROR) {
- DR_LOG(ERR, "Got comp with error");
- rte_errno = ENOENT;
- }
- break;
- }
- }
- return (ret == 1 ? 0 : ret);
-}
-
void mlx5dr_arg_write(struct mlx5dr_send_engine *queue,
void *comp_data,
uint32_t arg_idx,
@@ -388,9 +367,11 @@ int mlx5dr_arg_write_inline_arg_data(struct mlx5dr_context
*ctx,
mlx5dr_send_engine_flush_queue(queue);
/* Poll for completion */
- ret = mlx5dr_arg_poll_for_comp(ctx, ctx->queues - 1);
+ ret = mlx5dr_send_queue_action(ctx, ctx->queues - 1,
+ MLX5DR_SEND_QUEUE_ACTION_DRAIN_SYNC);
+
if (ret)
- DR_LOG(ERR, "Failed to get completions for shared action");
+ DR_LOG(ERR, "Failed to drain arg queue");
pthread_spin_unlock(&ctx->ctrl_lock);
diff --git a/drivers/net/mlx5/hws/mlx5dr_send.c
b/drivers/net/mlx5/hws/mlx5dr_send.c
index 5c8bbe6fc6..a507e5f626 100644
--- a/drivers/net/mlx5/hws/mlx5dr_send.c
+++ b/drivers/net/mlx5/hws/mlx5dr_send.c
@@ -830,18 +830,30 @@ int mlx5dr_send_queue_action(struct mlx5dr_context *ctx,
{
struct mlx5dr_send_ring_sq *send_sq;
struct mlx5dr_send_engine *queue;
+ bool wait_comp = false;
+ int64_t polled = 0;
queue = &ctx->send_queue[queue_id];
send_sq = &queue->send_ring->send_sq;
- if (actions == MLX5DR_SEND_QUEUE_ACTION_DRAIN) {
+ switch (actions) {
+ case MLX5DR_SEND_QUEUE_ACTION_DRAIN_SYNC:
+ wait_comp = true;
+ /* FALLTHROUGH */
+ case MLX5DR_SEND_QUEUE_ACTION_DRAIN_ASYNC:
if (send_sq->head_dep_idx != send_sq->tail_dep_idx)
/* Send dependent WQEs to drain the queue */
mlx5dr_send_all_dep_wqe(queue);
else
/* Signal on the last posted WQE */
mlx5dr_send_engine_flush_queue(queue);
- } else {
+
+ /* Poll queue until empty */
+ while (wait_comp && !mlx5dr_send_engine_empty(queue))
+ mlx5dr_send_engine_poll_cqs(queue, NULL, &polled, 0);
+
+ break;
+ default:
rte_errno = -EINVAL;
return rte_errno;
}
diff --git a/drivers/net/mlx5/hws/mlx5dr_send.h
b/drivers/net/mlx5/hws/mlx5dr_send.h
index 8d4769495d..fcddcc6366 100644
--- a/drivers
[v1 05/16] net/mlx5/hws: align RTC create command with PRM format
Rename rtc params create for new format.
Signed-off-by: Alex Vesker
---
drivers/common/mlx5/mlx5_prm.h| 16 ++--
drivers/net/mlx5/hws/mlx5dr_cmd.c | 13 +++--
drivers/net/mlx5/hws/mlx5dr_cmd.h | 11 +++
drivers/net/mlx5/hws/mlx5dr_matcher.c | 19 ---
4 files changed, 40 insertions(+), 19 deletions(-)
diff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h
index 6d0b5e640c..cf46296afb 100644
--- a/drivers/common/mlx5/mlx5_prm.h
+++ b/drivers/common/mlx5/mlx5_prm.h
@@ -3237,6 +3237,7 @@ enum mlx5_ifc_rtc_access_mode {
enum mlx5_ifc_rtc_ste_format {
MLX5_IFC_RTC_STE_FORMAT_8DW = 0x4,
MLX5_IFC_RTC_STE_FORMAT_11DW = 0x5,
+ MLX5_IFC_RTC_STE_FORMAT_RANGE = 0x7,
};
enum mlx5_ifc_rtc_reparse_mode {
@@ -3251,24 +3252,27 @@ struct mlx5_ifc_rtc_bits {
u8 reserved_at_40[0x40];
u8 update_index_mode[0x2];
u8 reparse_mode[0x2];
- u8 reserved_at_84[0x4];
+ u8 num_match_ste[0x4];
u8 pd[0x18];
u8 reserved_at_a0[0x9];
u8 access_index_mode[0x3];
u8 num_hash_definer[0x4];
- u8 reserved_at_b0[0x3];
+ u8 update_method[0x1];
+ u8 reserved_at_b1[0x2];
u8 log_depth[0x5];
u8 log_hash_size[0x8];
- u8 ste_format[0x8];
+ u8 ste_format_0[0x8];
u8 table_type[0x8];
- u8 reserved_at_d0[0x10];
- u8 match_definer_id[0x20];
+ u8 ste_format_1[0x8];
+ u8 reserved_at_d8[0x8];
+ u8 match_definer_0[0x20];
u8 stc_id[0x20];
u8 ste_table_base_id[0x20];
u8 ste_table_offset[0x20];
u8 reserved_at_160[0x8];
u8 miss_flow_table_id[0x18];
- u8 reserved_at_180[0x280];
+ u8 match_definer_1[0x20];
+ u8 reserved_at_1a0[0x260];
};
struct mlx5_ifc_alias_context_bits {
diff --git a/drivers/net/mlx5/hws/mlx5dr_cmd.c
b/drivers/net/mlx5/hws/mlx5dr_cmd.c
index e311be780b..a8d1cf0322 100644
--- a/drivers/net/mlx5/hws/mlx5dr_cmd.c
+++ b/drivers/net/mlx5/hws/mlx5dr_cmd.c
@@ -259,17 +259,26 @@ mlx5dr_cmd_rtc_create(struct ibv_context *ctx,
attr, obj_type, MLX5_GENERAL_OBJ_TYPE_RTC);
attr = MLX5_ADDR_OF(create_rtc_in, in, rtc);
- MLX5_SET(rtc, attr, ste_format, rtc_attr->is_jumbo ?
+ MLX5_SET(rtc, attr, ste_format_0, rtc_attr->is_frst_jumbo ?
MLX5_IFC_RTC_STE_FORMAT_11DW :
MLX5_IFC_RTC_STE_FORMAT_8DW);
+
+ if (rtc_attr->is_scnd_range) {
+ MLX5_SET(rtc, attr, ste_format_1,
MLX5_IFC_RTC_STE_FORMAT_RANGE);
+ MLX5_SET(rtc, attr, num_match_ste, 2);
+ }
+
MLX5_SET(rtc, attr, pd, rtc_attr->pd);
+ MLX5_SET(rtc, attr, update_method, rtc_attr->fw_gen_wqe);
MLX5_SET(rtc, attr, update_index_mode, rtc_attr->update_index_mode);
MLX5_SET(rtc, attr, access_index_mode, rtc_attr->access_index_mode);
MLX5_SET(rtc, attr, num_hash_definer, rtc_attr->num_hash_definer);
MLX5_SET(rtc, attr, log_depth, rtc_attr->log_depth);
MLX5_SET(rtc, attr, log_hash_size, rtc_attr->log_size);
MLX5_SET(rtc, attr, table_type, rtc_attr->table_type);
- MLX5_SET(rtc, attr, match_definer_id, rtc_attr->definer_id);
+ MLX5_SET(rtc, attr, num_hash_definer, rtc_attr->num_hash_definer);
+ MLX5_SET(rtc, attr, match_definer_0, rtc_attr->match_definer_0);
+ MLX5_SET(rtc, attr, match_definer_1, rtc_attr->match_definer_1);
MLX5_SET(rtc, attr, stc_id, rtc_attr->stc_base);
MLX5_SET(rtc, attr, ste_table_base_id, rtc_attr->ste_base);
MLX5_SET(rtc, attr, ste_table_offset, rtc_attr->ste_offset);
diff --git a/drivers/net/mlx5/hws/mlx5dr_cmd.h
b/drivers/net/mlx5/hws/mlx5dr_cmd.h
index a42218ba74..e062cb8171 100644
--- a/drivers/net/mlx5/hws/mlx5dr_cmd.h
+++ b/drivers/net/mlx5/hws/mlx5dr_cmd.h
@@ -23,8 +23,8 @@ struct mlx5dr_cmd_ft_modify_attr {
};
struct mlx5dr_cmd_fg_attr {
- uint32_ttable_id;
- uint32_ttable_type;
+ uint32_t table_id;
+ uint32_t table_type;
};
struct mlx5dr_cmd_forward_tbl {
@@ -40,14 +40,17 @@ struct mlx5dr_cmd_rtc_create_attr {
uint32_t ste_base;
uint32_t ste_offset;
uint32_t miss_ft_id;
+ bool fw_gen_wqe;
uint8_t update_index_mode;
uint8_t access_index_mode;
uint8_t num_hash_definer;
uint8_t log_depth;
uint8_t log_size;
uint8_t table_type;
- uint8_t definer_id;
- bool is_jumbo;
+ uint8_t match_definer_0;
+ uint8_t match_definer_1;
+ bool is_frst_jumbo;
+ bool is_scnd_range;
};
struct mlx5dr_cmd_alias_obj_create_attr {
diff --git a/drivers/net/mlx5/hws/mlx5dr_matcher.c
b/drivers/net/mlx5/hws/mlx5dr_matcher.c
index 913bb9d447..101a12d361 100644
--- a/drivers/net/mlx5/hws/mlx5dr_matcher.c
+++ b/drivers/net/mlx5/hws/mlx5dr_matcher.c
@@ -413,6 +413,8 @@ stat
[v1 04/16] net/mlx5/hws: add capability query for gen wqe command
Read the capabilities required to determine support for GENERATE_WQE.
Signed-off-by: Alex Vesker
---
drivers/common/mlx5/mlx5_prm.h| 6 --
drivers/net/mlx5/hws/mlx5dr_cmd.c | 12
drivers/net/mlx5/hws/mlx5dr_cmd.h | 3 +++
3 files changed, 19 insertions(+), 2 deletions(-)
diff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h
index d4d8ddcd2a..6d0b5e640c 100644
--- a/drivers/common/mlx5/mlx5_prm.h
+++ b/drivers/common/mlx5/mlx5_prm.h
@@ -2205,10 +2205,12 @@ struct mlx5_ifc_wqe_based_flow_table_cap_bits {
u8 header_insert_type[0x10];
u8 header_remove_type[0x10];
u8 trivial_match_definer[0x20];
- u8 reserved_at_140[0x20];
+ u8 reserved_at_140[0x1b];
+ u8 rtc_max_num_hash_definer_gen_wqe[0x5];
u8 reserved_at_160[0x18];
u8 access_index_mode[0x8];
- u8 reserved_at_180[0x20];
+ u8 reserved_at_180[0x10];
+ u8 ste_fromat_gen_wqe[0x10];
u8 linear_match_definer_reg_c3[0x20];
};
diff --git a/drivers/net/mlx5/hws/mlx5dr_cmd.c
b/drivers/net/mlx5/hws/mlx5dr_cmd.c
index c648eacd03..e311be780b 100644
--- a/drivers/net/mlx5/hws/mlx5dr_cmd.c
+++ b/drivers/net/mlx5/hws/mlx5dr_cmd.c
@@ -928,6 +928,10 @@ int mlx5dr_cmd_query_caps(struct ibv_context *ctx,
capability.cmd_hca_cap_2.
format_select_dw_gtpu_first_ext_dw_0);
+ caps->supp_type_gen_wqe = MLX5_GET(query_hca_cap_out, out,
+ capability.cmd_hca_cap_2.
+ generate_wqe_type);
+
/* check cross-VHCA support in cap2 */
res =
MLX5_GET(query_hca_cap_out, out,
@@ -1033,6 +1037,14 @@ int mlx5dr_cmd_query_caps(struct ibv_context *ctx,
caps->linear_match_definer = MLX5_GET(query_hca_cap_out, out,
capability.wqe_based_flow_table_cap.
linear_match_definer_reg_c3);
+
+ caps->rtc_max_hash_def_gen_wqe = MLX5_GET(query_hca_cap_out,
out,
+
capability.wqe_based_flow_table_cap.
+
rtc_max_num_hash_definer_gen_wqe);
+
+ caps->supp_ste_fromat_gen_wqe = MLX5_GET(query_hca_cap_out, out,
+
capability.wqe_based_flow_table_cap.
+ste_fromat_gen_wqe);
}
if (caps->eswitch_manager) {
diff --git a/drivers/net/mlx5/hws/mlx5dr_cmd.h
b/drivers/net/mlx5/hws/mlx5dr_cmd.h
index 3689d09897..a42218ba74 100644
--- a/drivers/net/mlx5/hws/mlx5dr_cmd.h
+++ b/drivers/net/mlx5/hws/mlx5dr_cmd.h
@@ -183,6 +183,9 @@ struct mlx5dr_cmd_query_caps {
bool full_dw_jumbo_support;
bool rtc_hash_split_table;
bool rtc_linear_lookup_table;
+ uint32_t supp_type_gen_wqe;
+ uint8_t rtc_max_hash_def_gen_wqe;
+ uint16_t supp_ste_fromat_gen_wqe;
struct mlx5dr_cmd_query_ft_caps nic_ft;
struct mlx5dr_cmd_query_ft_caps fdb_ft;
bool eswitch_manager;
--
2.18.1
[v2 02/16] net/mlx5/hws: matcher remove AT and MT limitation
The action and match templates were stored on the matcher in
a fixed size array to reduce cache misses and reuse template
calculations. This approuch introduced two issues:
-limitation of fixed array
-definer is bindind to match template and cannot be used with
union definer since the layout is fixed
Signed-off-by: Alex Vesker
---
drivers/net/mlx5/hws/mlx5dr_debug.c | 4 +-
drivers/net/mlx5/hws/mlx5dr_matcher.c | 109 --
drivers/net/mlx5/hws/mlx5dr_matcher.h | 8 +-
drivers/net/mlx5/hws/mlx5dr_rule.c| 15 ++--
4 files changed, 78 insertions(+), 58 deletions(-)
diff --git a/drivers/net/mlx5/hws/mlx5dr_debug.c
b/drivers/net/mlx5/hws/mlx5dr_debug.c
index 0815327b18..9199ec16e0 100644
--- a/drivers/net/mlx5/hws/mlx5dr_debug.c
+++ b/drivers/net/mlx5/hws/mlx5dr_debug.c
@@ -92,7 +92,7 @@ mlx5dr_debug_dump_matcher_match_template(FILE *f, struct
mlx5dr_matcher *matcher
int i, ret;
for (i = 0; i < matcher->num_of_mt; i++) {
- struct mlx5dr_match_template *mt = matcher->mt[i];
+ struct mlx5dr_match_template *mt = &matcher->mt[i];
ret = fprintf(f, "%d,0x%" PRIx64 ",0x%" PRIx64 ",%d,%d\n",
MLX5DR_DEBUG_RES_TYPE_MATCHER_MATCH_TEMPLATE,
@@ -123,7 +123,7 @@ mlx5dr_debug_dump_matcher_action_template(FILE *f, struct
mlx5dr_matcher *matche
int i, j, ret;
for (i = 0; i < matcher->num_of_at; i++) {
- struct mlx5dr_action_template *at = matcher->at[i];
+ struct mlx5dr_action_template *at = &matcher->at[i];
ret = fprintf(f, "%d,0x%" PRIx64 ",0x%" PRIx64 ",%d,%d,%d",
MLX5DR_DEBUG_RES_TYPE_MATCHER_ACTION_TEMPLATE,
diff --git a/drivers/net/mlx5/hws/mlx5dr_matcher.c
b/drivers/net/mlx5/hws/mlx5dr_matcher.c
index d509a2f0e1..913bb9d447 100644
--- a/drivers/net/mlx5/hws/mlx5dr_matcher.c
+++ b/drivers/net/mlx5/hws/mlx5dr_matcher.c
@@ -430,8 +430,8 @@ static int mlx5dr_matcher_create_rtc(struct mlx5dr_matcher
*matcher,
/* The usual Hash Table */
rtc_attr.update_index_mode =
MLX5_IFC_RTC_STE_UPDATE_MODE_BY_HASH;
/* The first match template is used since all share the
same definer */
- rtc_attr.definer_id =
mlx5dr_definer_get_id(matcher->mt[0]->definer);
- rtc_attr.is_jumbo =
mlx5dr_definer_is_jumbo(matcher->mt[0]->definer);
+ rtc_attr.definer_id =
mlx5dr_definer_get_id(matcher->mt->definer);
+ rtc_attr.is_jumbo =
mlx5dr_definer_is_jumbo(matcher->mt->definer);
} else if (attr->insert_mode == MLX5DR_MATCHER_INSERT_BY_INDEX)
{
rtc_attr.update_index_mode =
MLX5_IFC_RTC_STE_UPDATE_MODE_BY_OFFSET;
rtc_attr.num_hash_definer = 1;
@@ -439,10 +439,8 @@ static int mlx5dr_matcher_create_rtc(struct mlx5dr_matcher
*matcher,
if (attr->distribute_mode ==
MLX5DR_MATCHER_DISTRIBUTE_BY_HASH) {
/* Hash Split Table */
rtc_attr.access_index_mode =
MLX5_IFC_RTC_STE_ACCESS_MODE_BY_HASH;
- rtc_attr.definer_id =
-
mlx5dr_definer_get_id(matcher->mt[0]->definer);
- rtc_attr.is_jumbo =
-
mlx5dr_definer_is_jumbo(matcher->mt[0]->definer);
+ rtc_attr.definer_id =
mlx5dr_definer_get_id(matcher->mt->definer);
+ rtc_attr.is_jumbo =
mlx5dr_definer_is_jumbo(matcher->mt->definer);
} else if (attr->distribute_mode ==
MLX5DR_MATCHER_DISTRIBUTE_BY_LINEAR) {
/* Linear Lookup Table */
rtc_attr.access_index_mode =
MLX5_IFC_RTC_STE_ACCESS_MODE_LINEAR;
@@ -579,7 +577,7 @@ static void mlx5dr_matcher_set_pool_attr(struct
mlx5dr_pool_attr *attr,
static int mlx5dr_matcher_bind_at(struct mlx5dr_matcher *matcher)
{
- bool is_jumbo = mlx5dr_definer_is_jumbo(matcher->mt[0]->definer);
+ bool is_jumbo = mlx5dr_definer_is_jumbo(matcher->mt->definer);
struct mlx5dr_cmd_stc_modify_attr stc_attr = {0};
struct mlx5dr_table *tbl = matcher->tbl;
struct mlx5dr_pool_attr pool_attr = {0};
@@ -589,7 +587,7 @@ static int mlx5dr_matcher_bind_at(struct mlx5dr_matcher
*matcher)
bool valid;
for (i = 0; i < matcher->num_of_at; i++) {
- struct mlx5dr_action_template *at = matcher->at[i];
+ struct mlx5dr_action_template *at = &matcher->at[i];
/* Check if action combinabtion is valid */
[v2 00/16] net/mlx5/hws: support range and partial hash matching
ConnectX and BlueField devices which support HWS are capable of performing advanced match, range-match and hashing operations on packet headers. This patch-set introduces support for (1) range matching - allows combining range and exact match over provided item fields using spec, last and mask. A useful example for Range and exact match is matching exact IP and port range. This series also introduces (2) partial hash, which allows using the same matcher and performing partial hashing in case of multiple match templates are provided. The matcher create code finds the union and intersection fields to split the hash and match in a way the HW can support. This increases the match templates combinations possible although it may limit the maximum PPS. Since (1) and (2) are not fully support on current HW the implementation is done on top of a HW like FW API using WQEs. v2: -Fix missed checkpatch comments Alex Vesker (16): net/mlx5/hws: support synchronous drain net/mlx5/hws: matcher remove AT and MT limitation net/mlx5/hws: support GTA WQE write using FW command net/mlx5/hws: add capability query for gen wqe command net/mlx5/hws: align RTC create command with PRM format net/mlx5/hws: add send FW match STE using gen WQE net/mlx5/hws: add send FW range STE WQE net/mlx5/hws: move matcher size check to function net/mlx5/hws: support range match net/mlx5/hws: redesign definer create net/mlx5/hws: support partial hash net/mlx5/hws: add range definer creation support net/mlx5/hws: add FW WQE rule creation logic net/mlx5/hws: add debug dump support for range and hash net/mlx5/hws: rename pattern cache object net/mlx5/hws: cache definer for reuse drivers/common/mlx5/mlx5_prm.h | 53 +- drivers/net/mlx5/hws/mlx5dr.h | 6 +- drivers/net/mlx5/hws/mlx5dr_cmd.c | 72 ++- drivers/net/mlx5/hws/mlx5dr_cmd.h | 27 +- drivers/net/mlx5/hws/mlx5dr_context.c | 12 +- drivers/net/mlx5/hws/mlx5dr_context.h | 1 + drivers/net/mlx5/hws/mlx5dr_debug.c| 41 +- drivers/net/mlx5/hws/mlx5dr_debug.h| 4 +- drivers/net/mlx5/hws/mlx5dr_definer.c | 750 + drivers/net/mlx5/hws/mlx5dr_definer.h | 44 +- drivers/net/mlx5/hws/mlx5dr_internal.h | 2 +- drivers/net/mlx5/hws/mlx5dr_matcher.c | 279 ++--- drivers/net/mlx5/hws/mlx5dr_matcher.h | 43 +- drivers/net/mlx5/hws/mlx5dr_pat_arg.c | 49 +- drivers/net/mlx5/hws/mlx5dr_pat_arg.h | 6 +- drivers/net/mlx5/hws/mlx5dr_rule.c | 193 ++- drivers/net/mlx5/hws/mlx5dr_rule.h | 2 + drivers/net/mlx5/hws/mlx5dr_send.c | 169 +- drivers/net/mlx5/hws/mlx5dr_send.h | 24 +- drivers/net/mlx5/mlx5_flow_hw.c| 2 +- 20 files changed, 1475 insertions(+), 304 deletions(-) -- 2.18.1
[v2 01/16] net/mlx5/hws: support synchronous drain
Until now we supported asynchronous drain, triggering the queue
to start the drain, now we added support for synchronous which
assures all the work was processed on the queue.
This is useful when working over a FW command and HW queue in parallel
sending arguments over the HW queue and match over the FW command
which requires synchronization.
This also fixes an issue with shared arguments send that require more than
one WQE.
Signed-off-by: Erez Shitrit
Signed-off-by: Alex Vesker
---
drivers/net/mlx5/hws/mlx5dr.h | 6 --
drivers/net/mlx5/hws/mlx5dr_pat_arg.c | 27 ---
drivers/net/mlx5/hws/mlx5dr_send.c| 16 ++--
drivers/net/mlx5/hws/mlx5dr_send.h| 5 +
drivers/net/mlx5/mlx5_flow_hw.c | 2 +-
5 files changed, 28 insertions(+), 28 deletions(-)
diff --git a/drivers/net/mlx5/hws/mlx5dr.h b/drivers/net/mlx5/hws/mlx5dr.h
index b3b2bf34f2..2b02884dc3 100644
--- a/drivers/net/mlx5/hws/mlx5dr.h
+++ b/drivers/net/mlx5/hws/mlx5dr.h
@@ -86,8 +86,10 @@ enum mlx5dr_match_template_flags {
};
enum mlx5dr_send_queue_actions {
- /* Start executing all pending queued rules and write to HW */
- MLX5DR_SEND_QUEUE_ACTION_DRAIN = 1 << 0,
+ /* Start executing all pending queued rules */
+ MLX5DR_SEND_QUEUE_ACTION_DRAIN_ASYNC = 1 << 0,
+ /* Start executing all pending queued rules wait till completion */
+ MLX5DR_SEND_QUEUE_ACTION_DRAIN_SYNC = 1 << 1,
};
struct mlx5dr_context_attr {
diff --git a/drivers/net/mlx5/hws/mlx5dr_pat_arg.c
b/drivers/net/mlx5/hws/mlx5dr_pat_arg.c
index df451f1ae0..152025d302 100644
--- a/drivers/net/mlx5/hws/mlx5dr_pat_arg.c
+++ b/drivers/net/mlx5/hws/mlx5dr_pat_arg.c
@@ -306,27 +306,6 @@ void mlx5dr_arg_decapl3_write(struct mlx5dr_send_engine
*queue,
mlx5dr_send_engine_post_end(&ctrl, &send_attr);
}
-static int
-mlx5dr_arg_poll_for_comp(struct mlx5dr_context *ctx, uint16_t queue_id)
-{
- struct rte_flow_op_result comp[1];
- int ret;
-
- while (true) {
- ret = mlx5dr_send_queue_poll(ctx, queue_id, comp, 1);
- if (ret) {
- if (ret < 0) {
- DR_LOG(ERR, "Failed mlx5dr_send_queue_poll");
- } else if (comp[0].status == RTE_FLOW_OP_ERROR) {
- DR_LOG(ERR, "Got comp with error");
- rte_errno = ENOENT;
- }
- break;
- }
- }
- return (ret == 1 ? 0 : ret);
-}
-
void mlx5dr_arg_write(struct mlx5dr_send_engine *queue,
void *comp_data,
uint32_t arg_idx,
@@ -388,9 +367,11 @@ int mlx5dr_arg_write_inline_arg_data(struct mlx5dr_context
*ctx,
mlx5dr_send_engine_flush_queue(queue);
/* Poll for completion */
- ret = mlx5dr_arg_poll_for_comp(ctx, ctx->queues - 1);
+ ret = mlx5dr_send_queue_action(ctx, ctx->queues - 1,
+ MLX5DR_SEND_QUEUE_ACTION_DRAIN_SYNC);
+
if (ret)
- DR_LOG(ERR, "Failed to get completions for shared action");
+ DR_LOG(ERR, "Failed to drain arg queue");
pthread_spin_unlock(&ctx->ctrl_lock);
diff --git a/drivers/net/mlx5/hws/mlx5dr_send.c
b/drivers/net/mlx5/hws/mlx5dr_send.c
index 5c8bbe6fc6..a507e5f626 100644
--- a/drivers/net/mlx5/hws/mlx5dr_send.c
+++ b/drivers/net/mlx5/hws/mlx5dr_send.c
@@ -830,18 +830,30 @@ int mlx5dr_send_queue_action(struct mlx5dr_context *ctx,
{
struct mlx5dr_send_ring_sq *send_sq;
struct mlx5dr_send_engine *queue;
+ bool wait_comp = false;
+ int64_t polled = 0;
queue = &ctx->send_queue[queue_id];
send_sq = &queue->send_ring->send_sq;
- if (actions == MLX5DR_SEND_QUEUE_ACTION_DRAIN) {
+ switch (actions) {
+ case MLX5DR_SEND_QUEUE_ACTION_DRAIN_SYNC:
+ wait_comp = true;
+ /* FALLTHROUGH */
+ case MLX5DR_SEND_QUEUE_ACTION_DRAIN_ASYNC:
if (send_sq->head_dep_idx != send_sq->tail_dep_idx)
/* Send dependent WQEs to drain the queue */
mlx5dr_send_all_dep_wqe(queue);
else
/* Signal on the last posted WQE */
mlx5dr_send_engine_flush_queue(queue);
- } else {
+
+ /* Poll queue until empty */
+ while (wait_comp && !mlx5dr_send_engine_empty(queue))
+ mlx5dr_send_engine_poll_cqs(queue, NULL, &polled, 0);
+
+ break;
+ default:
rte_errno = -EINVAL;
return rte_errno;
}
diff --git a/drivers/net/mlx5/hws/mlx5dr_send.h
b/drivers/net/mlx5/hws/mlx5dr_send.h
index 8d4769495d..fcddcc6366 100644
--- a/drivers
