Author: hselasky
Date: Wed Jul 18 10:03:30 2018
New Revision: 336450
URL: https://svnweb.freebsd.org/changeset/base/336450
Log:
Do not inline transmit headers and use HW VLAN tagging if supported by
mlx5en(4).
Query the minimal inline mode supported by the card.
When creating a send queue, cache the queried mode and optimize the transmit
if no inlining is required. In this case, we can avoid touching the headers
cache line and avoid dirtying several more lines by copying headers into
the send WQEs. Also, if no inline headers are used, hardware assists in
the VLAN tag framing.
Submitted by: kib@, slavash@
MFC after: 1 week
Sponsored by: Mellanox Technologies
Modified:
head/sys/dev/mlx5/device.h
head/sys/dev/mlx5/mlx5_core/mlx5_vport.c
head/sys/dev/mlx5/mlx5_en/en.h
head/sys/dev/mlx5/mlx5_en/mlx5_en_main.c
head/sys/dev/mlx5/mlx5_en/mlx5_en_rl.c
head/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c
head/sys/dev/mlx5/qp.h
head/sys/dev/mlx5/vport.h
Modified: head/sys/dev/mlx5/device.h
==============================================================================
--- head/sys/dev/mlx5/device.h Wed Jul 18 09:54:32 2018 (r336449)
+++ head/sys/dev/mlx5/device.h Wed Jul 18 10:03:30 2018 (r336450)
@@ -1093,6 +1093,13 @@ enum {
MLX5_CMD_HCA_CAP_MIN_WQE_INLINE_MODE_NOT_REQUIRED = 0x2
};
+enum mlx5_inline_modes {
+ MLX5_INLINE_MODE_NONE,
+ MLX5_INLINE_MODE_L2,
+ MLX5_INLINE_MODE_IP,
+ MLX5_INLINE_MODE_TCP_UDP,
+};
+
enum {
MLX5_QUERY_VPORT_STATE_OUT_STATE_FOLLOW = 0x2,
};
Modified: head/sys/dev/mlx5/mlx5_core/mlx5_vport.c
==============================================================================
--- head/sys/dev/mlx5/mlx5_core/mlx5_vport.c Wed Jul 18 09:54:32 2018
(r336449)
+++ head/sys/dev/mlx5/mlx5_core/mlx5_vport.c Wed Jul 18 10:03:30 2018
(r336450)
@@ -208,6 +208,58 @@ int mlx5_vport_query_out_of_rx_buffer(struct mlx5_core
return err;
}
+int mlx5_query_nic_vport_min_inline(struct mlx5_core_dev *mdev,
+ u16 vport, u8 *min_inline)
+{
+ u32 out[MLX5_ST_SZ_DW(query_nic_vport_context_out)] = {0};
+ int err;
+
+ err = mlx5_query_nic_vport_context(mdev, vport, out, sizeof(out));
+ if (!err)
+ *min_inline = MLX5_GET(query_nic_vport_context_out, out,
+ nic_vport_context.min_wqe_inline_mode);
+ return err;
+}
+EXPORT_SYMBOL_GPL(mlx5_query_nic_vport_min_inline);
+
+void mlx5_query_min_inline(struct mlx5_core_dev *mdev,
+ u8 *min_inline_mode)
+{
+ switch (MLX5_CAP_ETH(mdev, wqe_inline_mode)) {
+ case MLX5_CAP_INLINE_MODE_L2:
+ *min_inline_mode = MLX5_INLINE_MODE_L2;
+ break;
+ case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT:
+ mlx5_query_nic_vport_min_inline(mdev, 0, min_inline_mode);
+ break;
+ case MLX5_CAP_INLINE_MODE_NOT_REQUIRED:
+ *min_inline_mode = MLX5_INLINE_MODE_NONE;
+ break;
+ }
+}
+EXPORT_SYMBOL_GPL(mlx5_query_min_inline);
+
+int mlx5_modify_nic_vport_min_inline(struct mlx5_core_dev *mdev,
+ u16 vport, u8 min_inline)
+{
+ u32 in[MLX5_ST_SZ_DW(modify_nic_vport_context_in)] = {0};
+ int inlen = MLX5_ST_SZ_BYTES(modify_nic_vport_context_in);
+ void *nic_vport_ctx;
+
+ MLX5_SET(modify_nic_vport_context_in, in,
+ field_select.min_wqe_inline_mode, 1);
+ MLX5_SET(modify_nic_vport_context_in, in, vport_number, vport);
+ MLX5_SET(modify_nic_vport_context_in, in, other_vport, 1);
+
+ nic_vport_ctx = MLX5_ADDR_OF(modify_nic_vport_context_in,
+ in, nic_vport_context);
+ MLX5_SET(nic_vport_context, nic_vport_ctx,
+ min_wqe_inline_mode, min_inline);
+
+ return mlx5_modify_nic_vport_context(mdev, in, inlen);
+}
+EXPORT_SYMBOL_GPL(mlx5_modify_nic_vport_min_inline);
+
int mlx5_query_nic_vport_mac_address(struct mlx5_core_dev *mdev,
u16 vport, u8 *addr)
{
Modified: head/sys/dev/mlx5/mlx5_en/en.h
==============================================================================
--- head/sys/dev/mlx5/mlx5_en/en.h Wed Jul 18 09:54:32 2018
(r336449)
+++ head/sys/dev/mlx5/mlx5_en/en.h Wed Jul 18 10:03:30 2018
(r336450)
@@ -451,6 +451,8 @@ struct mlx5e_params {
u32 rx_pauseframe_control __aligned(4);
u32 tx_priority_flow_control __aligned(4);
u32 rx_priority_flow_control __aligned(4);
+ u16 tx_max_inline;
+ u8 tx_min_inline_mode;
};
#define MLX5E_PARAMS(m)
\
@@ -613,6 +615,9 @@ struct mlx5e_sq {
u32 sqn;
u32 bf_buf_size;
u32 mkey_be;
+ u16 max_inline;
+ u8 min_inline_mode;
+ u8 vlan_inline_cap;
/* control path */
struct mlx5_wq_ctrl wq_ctrl;
@@ -933,5 +938,6 @@ void mlx5e_drain_sq(struct mlx5e_sq *);
void mlx5e_modify_tx_dma(struct mlx5e_priv *priv, uint8_t value);
void mlx5e_modify_rx_dma(struct mlx5e_priv *priv, uint8_t value);
void mlx5e_resume_sq(struct mlx5e_sq *sq);
+u8 mlx5e_params_calculate_tx_min_inline(struct mlx5_core_dev *mdev);
#endif /* _MLX5_EN_H_ */
Modified: head/sys/dev/mlx5/mlx5_en/mlx5_en_main.c
==============================================================================
--- head/sys/dev/mlx5/mlx5_en/mlx5_en_main.c Wed Jul 18 09:54:32 2018
(r336449)
+++ head/sys/dev/mlx5/mlx5_en/mlx5_en_main.c Wed Jul 18 10:03:30 2018
(r336450)
@@ -1174,6 +1174,9 @@ mlx5e_create_sq(struct mlx5e_channel *c,
sq->ifp = priv->ifp;
sq->priv = priv;
sq->tc = tc;
+ sq->max_inline = priv->params.tx_max_inline;
+ sq->min_inline_mode = priv->params.tx_min_inline_mode;
+ sq->vlan_inline_cap = MLX5_CAP_ETH(mdev, wqe_vlan_insert);
/* check if we should allocate a second packet buffer */
if (priv->params_ethtool.tx_bufring_disable == 0) {
@@ -3021,6 +3024,16 @@ mlx5e_check_required_hca_cap(struct mlx5_core_dev *mde
return (0);
}
+static u16
+mlx5e_get_max_inline_cap(struct mlx5_core_dev *mdev)
+{
+ int bf_buf_size = (1 << MLX5_CAP_GEN(mdev, log_bf_reg_size)) / 2;
+
+ return bf_buf_size -
+ sizeof(struct mlx5e_tx_wqe) +
+ 2 /*sizeof(mlx5e_tx_wqe.inline_hdr_start)*/;
+}
+
static void
mlx5e_build_ifp_priv(struct mlx5_core_dev *mdev,
struct mlx5e_priv *priv,
@@ -3056,6 +3069,8 @@ mlx5e_build_ifp_priv(struct mlx5_core_dev *mdev,
priv->params.num_tc = 1;
priv->params.default_vlan_prio = 0;
priv->counter_set_id = -1;
+ priv->params.tx_max_inline = mlx5e_get_max_inline_cap(mdev);
+ mlx5_query_min_inline(mdev, &priv->params.tx_min_inline_mode);
/*
* hw lro is currently defaulted to off. when it won't anymore we
@@ -3314,6 +3329,20 @@ mlx5e_modify_rx_dma(struct mlx5e_priv *priv, uint8_t v
else
mlx5e_enable_rx_dma(priv->channel[i]);
}
+}
+
+u8
+mlx5e_params_calculate_tx_min_inline(struct mlx5_core_dev *mdev)
+{
+ u8 min_inline_mode;
+
+ min_inline_mode = MLX5_INLINE_MODE_L2;
+ mlx5_query_min_inline(mdev, &min_inline_mode);
+ if (min_inline_mode == MLX5_INLINE_MODE_NONE &&
+ !MLX5_CAP_ETH(mdev, wqe_vlan_insert))
+ min_inline_mode = MLX5_INLINE_MODE_L2;
+
+ return (min_inline_mode);
}
static void
Modified: head/sys/dev/mlx5/mlx5_en/mlx5_en_rl.c
==============================================================================
--- head/sys/dev/mlx5/mlx5_en/mlx5_en_rl.c Wed Jul 18 09:54:32 2018
(r336449)
+++ head/sys/dev/mlx5/mlx5_en/mlx5_en_rl.c Wed Jul 18 10:03:30 2018
(r336450)
@@ -137,6 +137,9 @@ mlx5e_rl_create_sq(struct mlx5e_priv *priv, struct mlx
sq->mkey_be = cpu_to_be32(priv->mr.key);
sq->ifp = priv->ifp;
sq->priv = priv;
+ sq->max_inline = priv->params.tx_max_inline;
+ sq->min_inline_mode = priv->params.tx_min_inline_mode;
+ sq->vlan_inline_cap = MLX5_CAP_ETH(mdev, wqe_vlan_insert);
return (0);
Modified: head/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c
==============================================================================
--- head/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c Wed Jul 18 09:54:32 2018
(r336449)
+++ head/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c Wed Jul 18 10:03:30 2018
(r336450)
@@ -156,7 +156,43 @@ mlx5e_select_queue(struct ifnet *ifp, struct mbuf *mb)
static inline u16
mlx5e_get_inline_hdr_size(struct mlx5e_sq *sq, struct mbuf *mb)
{
- return (MIN(MLX5E_MAX_TX_INLINE, mb->m_len));
+
+ switch(sq->min_inline_mode) {
+ case MLX5_INLINE_MODE_NONE:
+ /*
+ * When inline mode is NONE, we do not need to copy
+ * headers into WQEs, except when vlan tag framing is
+ * requested. Hardware might offload vlan tagging on
+ * transmit. This is a separate capability, which is
+ * known to be disabled on ConnectX-5 due to a hardware
+ * bug RM 931383. If vlan_inline_cap is not present and
+ * the packet has vlan tag, fall back to inlining.
+ */
+ if ((mb->m_flags & M_VLANTAG) != 0 &&
+ sq->vlan_inline_cap == 0)
+ break;
+ return (0);
+ case MLX5_INLINE_MODE_L2:
+ /*
+ * Due to hardware limitations, when trust mode is
+ * DSCP, the hardware may request MLX5_INLINE_MODE_L2
+ * while it really needs all L2 headers and the 4 first
+ * bytes of the IP header (which include the
+ * TOS/traffic-class).
+ *
+ * To avoid doing a firmware command for querying the
+ * trust state and parsing the mbuf for doing
+ * unnecessary checks (VLAN/eth_type) in the fast path,
+ * we are going for the worth case (22 Bytes) if
+ * the mb->m_pkthdr.len allows it.
+ */
+ if (mb->m_pkthdr.len > ETHER_HDR_LEN +
+ ETHER_VLAN_ENCAP_LEN + 4)
+ return (MIN(sq->max_inline, ETHER_HDR_LEN +
+ ETHER_VLAN_ENCAP_LEN + 4));
+ break;
+ }
+ return (MIN(sq->max_inline, mb->m_pkthdr.len));
}
static int
@@ -294,37 +330,47 @@ mlx5e_sq_xmit(struct mlx5e_sq *sq, struct mbuf **mbp)
sq->mbuf[pi].num_bytes = max_t (unsigned int,
mb->m_pkthdr.len, ETHER_MIN_LEN - ETHER_CRC_LEN);
}
- if (mb->m_flags & M_VLANTAG) {
- struct ether_vlan_header *eh =
- (struct ether_vlan_header *)wqe->eth.inline_hdr_start;
-
- /* Range checks */
- if (ihs > (MLX5E_MAX_TX_INLINE - ETHER_VLAN_ENCAP_LEN))
- ihs = (MLX5E_MAX_TX_INLINE - ETHER_VLAN_ENCAP_LEN);
- else if (ihs < ETHER_HDR_LEN) {
- err = EINVAL;
- goto tx_drop;
+ if (ihs == 0) {
+ if ((mb->m_flags & M_VLANTAG) != 0) {
+ wqe->eth.vlan_cmd = htons(0x8000); /* bit 0 CVLAN */
+ wqe->eth.vlan_hdr = htons(mb->m_pkthdr.ether_vtag);
+ } else {
+ wqe->eth.inline_hdr_sz = 0;
}
- m_copydata(mb, 0, ETHER_HDR_LEN, (caddr_t)eh);
- m_adj(mb, ETHER_HDR_LEN);
- /* Insert 4 bytes VLAN tag into data stream */
- eh->evl_proto = eh->evl_encap_proto;
- eh->evl_encap_proto = htons(ETHERTYPE_VLAN);
- eh->evl_tag = htons(mb->m_pkthdr.ether_vtag);
- /* Copy rest of header data, if any */
- m_copydata(mb, 0, ihs - ETHER_HDR_LEN, (caddr_t)(eh + 1));
- m_adj(mb, ihs - ETHER_HDR_LEN);
- /* Extend header by 4 bytes */
- ihs += ETHER_VLAN_ENCAP_LEN;
} else {
- m_copydata(mb, 0, ihs, wqe->eth.inline_hdr_start);
- m_adj(mb, ihs);
+ if ((mb->m_flags & M_VLANTAG) != 0) {
+ struct ether_vlan_header *eh = (struct ether_vlan_header
+ *)wqe->eth.inline_hdr_start;
+
+ /* Range checks */
+ if (ihs > (MLX5E_MAX_TX_INLINE - ETHER_VLAN_ENCAP_LEN))
+ ihs = (MLX5E_MAX_TX_INLINE -
+ ETHER_VLAN_ENCAP_LEN);
+ else if (ihs < ETHER_HDR_LEN) {
+ err = EINVAL;
+ goto tx_drop;
+ }
+ m_copydata(mb, 0, ETHER_HDR_LEN, (caddr_t)eh);
+ m_adj(mb, ETHER_HDR_LEN);
+ /* Insert 4 bytes VLAN tag into data stream */
+ eh->evl_proto = eh->evl_encap_proto;
+ eh->evl_encap_proto = htons(ETHERTYPE_VLAN);
+ eh->evl_tag = htons(mb->m_pkthdr.ether_vtag);
+ /* Copy rest of header data, if any */
+ m_copydata(mb, 0, ihs - ETHER_HDR_LEN, (caddr_t)(eh +
+ 1));
+ m_adj(mb, ihs - ETHER_HDR_LEN);
+ /* Extend header by 4 bytes */
+ ihs += ETHER_VLAN_ENCAP_LEN;
+ } else {
+ m_copydata(mb, 0, ihs, wqe->eth.inline_hdr_start);
+ m_adj(mb, ihs);
+ }
+ wqe->eth.inline_hdr_sz = cpu_to_be16(ihs);
}
- wqe->eth.inline_hdr_sz = cpu_to_be16(ihs);
-
ds_cnt = sizeof(*wqe) / MLX5_SEND_WQE_DS;
- if (likely(ihs > sizeof(wqe->eth.inline_hdr_start))) {
+ if (ihs > sizeof(wqe->eth.inline_hdr_start)) {
ds_cnt += DIV_ROUND_UP(ihs - sizeof(wqe->eth.inline_hdr_start),
MLX5_SEND_WQE_DS);
}
Modified: head/sys/dev/mlx5/qp.h
==============================================================================
--- head/sys/dev/mlx5/qp.h Wed Jul 18 09:54:32 2018 (r336449)
+++ head/sys/dev/mlx5/qp.h Wed Jul 18 10:03:30 2018 (r336450)
@@ -240,8 +240,16 @@ struct mlx5_wqe_eth_seg {
u8 swp_flags;
__be16 mss;
__be32 rsvd2;
- __be16 inline_hdr_sz;
- u8 inline_hdr_start[2];
+ union {
+ struct {
+ __be16 inline_hdr_sz;
+ u8 inline_hdr_start[2];
+ };
+ struct {
+ __be16 vlan_cmd;
+ __be16 vlan_hdr;
+ };
+ };
};
struct mlx5_wqe_xrc_seg {
Modified: head/sys/dev/mlx5/vport.h
==============================================================================
--- head/sys/dev/mlx5/vport.h Wed Jul 18 09:54:32 2018 (r336449)
+++ head/sys/dev/mlx5/vport.h Wed Jul 18 10:03:30 2018 (r336450)
@@ -29,6 +29,13 @@
#define __MLX5_VPORT_H__
#include <dev/mlx5/driver.h>
+
+enum {
+ MLX5_CAP_INLINE_MODE_L2,
+ MLX5_CAP_INLINE_MODE_VPORT_CONTEXT,
+ MLX5_CAP_INLINE_MODE_NOT_REQUIRED,
+};
+
int mlx5_vport_alloc_q_counter(struct mlx5_core_dev *mdev, int client_id,
u16 *counter_set_id);
int mlx5_vport_dealloc_q_counter(struct mlx5_core_dev *mdev, int client_id,
@@ -79,6 +86,11 @@ int mlx5_modify_nic_vport_mac_address(struct mlx5_core
u16 vport, u8 mac[ETH_ALEN]);
int mlx5_set_nic_vport_current_mac(struct mlx5_core_dev *mdev, int vport,
bool other_vport, u8 *addr);
+int mlx5_query_nic_vport_min_inline(struct mlx5_core_dev *mdev,
+ u16 vport, u8 *min_inline);
+void mlx5_query_min_inline(struct mlx5_core_dev *mdev, u8 *min_inline);
+int mlx5_modify_nic_vport_min_inline(struct mlx5_core_dev *mdev,
+ u16 vport, u8 min_inline);
int mlx5_modify_nic_vport_port_guid(struct mlx5_core_dev *mdev,
u32 vport, u64 port_guid);
int mlx5_modify_nic_vport_node_guid(struct mlx5_core_dev *mdev,
_______________________________________________
svn-src-head@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-head
To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org"
