Author: hselasky Date: Mon Dec 28 18:50:18 2015 New Revision: 292838 URL: https://svnweb.freebsd.org/changeset/base/292838
Log: Add support for CQE zipping. CQE zipping reduces PCI overhead by coalescing and zipping multiple CQEs into a single merged CQE. The feature is enabled by default and can be disabled by a sysctl. Implementing this feature mlx5_cqwq_pop() has been separated from mlx5e_get_cqe(). MFC after: 1 week Submitted by: Mark Bloch <ma...@mellanox.com> Differential Revision: https://reviews.freebsd.org/D4598 Sponsored by: Mellanox Technologies Modified: head/sys/dev/mlx5/device.h head/sys/dev/mlx5/mlx5_en/en.h head/sys/dev/mlx5/mlx5_en/mlx5_en_ethtool.c head/sys/dev/mlx5/mlx5_en/mlx5_en_main.c head/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c head/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c head/sys/dev/mlx5/mlx5_en/mlx5_en_txrx.c Modified: head/sys/dev/mlx5/device.h ============================================================================== --- head/sys/dev/mlx5/device.h Mon Dec 28 18:36:00 2015 (r292837) +++ head/sys/dev/mlx5/device.h Mon Dec 28 18:50:18 2015 (r292838) @@ -1042,6 +1042,7 @@ enum { MLX5_ESW_VPORT_ADMIN_STATE_UP = 0x1, MLX5_ESW_VPORT_ADMIN_STATE_AUTO = 0x2, }; + /* MLX5 DEV CAPs */ /* TODO: EAT.ME */ @@ -1219,4 +1220,36 @@ struct mlx5_ifc_mcia_reg_bits { }; #define MLX5_CMD_OP_QUERY_EEPROM 0x93c + +struct mlx5_mini_cqe8 { + union { + u32 rx_hash_result; + u32 checksum; + struct { + u16 wqe_counter; + u8 s_wqe_opcode; + u8 reserved; + } s_wqe_info; + }; + u32 byte_cnt; +}; + +enum { + MLX5_NO_INLINE_DATA, + MLX5_INLINE_DATA32_SEG, + MLX5_INLINE_DATA64_SEG, + MLX5_COMPRESSED, +}; + +enum mlx5_exp_cqe_zip_recv_type { + MLX5_CQE_FORMAT_HASH, + MLX5_CQE_FORMAT_CSUM, +}; + +#define MLX5E_CQE_FORMAT_MASK 0xc +static inline int mlx5_get_cqe_format(const struct mlx5_cqe64 *cqe) +{ + return (cqe->op_own & MLX5E_CQE_FORMAT_MASK) >> 2; +} + #endif /* MLX5_DEVICE_H */ Modified: head/sys/dev/mlx5/mlx5_en/en.h ============================================================================== --- head/sys/dev/mlx5/mlx5_en/en.h Mon Dec 28 18:36:00 2015 (r292837) +++ head/sys/dev/mlx5/mlx5_en/en.h Mon Dec 28 18:50:18 2015 (r292838) @@ -370,6 +370,7 @@ struct mlx5e_params { u16 tx_cq_moderation_pkts; u16 min_rx_wqes; bool hw_lro_en; + bool cqe_zipping_en; u32 lro_wqe_sz; u16 rx_hash_log_tbl_sz; }; @@ -390,7 +391,8 @@ struct mlx5e_params { m(+1, u64 tx_coalesce_usecs, "tx_coalesce_usecs", "Limit in usec for joining tx packets") \ m(+1, u64 tx_coalesce_pkts, "tx_coalesce_pkts", "Maximum number of tx packets to join") \ m(+1, u64 tx_coalesce_mode, "tx_coalesce_mode", "0: EQE mode 1: CQE mode") \ - m(+1, u64 hw_lro, "hw_lro", "set to enable hw_lro") + m(+1, u64 hw_lro, "hw_lro", "set to enable hw_lro") \ + m(+1, u64 cqe_zipping, "cqe_zipping", "0 : CQE zipping disabled") #define MLX5E_PARAMS_NUM (0 MLX5E_PARAMS(MLX5E_STATS_COUNT)) Modified: head/sys/dev/mlx5/mlx5_en/mlx5_en_ethtool.c ============================================================================== --- head/sys/dev/mlx5/mlx5_en/mlx5_en_ethtool.c Mon Dec 28 18:36:00 2015 (r292837) +++ head/sys/dev/mlx5/mlx5_en/mlx5_en_ethtool.c Mon Dec 28 18:50:18 2015 (r292838) @@ -204,6 +204,18 @@ mlx5e_ethtool_handler(SYSCTL_HANDLER_ARG priv->params.hw_lro_en = false; } + if (&priv->params_ethtool.arg[arg2] == + &priv->params_ethtool.cqe_zipping) { + if (priv->params_ethtool.cqe_zipping && + MLX5_CAP_GEN(priv->mdev, cqe_compression)) { + priv->params.cqe_zipping_en = true; + priv->params_ethtool.cqe_zipping = 1; + } else { + priv->params.cqe_zipping_en = false; + priv->params_ethtool.cqe_zipping = 0; + } + } + if (was_opened) mlx5e_open_locked(priv->ifp); done: @@ -472,6 +484,7 @@ mlx5e_create_ethtool(struct mlx5e_priv * priv->params_ethtool.tx_coalesce_usecs = priv->params.tx_cq_moderation_usec; priv->params_ethtool.tx_coalesce_pkts = priv->params.tx_cq_moderation_pkts; priv->params_ethtool.hw_lro = priv->params.hw_lro_en; + priv->params_ethtool.cqe_zipping = priv->params.cqe_zipping_en; /* create root node */ node = SYSCTL_ADD_NODE(&priv->sysctl_ctx, Modified: head/sys/dev/mlx5/mlx5_en/mlx5_en_main.c ============================================================================== --- head/sys/dev/mlx5/mlx5_en/mlx5_en_main.c Mon Dec 28 18:36:00 2015 (r292837) +++ head/sys/dev/mlx5/mlx5_en/mlx5_en_main.c Mon Dec 28 18:50:18 2015 (r292838) @@ -1604,6 +1604,16 @@ mlx5e_build_rx_cq_param(struct mlx5e_pri { void *cqc = param->cqc; + + /* + * TODO The sysctl to control on/off is a bool value for now, which means + * we only support CSUM, once HASH is implemnted we'll need to address that. + */ + if (priv->params.cqe_zipping_en) { + MLX5_SET(cqc, cqc, mini_cqe_res_format, MLX5_CQE_FORMAT_CSUM); + MLX5_SET(cqc, cqc, cqe_compression_en, 1); + } + MLX5_SET(cqc, cqc, log_cq_size, priv->params.log_rq_size); MLX5_SET(cqc, cqc, cq_period, priv->params.rx_cq_moderation_usec); MLX5_SET(cqc, cqc, cq_max_count, priv->params.rx_cq_moderation_pkts); @@ -2571,6 +2581,8 @@ mlx5e_build_ifp_priv(struct mlx5_core_de priv->params.hw_lro_en = false; priv->params.lro_wqe_sz = MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ; + priv->params.cqe_zipping_en = !!MLX5_CAP_GEN(mdev, cqe_compression); + priv->mdev = mdev; priv->params.num_channels = num_comp_vectors; priv->order_base_2_num_channels = order_base_2(num_comp_vectors); Modified: head/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c ============================================================================== --- head/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c Mon Dec 28 18:36:00 2015 (r292837) +++ head/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c Mon Dec 28 18:50:18 2015 (r292838) @@ -248,6 +248,69 @@ mlx5e_build_rx_mbuf(struct mlx5_cqe64 *c } } +static inline void +mlx5e_read_cqe_slot(struct mlx5e_cq *cq, u32 cc, void *data) +{ + memcpy(data, mlx5_cqwq_get_wqe(&cq->wq, (cc & cq->wq.sz_m1)), + sizeof(struct mlx5_cqe64)); +} + +static inline void +mlx5e_write_cqe_slot(struct mlx5e_cq *cq, u32 cc, void *data) +{ + memcpy(mlx5_cqwq_get_wqe(&cq->wq, cc & cq->wq.sz_m1), + data, sizeof(struct mlx5_cqe64)); +} + +static inline void +mlx5e_decompress_cqe(struct mlx5e_cq *cq, struct mlx5_cqe64 *title, + struct mlx5_mini_cqe8 *mini, + u16 wqe_counter, int i) +{ + title->byte_cnt = mini->byte_cnt; + title->wqe_counter = cpu_to_be16((wqe_counter + i) & cq->wq.sz_m1); + title->check_sum = mini->checksum; + title->op_own = (title->op_own & 0xf0) | + (((cq->wq.cc + i) >> cq->wq.log_sz) & 1); +} + +#define MLX5E_MINI_ARRAY_SZ 8 +/* Make sure structs are not packet differently */ +CTASSERT(sizeof(struct mlx5_cqe64) == + sizeof(struct mlx5_mini_cqe8) * MLX5E_MINI_ARRAY_SZ); +static void +mlx5e_decompress_cqes(struct mlx5e_cq *cq) +{ + struct mlx5_mini_cqe8 mini_array[MLX5E_MINI_ARRAY_SZ]; + struct mlx5_cqe64 title; + u32 cqe_count; + u32 i = 0; + u16 title_wqe_counter; + + mlx5e_read_cqe_slot(cq, cq->wq.cc, &title); + title_wqe_counter = be16_to_cpu(title.wqe_counter); + cqe_count = be32_to_cpu(title.byte_cnt); + + /* Make sure we won't overflow */ + KASSERT(cqe_count <= cq->wq.sz_m1, + ("%s: cqe_count %u > cq->wq.sz_m1 %u", __func__, + cqe_count, cq->wq.sz_m1)); + + mlx5e_read_cqe_slot(cq, cq->wq.cc + 1, mini_array); + while (true) { + mlx5e_decompress_cqe(cq, &title, + &mini_array[i % MLX5E_MINI_ARRAY_SZ], + title_wqe_counter, i); + mlx5e_write_cqe_slot(cq, cq->wq.cc + i, &title); + i++; + + if (i == cqe_count) + break; + if (i % MLX5E_MINI_ARRAY_SZ == 0) + mlx5e_read_cqe_slot(cq, cq->wq.cc + i, mini_array); + } +} + static int mlx5e_poll_rx_cq(struct mlx5e_rq *rq, int budget) { @@ -268,6 +331,11 @@ mlx5e_poll_rx_cq(struct mlx5e_rq *rq, in if (!cqe) break; + if (mlx5_get_cqe_format(cqe) == MLX5_COMPRESSED) + mlx5e_decompress_cqes(&rq->cq); + + mlx5_cqwq_pop(&rq->cq.wq); + wqe_counter_be = cqe->wqe_counter; wqe_counter = be16_to_cpu(wqe_counter_be); wqe = mlx5_wq_ll_get_wqe(&rq->wq, wqe_counter); Modified: head/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c ============================================================================== --- head/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c Mon Dec 28 18:36:00 2015 (r292837) +++ head/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c Mon Dec 28 18:50:18 2015 (r292838) @@ -383,6 +383,8 @@ mlx5e_poll_tx_cq(struct mlx5e_sq *sq, in if (!cqe) break; + mlx5_cqwq_pop(&sq->cq.wq); + ci = sqcc & sq->wq.sz_m1; mb = sq->mbuf[ci].mbuf; sq->mbuf[ci].mbuf = NULL; /* Safety clear */ Modified: head/sys/dev/mlx5/mlx5_en/mlx5_en_txrx.c ============================================================================== --- head/sys/dev/mlx5/mlx5_en/mlx5_en_txrx.c Mon Dec 28 18:36:00 2015 (r292837) +++ head/sys/dev/mlx5/mlx5_en/mlx5_en_txrx.c Mon Dec 28 18:50:18 2015 (r292838) @@ -37,8 +37,6 @@ mlx5e_get_cqe(struct mlx5e_cq *cq) if ((cqe->op_own ^ mlx5_cqwq_get_wrap_cnt(&cq->wq)) & MLX5_CQE_OWNER_MASK) return (NULL); - mlx5_cqwq_pop(&cq->wq); - /* ensure cqe content is read after cqe ownership bit */ rmb(); _______________________________________________ svn-src-head@freebsd.org mailing list https://lists.freebsd.org/mailman/listinfo/svn-src-head To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org"
