From: Viacheslav Ovsiienko
> This patch handles the NIC supported timestamp formats via DevX.
> There are two different timestamp format can be provided potentially.
> The free-running is some opaque values captured from internal clock counter
> clocked by some independent oscillator. The free-running frequency is not pre-
> defined and should be queried from the NIC.
> The real-time timestamps are expressed in nanosecond, captured from the
> dedicated UTC counter, that can be adjusted on the fly and synchronized with
> some external master clock.
>
> Depending on the version and configuration the hardware might support either
> FR or RT timestamps, or both in the same time on per queue basis. The commit
> provides the querying information about the supported timestamp formats and
> provides the means to configure ones at queue creation.
>
> Fixes: e2b4925ef7c1 ("net/mlx5: support Direct Rules E-Switch")
> Cc: sta...@dpdk.org
>
> Signed-off-by: Viacheslav Ovsiienko <viachesl...@nvidia.com>
> ---
> drivers/common/mlx5/mlx5_devx_cmds.c | 35 ++++++++++++++++++++++--
> drivers/common/mlx5/mlx5_devx_cmds.h | 7 +++++
> drivers/common/mlx5/mlx5_prm.h | 41 +++++++++++++++++++++++++---
> 3 files changed, 76 insertions(+), 7 deletions(-)
>
> diff --git a/drivers/common/mlx5/mlx5_devx_cmds.c
> b/drivers/common/mlx5/mlx5_devx_cmds.c
> index 0060c37fc0..bf9921b78a 100644
> --- a/drivers/common/mlx5/mlx5_devx_cmds.c
> +++ b/drivers/common/mlx5/mlx5_devx_cmds.c
> @@ -715,6 +715,9 @@ mlx5_devx_cmd_query_hca_attr(void *ctx,
> device_frequency_khz);
> attr->scatter_fcs_w_decap_disable =
> MLX5_GET(cmd_hca_cap, hcattr,
> scatter_fcs_w_decap_disable);
> + attr->roce = MLX5_GET(cmd_hca_cap, hcattr, roce);
> + attr->rq_ts_format = MLX5_GET(cmd_hca_cap, hcattr, rq_ts_format);
> + attr->sq_ts_format = MLX5_GET(cmd_hca_cap, hcattr, sq_ts_format);
> attr->regex = MLX5_GET(cmd_hca_cap, hcattr, regexp);
> attr->regexp_num_of_engines = MLX5_GET(cmd_hca_cap, hcattr,
> regexp_num_of_engines);
> @@ -859,9 +862,32 @@ mlx5_devx_cmd_query_hca_attr(void *ctx,
> attr->rss_ind_tbl_cap = MLX5_GET
>
> (per_protocol_networking_offload_caps,
> hcattr, rss_ind_tbl_cap);
> - if (attr->wqe_inline_mode !=
> MLX5_CAP_INLINE_MODE_VPORT_CONTEXT)
> - return 0;
> - if (attr->eth_virt) {
> + /* Query HCA attribute for ROCE. */
> + if (attr->roce) {
> + memset(in, 0, sizeof(in));
> + memset(out, 0, sizeof(out));
> + MLX5_SET(query_hca_cap_in, in, opcode,
> + MLX5_CMD_OP_QUERY_HCA_CAP);
> + MLX5_SET(query_hca_cap_in, in, op_mod,
> + MLX5_GET_HCA_CAP_OP_MOD_ROCE |
> + MLX5_HCA_CAP_OPMOD_GET_CUR);
> + rc = mlx5_glue->devx_general_cmd(ctx, in, sizeof(in),
> + out, sizeof(out));
> + if (rc)
> + goto error;
> + status = MLX5_GET(query_hca_cap_out, out, status);
I think, like other DevX commands, status and syndrome are not filled in error
case...
> + syndrome = MLX5_GET(query_hca_cap_out, out, syndrome);
> + if (status) {
> + DRV_LOG(DEBUG,
> + "Failed to query devx HCA ROCE capabilities, "
> + "status %x, syndrome = %x", status, syndrome);
> + return -1;
> + }
> + hcattr = MLX5_ADDR_OF(query_hca_cap_out, out, capability);
> + attr->qp_ts_format = MLX5_GET(roce_caps, hcattr,
> qp_ts_format);
> + }
> + if (attr->eth_virt &&
> + attr->wqe_inline_mode ==
> MLX5_CAP_INLINE_MODE_VPORT_CONTEXT) {
> rc = mlx5_devx_cmd_query_nic_vport_context(ctx, 0, attr);
> if (rc) {
> attr->eth_virt = 0;
> @@ -1004,6 +1030,7 @@ mlx5_devx_cmd_create_rq(void *ctx,
> MLX5_SET(rqc, rq_ctx, cqn, rq_attr->cqn);
> MLX5_SET(rqc, rq_ctx, counter_set_id, rq_attr->counter_set_id);
> MLX5_SET(rqc, rq_ctx, rmpn, rq_attr->rmpn);
> + MLX5_SET(sqc, rq_ctx, ts_format, rq_attr->ts_format);
> wq_ctx = MLX5_ADDR_OF(rqc, rq_ctx, wq);
> wq_attr = &rq_attr->wq_attr;
> devx_cmd_fill_wq_data(wq_ctx, wq_attr); @@ -1376,6 +1403,7 @@
> mlx5_devx_cmd_create_sq(void *ctx,
> sq_attr->packet_pacing_rate_limit_index);
> MLX5_SET(sqc, sq_ctx, tis_lst_sz, sq_attr->tis_lst_sz);
> MLX5_SET(sqc, sq_ctx, tis_num_0, sq_attr->tis_num);
> + MLX5_SET(sqc, sq_ctx, ts_format, sq_attr->ts_format);
> wq_ctx = MLX5_ADDR_OF(sqc, sq_ctx, wq);
> wq_attr = &sq_attr->wq_attr;
> devx_cmd_fill_wq_data(wq_ctx, wq_attr); @@ -1825,6 +1853,7 @@
> mlx5_devx_cmd_create_qp(void *ctx,
> MLX5_SET(create_qp_in, in, opcode, MLX5_CMD_OP_CREATE_QP);
> MLX5_SET(qpc, qpc, st, MLX5_QP_ST_RC);
> MLX5_SET(qpc, qpc, pd, attr->pd);
> + MLX5_SET(qpc, qpc, ts_format, attr->ts_format);
> if (attr->uar_index) {
> MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
> MLX5_SET(qpc, qpc, uar_page, attr->uar_index); diff --git
> a/drivers/common/mlx5/mlx5_devx_cmds.h
> b/drivers/common/mlx5/mlx5_devx_cmds.h
> index bc66d28e83..2826c0b2c6 100644
> --- a/drivers/common/mlx5/mlx5_devx_cmds.h
> +++ b/drivers/common/mlx5/mlx5_devx_cmds.h
> @@ -121,6 +121,10 @@ struct mlx5_hca_attr {
> uint32_t dev_freq_khz; /* Timestamp counter frequency, kHz. */
> uint32_t scatter_fcs_w_decap_disable:1;
> uint32_t flow_hit_aso:1; /* General obj type FLOW_HIT_ASO
> supported. */
> + uint32_t roce:1;
> + uint32_t rq_ts_format:2;
> + uint32_t sq_ts_format:2;
> + uint32_t qp_ts_format:2;
> uint32_t regex:1;
> uint32_t reg_c_preserve:1;
> uint32_t regexp_num_of_engines;
> @@ -188,6 +192,7 @@ struct mlx5_devx_create_rq_attr {
> uint32_t state:4;
> uint32_t flush_in_error_en:1;
> uint32_t hairpin:1;
> + uint32_t ts_format:2;
> uint32_t user_index:24;
> uint32_t cqn:24;
> uint32_t counter_set_id:8;
> @@ -271,6 +276,7 @@ struct mlx5_devx_create_sq_attr {
> uint32_t hairpin:1;
> uint32_t non_wire:1;
> uint32_t static_sq_wq:1;
> + uint32_t ts_format:2;
> uint32_t user_index:24;
> uint32_t cqn:24;
> uint32_t packet_pacing_rate_limit_index:16;
> @@ -354,6 +360,7 @@ struct mlx5_devx_qp_attr {
> uint32_t rq_size:17; /* Must be power of 2. */
> uint32_t log_rq_stride:3;
> uint32_t sq_size:17; /* Must be power of 2. */
> + uint32_t ts_format:2;
> uint32_t dbr_umem_valid:1;
> uint32_t dbr_umem_id;
> uint64_t dbr_address;
> diff --git a/drivers/common/mlx5/mlx5_prm.h
> b/drivers/common/mlx5/mlx5_prm.h index 01a039f1f7..b0120e05d7 100644
> --- a/drivers/common/mlx5/mlx5_prm.h
> +++ b/drivers/common/mlx5/mlx5_prm.h
> @@ -1116,6 +1116,7 @@ enum {
> MLX5_GET_HCA_CAP_OP_MOD_GENERAL_DEVICE = 0x0 << 1,
> MLX5_GET_HCA_CAP_OP_MOD_ETHERNET_OFFLOAD_CAPS = 0x1 <<
> 1,
> MLX5_GET_HCA_CAP_OP_MOD_QOS_CAP = 0xc << 1,
> + MLX5_GET_HCA_CAP_OP_MOD_ROCE = 0x4 << 1,
> MLX5_GET_HCA_CAP_OP_MOD_NIC_FLOW_TABLE = 0x7 << 1,
> MLX5_GET_HCA_CAP_OP_MOD_VDPA_EMULATION = 0x13 << 1, };
> @@ -1153,6 +1154,20 @@ enum {
> MLX5_INLINE_MODE_INNER_TCP_UDP,
> };
>
> +/* The supported timestamp formats reported in HCA attributes. */ enum
> +{
> + MLX5_HCA_CAP_TIMESTAMP_FORMAT_FR = 0x0,
> + MLX5_HCA_CAP_TIMESTAMP_FORMAT_RT = 0x1,
> + MLX5_HCA_CAP_TIMESTAMP_FORMAT_FR_RT = 0x2, };
> +
> +/* The timestamp format attributes to configure queues (RQ/SQ/QP). */
> +enum {
> + MLX5_QPC_TIMESTAMP_FORMAT_FREE_RUNNING = 0x0,
> + MLX5_QPC_TIMESTAMP_FORMAT_DEFAULT = 0x1,
> + MLX5_QPC_TIMESTAMP_FORMAT_REAL_TIME = 0x2,
> +};
> +
> /* HCA bit masks indicating which Flex parser protocols are already enabled.
> */ #define MLX5_HCA_FLEX_IPV4_OVER_VXLAN_ENABLED (1UL << 0) #define
> MLX5_HCA_FLEX_IPV6_OVER_VXLAN_ENABLED (1UL << 1) @@ -1431,7
> +1446,9 @@ struct mlx5_ifc_cmd_hca_cap_bits {
> u8 reserved_at_3f8[0x3];
> u8 log_max_current_uc_list[0x5];
> u8 general_obj_types[0x40];
> - u8 reserved_at_440[0x20];
> + u8 sq_ts_format[0x2];
> + u8 rq_ts_format[0x2];
> + u8 reserved_at_444[0x1C];
> u8 reserved_at_460[0x10];
> u8 max_num_eqs[0x10];
> u8 reserved_at_480[0x3];
> @@ -1623,6 +1640,15 @@ struct mlx5_ifc_flow_table_prop_layout_bits {
> u8 reserved_at_c0[0x140];
> };
>
> +struct mlx5_ifc_roce_caps_bits {
> + u8 reserved_0[0x5];
> + u8 fl_rc_qp_when_roce_disabled[0x1];
> + u8 fl_rc_qp_when_roce_enabled[0x1];
> + u8 reserved_at_7[0x17];
> + u8 qp_ts_format[0x2];
> + u8 reserved_at_20[0x7e0];
> +};
> +
> struct mlx5_ifc_flow_table_nic_cap_bits {
> u8 reserved_at_0[0x200];
> struct mlx5_ifc_flow_table_prop_layout_bits flow_table_properties;
> @@ -1635,6 +1661,7 @@ union mlx5_ifc_hca_cap_union_bits {
> struct mlx5_ifc_qos_cap_bits qos_cap;
> struct mlx5_ifc_virtio_emulation_cap_bits vdpa_caps;
> struct mlx5_ifc_flow_table_nic_cap_bits flow_table_nic_cap;
> + struct mlx5_ifc_roce_caps_bits roce_caps;
> u8 reserved_at_0[0x8000];
> };
>
> @@ -1851,7 +1878,9 @@ struct mlx5_ifc_rqc_bits {
> u8 reserved_at_c[0x1];
> u8 flush_in_error_en[0x1];
> u8 hairpin[0x1];
> - u8 reserved_at_f[0x11];
> + u8 reserved_at_f[0xB];
> + u8 ts_format[0x02];
> + u8 reserved_at_1c[0x4];
> u8 reserved_at_20[0x8];
> u8 user_index[0x18];
> u8 reserved_at_40[0x8];
> @@ -2157,7 +2186,9 @@ struct mlx5_ifc_sqc_bits {
> u8 hairpin[0x1];
> u8 non_wire[0x1];
> u8 static_sq_wq[0x1];
> - u8 reserved_at_11[0xf];
> + u8 reserved_at_11[0x9];
> + u8 ts_format[0x02];
> + u8 reserved_at_1c[0x4];
> u8 reserved_at_20[0x8];
> u8 user_index[0x18];
> u8 reserved_at_40[0x8];
> @@ -2684,7 +2715,9 @@ struct mlx5_ifc_qpc_bits {
> u8 log_rq_stride[0x3];
> u8 no_sq[0x1];
> u8 log_sq_size[0x4];
> - u8 reserved_at_55[0x6];
> + u8 reserved_at_55[0x3];
> + u8 ts_format[0x2];
> + u8 reserved_at_5a[0x1];
> u8 rlky[0x1];
> u8 ulp_stateless_offload_mode[0x4];
> u8 counter_set_id[0x8];
> --
> 2.28.0
Beside above,
Acked-by: Matan Azrad <ma...@nvidia.com>
