> -----Original Message-----
> From: Huang, Wei <wei.hu...@intel.com>
> Sent: Thursday, June 9, 2022 3:37 PM
> To: dev@dpdk.org; tho...@monjalon.net; nipun.gu...@nxp.com;
> hemant.agra...@nxp.com
> Cc: sta...@dpdk.org; Xu, Rosen <rosen...@intel.com>; Zhang, Tianfei
> <tianfei.zh...@intel.com>; Zhang, Qi Z <qi.z.zh...@intel.com>; Huang, Wei
> <wei.hu...@intel.com>
> Subject: [PATCH v7 5/5] raw/ifpga: add HE-HSSI AFU driver
>
> HE-HSSI is one of the host exerciser modules in OFS FPGA, which is used to
> test
> HSSI (High Speed Serial Interface).
> This driver initialize the module and report test result.
>
> Signed-off-by: Wei Huang <wei.hu...@intel.com>
> ---
> v2: move source files to ifpga and rename, refine code
> ---
> drivers/raw/ifpga/afu_pmd_he_hssi.c | 371
> ++++++++++++++++++++++++++++++++++++
> drivers/raw/ifpga/afu_pmd_he_hssi.h | 109 +++++++++++
> drivers/raw/ifpga/meson.build | 3 +-
> drivers/raw/ifpga/rte_pmd_afu.h | 18 ++
> 4 files changed, 500 insertions(+), 1 deletion(-) create mode 100644
> drivers/raw/ifpga/afu_pmd_he_hssi.c
> create mode 100644 drivers/raw/ifpga/afu_pmd_he_hssi.h
>
> diff --git a/drivers/raw/ifpga/afu_pmd_he_hssi.c
> b/drivers/raw/ifpga/afu_pmd_he_hssi.c
> new file mode 100644
> index 0000000..102de50
> --- /dev/null
> +++ b/drivers/raw/ifpga/afu_pmd_he_hssi.c
> @@ -0,0 +1,371 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(c) 2022 Intel Corporation
> + */
> +
> +#include <errno.h>
> +#include <stdio.h>
> +#include <stdint.h>
> +#include <stdlib.h>
> +#include <inttypes.h>
> +#include <unistd.h>
> +#include <fcntl.h>
> +#include <poll.h>
> +#include <sys/eventfd.h>
> +#include <sys/ioctl.h>
> +
> +#include <rte_eal.h>
> +#include <rte_malloc.h>
> +#include <rte_memcpy.h>
> +#include <rte_io.h>
> +#include <rte_vfio.h>
> +#include <rte_bus_pci.h>
> +#include <rte_bus_ifpga.h>
> +#include <rte_rawdev.h>
> +
> +#include "afu_pmd_core.h"
> +#include "afu_pmd_he_hssi.h"
> +
> +static int he_hssi_indirect_write(struct he_hssi_ctx *ctx, uint32_t addr,
> + uint32_t value)
> +{
> + struct traffic_ctrl_cmd cmd;
> + struct traffic_ctrl_data data;
> + uint32_t i = 0;
> +
> + IFPGA_RAWDEV_PMD_DEBUG("Indirect write 0x%x, value 0x%08x",
> addr,
> +value);
> +
> + if (!ctx)
> + return -EINVAL;
> +
> + data.write_data = value;
> + rte_write64(data.csr, ctx->addr + TRAFFIC_CTRL_DATA);
> +
> + cmd.csr = 0;
> + cmd.write_cmd = 1;
> + cmd.afu_cmd_addr = addr;
> + rte_write64(cmd.csr, ctx->addr + TRAFFIC_CTRL_CMD);
> +
> + while (i < MAILBOX_TIMEOUT_MS) {
> + rte_delay_ms(MAILBOX_POLL_INTERVAL_MS);
> + cmd.csr = rte_read64(ctx->addr + TRAFFIC_CTRL_CMD);
> + if (cmd.ack_trans)
> + break;
> + i += MAILBOX_POLL_INTERVAL_MS;
> + }
> + if (i >= MAILBOX_TIMEOUT_MS)
> + return -ETIMEDOUT;
> +
> + i = 0;
> + cmd.csr = 0;
> + while (i < MAILBOX_TIMEOUT_MS) {
> + cmd.ack_trans = 1;
> + rte_write64(cmd.csr, ctx->addr + TRAFFIC_CTRL_CMD);
> + rte_delay_ms(MAILBOX_POLL_INTERVAL_MS);
> + cmd.csr = rte_read64(ctx->addr + TRAFFIC_CTRL_CMD);
> + if (!cmd.ack_trans)
> + break;
> + i += MAILBOX_POLL_INTERVAL_MS;
> + }
> + if (i >= MAILBOX_TIMEOUT_MS)
> + return -ETIMEDOUT;
> +
> + return 0;
> +}
> +
> +static int he_hssi_indirect_read(struct he_hssi_ctx *ctx, uint32_t addr,
> + uint32_t *value)
> +{
> + struct traffic_ctrl_cmd cmd;
> + struct traffic_ctrl_data data;
> + uint32_t i = 0;
> +
> + if (!ctx)
> + return -EINVAL;
> +
> + cmd.csr = 0;
> + cmd.read_cmd = 1;
> + cmd.afu_cmd_addr = addr;
> + rte_write64(cmd.csr, ctx->addr + TRAFFIC_CTRL_CMD);
> +
> + while (i < MAILBOX_TIMEOUT_MS) {
> + rte_delay_ms(MAILBOX_POLL_INTERVAL_MS);
> + cmd.csr = rte_read64(ctx->addr + TRAFFIC_CTRL_CMD);
> + if (cmd.ack_trans) {
> + data.csr = rte_read64(ctx->addr +
> TRAFFIC_CTRL_DATA);
> + *value = data.read_data;
> + break;
> + }
> + i += MAILBOX_POLL_INTERVAL_MS;
> + }
> + if (i >= MAILBOX_TIMEOUT_MS)
> + return -ETIMEDOUT;
> +
> + i = 0;
> + cmd.csr = 0;
> + while (i < MAILBOX_TIMEOUT_MS) {
> + cmd.ack_trans = 1;
> + rte_write64(cmd.csr, ctx->addr + TRAFFIC_CTRL_CMD);
> + rte_delay_ms(MAILBOX_POLL_INTERVAL_MS);
> + cmd.csr = rte_read64(ctx->addr + TRAFFIC_CTRL_CMD);
> + if (!cmd.ack_trans)
> + break;
> + i += MAILBOX_POLL_INTERVAL_MS;
> + }
> + if (i >= MAILBOX_TIMEOUT_MS)
> + return -ETIMEDOUT;
> +
> + IFPGA_RAWDEV_PMD_DEBUG("Indirect read 0x%x, value 0x%08x",
> addr, *value);
> + return 0;
> +}
> +
> +static void he_hssi_report(struct he_hssi_ctx *ctx) {
> + uint32_t val = 0;
> + uint64_t v64 = 0;
> + int ret = 0;
> +
> + ret = he_hssi_indirect_read(ctx, TM_PKT_GOOD, &val);
> + if (ret)
> + return;
> + printf("Number of good packets received: %u\n", val);
> +
> + ret = he_hssi_indirect_read(ctx, TM_PKT_BAD, &val);
> + if (ret)
> + return;
> + printf("Number of bad packets received: %u\n", val);
> +
> + ret = he_hssi_indirect_read(ctx, TM_BYTE_CNT1, &val);
> + if (ret)
> + return;
> + v64 = val;
> + ret = he_hssi_indirect_read(ctx, TM_BYTE_CNT0, &val);
> + if (ret)
> + return;
> + v64 = (v64 << 32) | val;
> + printf("Number of bytes received: %"PRIu64"\n", v64);
> +
> + ret = he_hssi_indirect_read(ctx, TM_AVST_RX_ERR, &val);
> + if (ret)
> + return;
> + if (val & ERR_VALID) {
> + printf("AVST rx error:");
> + if (val & OVERFLOW_ERR)
> + printf(" overflow");
> + if (val & LENGTH_ERR)
> + printf(" length");
> + if (val & OVERSIZE_ERR)
> + printf(" oversize");
> + if (val & UNDERSIZE_ERR)
> + printf(" undersize");
> + if (val & MAC_CRC_ERR)
> + printf(" crc");
> + if (val & PHY_ERR)
> + printf(" phy");
> + printf("\n");
> + }
> +
> + ret = he_hssi_indirect_read(ctx, LOOPBACK_FIFO_STATUS, &val);
> + if (ret)
> + return;
> + if (val & (ALMOST_EMPTY | ALMOST_FULL)) {
> + printf("FIFO status:");
> + if (val & ALMOST_EMPTY)
> + printf(" almost empty");
> + if (val & ALMOST_FULL)
> + printf(" almost full");
> + printf("\n");
> + }
> +}
> +
> +static int he_hssi_test(struct afu_rawdev *dev) {
> + struct he_hssi_priv *priv = NULL;
> + struct rte_pmd_afu_he_hssi_cfg *cfg = NULL;
> + struct he_hssi_ctx *ctx = NULL;
> + struct traffic_ctrl_ch_sel sel;
> + uint32_t val = 0;
> + uint32_t i = 0;
> + int ret = 0;
> +
> + if (!dev)
> + return -EINVAL;
> +
> + priv = (struct he_hssi_priv *)dev->priv;
> + if (!priv)
> + return -ENOENT;
> +
> + cfg = &priv->he_hssi_cfg;
> + ctx = &priv->he_hssi_ctx;
> +
> + ret = he_hssi_indirect_write(ctx, TG_STOP_XFR, 0);
> + if (ret)
> + return ret;
> +
> + sel.channel_sel = cfg->port;
> + rte_write64(sel.csr, ctx->addr + TRAFFIC_CTRL_CH_SEL);
> +
> + if (cfg->he_loopback >= 0) {
> + val = cfg->he_loopback ? 1 : 0;
> + IFPGA_RAWDEV_PMD_INFO("%s HE loopback on port %u",
> + val ? "Enable" : "Disable", cfg->port);
> + return he_hssi_indirect_write(ctx, LOOPBACK_EN, val);
> + }
> +
> + ret = he_hssi_indirect_write(ctx, TG_NUM_PKT, cfg->num_packets);
> + if (ret)
> + return ret;
> +
> + ret = he_hssi_indirect_write(ctx, TG_PKT_LEN, cfg->packet_length);
> + if (ret)
> + return ret;
> +
> + val = cfg->src_addr & 0xffffffff;
> + ret = he_hssi_indirect_write(ctx, TG_SRC_MAC_L, val);
> + if (ret)
> + return ret;
> + val = (cfg->src_addr >> 32) & 0xffff;
> + ret = he_hssi_indirect_write(ctx, TG_SRC_MAC_H, val);
> + if (ret)
> + return ret;
> +
> + val = cfg->dest_addr & 0xffffffff;
> + ret = he_hssi_indirect_write(ctx, TG_DST_MAC_L, val);
> + if (ret)
> + return ret;
> + val = (cfg->dest_addr >> 32) & 0xffff;
> + ret = he_hssi_indirect_write(ctx, TG_DST_MAC_H, val);
> + if (ret)
> + return ret;
> +
> + val = cfg->random_length ? 1 : 0;
> + ret = he_hssi_indirect_write(ctx, TG_PKT_LEN_TYPE, val);
> + if (ret)
> + return ret;
> +
> + val = cfg->random_payload ? 1 : 0;
> + ret = he_hssi_indirect_write(ctx, TG_DATA_PATTERN, val);
> + if (ret)
> + return ret;
> +
> + for (i = 0; i < TG_NUM_RND_SEEDS; i++) {
> + ret = he_hssi_indirect_write(ctx, TG_RANDOM_SEED(i),
> + cfg->rnd_seed[i]);
> + if (ret)
> + return ret;
> + }
> +
> + ret = he_hssi_indirect_write(ctx, TG_START_XFR, 1);
> + if (ret)
> + return ret;
> +
> + while (i++ < cfg->timeout) {
> + ret = he_hssi_indirect_read(ctx, TG_PKT_XFRD, &val);
> + if (ret)
> + break;
> + if (val == cfg->num_packets)
> + break;
> + sleep(1);
> + }
> +
> + he_hssi_report(ctx);
> +
> + return ret;
> +}
> +
> +static int he_hssi_init(struct afu_rawdev *dev) {
> + struct he_hssi_priv *priv = NULL;
> + struct he_hssi_ctx *ctx = NULL;
> +
> + if (!dev)
> + return -EINVAL;
> +
> + priv = (struct he_hssi_priv *)dev->priv;
> + if (!priv) {
> + priv = rte_zmalloc(NULL, sizeof(struct he_hssi_priv), 0);
> + if (!priv)
> + return -ENOMEM;
> + dev->priv = priv;
> + }
> +
> + ctx = &priv->he_hssi_ctx;
> + ctx->addr = (uint8_t *)dev->addr;
> +
> + return 0;
> +}
> +
> +static int he_hssi_config(struct afu_rawdev *dev, void *config,
> + size_t config_size)
> +{
> + struct he_hssi_priv *priv = NULL;
> + struct rte_pmd_afu_he_hssi_cfg *cfg = NULL;
> +
> + if (!dev || !config || !config_size)
> + return -EINVAL;
> +
> + priv = (struct he_hssi_priv *)dev->priv;
> + if (!priv)
> + return -ENOENT;
> +
> + if (config_size != sizeof(struct rte_pmd_afu_he_hssi_cfg))
> + return -EINVAL;
> +
> + cfg = (struct rte_pmd_afu_he_hssi_cfg *)config;
> + if (cfg->port >= NUM_HE_HSSI_PORTS)
> + return -EINVAL;
> +
> + rte_memcpy(&priv->he_hssi_cfg, cfg, sizeof(priv->he_hssi_cfg));
> +
> + return 0;
> +}
> +
> +static int he_hssi_close(struct afu_rawdev *dev) {
> + if (!dev)
> + return -EINVAL;
> +
> + rte_free(dev->priv);
> + dev->priv = NULL;
> +
> + return 0;
> +}
> +
> +static int he_hssi_dump(struct afu_rawdev *dev, FILE *f) {
> + struct he_hssi_priv *priv = NULL;
> + struct he_hssi_ctx *ctx = NULL;
> +
> + if (!dev)
> + return -EINVAL;
> +
> + priv = (struct he_hssi_priv *)dev->priv;
> + if (!priv)
> + return -ENOENT;
> +
> + if (!f)
> + f = stdout;
> +
> + ctx = &priv->he_hssi_ctx;
> +
> + fprintf(f, "addr:\t\t%p\n", (void *)ctx->addr);
> +
> + return 0;
> +}
> +
> +static struct afu_ops he_hssi_ops = {
> + .init = he_hssi_init,
> + .config = he_hssi_config,
> + .start = NULL,
> + .stop = NULL,
> + .test = he_hssi_test,
> + .close = he_hssi_close,
> + .dump = he_hssi_dump,
> + .reset = NULL
> +};
> +
> +struct afu_rawdev_drv he_hssi_drv = {
> + .uuid = { HE_HSSI_UUID_L, HE_HSSI_UUID_H },
> + .ops = &he_hssi_ops
> +};
> +
> +AFU_PMD_REGISTER(he_hssi_drv);
> diff --git a/drivers/raw/ifpga/afu_pmd_he_hssi.h
> b/drivers/raw/ifpga/afu_pmd_he_hssi.h
> new file mode 100644
> index 0000000..38b433c
> --- /dev/null
> +++ b/drivers/raw/ifpga/afu_pmd_he_hssi.h
> @@ -0,0 +1,109 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(c) 2022 Intel Corporation
> + */
> +
> +#ifndef _AFU_PMD_HE_HSSI_H_
> +#define _AFU_PMD_HE_HSSI_H_
> +
> +#ifdef __cplusplus
> +extern "C" {
> +#endif
> +
> +#include "afu_pmd_core.h"
> +#include "rte_pmd_afu.h"
> +
> +#define HE_HSSI_UUID_L 0xbb370242ac130002
> +#define HE_HSSI_UUID_H 0x823c334c98bf11ea
> +#define NUM_HE_HSSI_PORTS 8
> +
> +/* HE-HSSI registers definition */
> +#define TRAFFIC_CTRL_CMD 0x30
> +#define TRAFFIC_CTRL_DATA 0x38
> +#define TRAFFIC_CTRL_CH_SEL 0x40
> +#define AFU_SCRATCHPAD 0x48
> +
> +#define TG_NUM_PKT 0x3c00
> +#define TG_PKT_LEN_TYPE 0x3c01
> +#define TG_DATA_PATTERN 0x3c02
> +#define TG_START_XFR 0x3c03
> +#define TG_STOP_XFR 0x3c04
> +#define TG_SRC_MAC_L 0x3c05
> +#define TG_SRC_MAC_H 0x3c06
> +#define TG_DST_MAC_L 0x3c07
> +#define TG_DST_MAC_H 0x3c08
> +#define TG_PKT_XFRD 0x3c09
> +#define TG_NUM_RND_SEEDS 3
> +#define TG_RANDOM_SEED(n) (0x3c0a + (n))
> +#define TG_PKT_LEN 0x3c0d
> +
> +#define TM_NUM_PKT 0x3d00
> +#define TM_PKT_GOOD 0x3d01
> +#define TM_PKT_BAD 0x3d02
> +#define TM_BYTE_CNT0 0x3d03
> +#define TM_BYTE_CNT1 0x3d04
> +#define TM_AVST_RX_ERR 0x3d07
> +#define OVERFLOW_ERR (1 << 9)
> +#define LENGTH_ERR (1 << 8)
> +#define OVERSIZE_ERR (1 << 7)
> +#define UNDERSIZE_ERR (1 << 6)
> +#define MAC_CRC_ERR (1 << 5)
> +#define PHY_ERR (1 << 4)
> +#define ERR_VALID (1 << 3)
> +
> +#define LOOPBACK_EN 0x3e00
> +#define LOOPBACK_FIFO_STATUS 0x3e01
> +#define ALMOST_EMPTY (1 << 1)
> +#define ALMOST_FULL (1 << 0)
> +
> +#define MAILBOX_TIMEOUT_MS 100
> +#define MAILBOX_POLL_INTERVAL_MS 10
> +
> +struct traffic_ctrl_cmd {
> + union {
> + uint64_t csr;
> + struct {
> + uint32_t read_cmd:1;
> + uint32_t write_cmd:1;
> + uint32_t ack_trans:1;
> + uint32_t rsvd1:29;
> + uint32_t afu_cmd_addr:16;
> + uint32_t rsvd2:16;
> + };
> + };
> +};
> +
> +struct traffic_ctrl_data {
> + union {
> + uint64_t csr;
> + struct {
> + uint32_t read_data;
> + uint32_t write_data;
> + };
> + };
> +};
> +
> +struct traffic_ctrl_ch_sel {
> + union {
> + uint64_t csr;
> + struct {
> + uint32_t channel_sel:3;
> + uint32_t rsvd1:29;
> + uint32_t rsvd2;
> + };
> + };
> +};
> +
> +struct he_hssi_ctx {
> + uint8_t *addr;
> +};
> +
> +struct he_hssi_priv {
> + struct rte_pmd_afu_he_hssi_cfg he_hssi_cfg;
> + struct he_hssi_ctx he_hssi_ctx;
> +};
> +
> +#ifdef __cplusplus
> +}
> +#endif
> +
> +#endif /* _AFU_PMD_HE_HSSI_H_ */
> diff --git a/drivers/raw/ifpga/meson.build b/drivers/raw/ifpga/meson.build
> index dc6941d..cc30dc8 100644
> --- a/drivers/raw/ifpga/meson.build
> +++ b/drivers/raw/ifpga/meson.build
> @@ -14,7 +14,8 @@ deps += ['ethdev', 'rawdev', 'pci', 'bus_pci', 'kvargs',
> 'bus_vdev', 'bus_ifpga', 'net', 'net_i40e', 'net_ipn3ke']
>
> sources = files('ifpga_rawdev.c', 'rte_pmd_ifpga.c', 'afu_pmd_core.c',
> - 'afu_pmd_n3000.c', 'afu_pmd_he_lpbk.c', 'afu_pmd_he_mem.c')
> + 'afu_pmd_n3000.c', 'afu_pmd_he_lpbk.c', 'afu_pmd_he_mem.c',
> + 'afu_pmd_he_hssi.c')
>
> includes += include_directories('base') includes +=
> include_directories('../../net/ipn3ke')
> diff --git a/drivers/raw/ifpga/rte_pmd_afu.h b/drivers/raw/ifpga/rte_pmd_afu.h
> index 213e854..b1cebbe 100644
> --- a/drivers/raw/ifpga/rte_pmd_afu.h
> +++ b/drivers/raw/ifpga/rte_pmd_afu.h
> @@ -111,6 +111,24 @@ struct rte_pmd_afu_he_mem_tg_cfg {
> uint32_t channel_mask; /* mask of traffic generator channel */
> };
>
> +#define NUM_RND_SEEDS 3
> +
> +/**
> + * HE-HSSI AFU configuration data structure.
> + */
> +struct rte_pmd_afu_he_hssi_cfg {
> + uint32_t port;
> + uint32_t timeout;
> + uint32_t num_packets;
> + uint32_t random_length;
> + uint32_t packet_length;
> + uint32_t random_payload;
> + uint32_t rnd_seed[NUM_RND_SEEDS];
> + uint64_t src_addr;
> + uint64_t dest_addr;
> + int he_loopback;
> +};
> +
> #ifdef __cplusplus
> }
> #endif
> --
It looks good for me, you can add:
Acked-by: Tianfei Zhang <tianfei.zh...@intel.com>
