On Tue, Oct 26, 2021 at 1:34 AM Jerin Jacob <jerinjac...@gmail.com> wrote:
>
> On Tue, Oct 26, 2021 at 9:43 AM Radha Mohan Chintakuntla
> <rad...@marvell.com> wrote:
> >
> > Add base support as ROC(Rest of Chip) API which will be used by PMD
> > dmadev driver.
> >
> > This patch adds routines to init, fini, configure the DPI DMA device
> > found in Marvell's CN9k or CN10k SoC familes.
>
> families
ack.
>
> >
> > Signed-off-by: Radha Mohan Chintakuntla <rad...@marvell.com>
> > ---
> > drivers/common/cnxk/hw/dpi.h | 136 ++++++++++++++++++++
> > drivers/common/cnxk/meson.build | 1 +
> > drivers/common/cnxk/roc_api.h | 4 +
> > drivers/common/cnxk/roc_dpi.c | 193 +++++++++++++++++++++++++++++
> > drivers/common/cnxk/roc_dpi.h | 44 +++++++
> > drivers/common/cnxk/roc_dpi_priv.h | 40 ++++++
> > drivers/common/cnxk/roc_platform.h | 1 +
> > drivers/common/cnxk/roc_priv.h | 3 +
> > drivers/common/cnxk/version.map | 5 +
> > 9 files changed, 427 insertions(+)
> > create mode 100644 drivers/common/cnxk/hw/dpi.h
> > create mode 100644 drivers/common/cnxk/roc_dpi.c
> > create mode 100644 drivers/common/cnxk/roc_dpi.h
> > create mode 100644 drivers/common/cnxk/roc_dpi_priv.h
> >
> > diff --git a/drivers/common/cnxk/hw/dpi.h b/drivers/common/cnxk/hw/dpi.h
> > new file mode 100644
> > index 0000000000..aa1e66aa11
> > --- /dev/null
> > +++ b/drivers/common/cnxk/hw/dpi.h
> > @@ -0,0 +1,136 @@
> > +/* SPDX-License-Identifier: BSD-3-Clause
> > + * Copyright(C) 2021 Marvell.
> > + */
> > +/**
> > + * DPI device HW definitions.
> > + */
> > +#ifndef __DEV_DPI_HW_H__
> > +#define __DEV_DPI_HW_H__
> > +
> > +#include <stdint.h>
> > +
> > +/** @cond __INTERNAL_DOCUMENTATION__ */
>
> This is not required.
>
> > +
> > +/* DPI VF register offsets from VF_BAR0 */
> > +#define DPI_VDMA_EN (0x0)
> > +#define DPI_VDMA_REQQ_CTL (0x8)
> > +#define DPI_VDMA_DBELL (0x10)
> > +#define DPI_VDMA_SADDR (0x18)
> > +#define DPI_VDMA_COUNTS (0x20)
> > +#define DPI_VDMA_NADDR (0x28)
> > +#define DPI_VDMA_IWBUSY (0x30)
> > +#define DPI_VDMA_CNT (0x38)
> > +#define DPI_VF_INT (0x100)
> > +#define DPI_VF_INT_W1S (0x108)
> > +#define DPI_VF_INT_ENA_W1C (0x110)
> > +#define DPI_VF_INT_ENA_W1S (0x118)
> > +
> > +/**
> > + * Enumeration dpi_hdr_xtype_e
> > + *
> > + * DPI Transfer Type Enumeration
> > + * Enumerates the pointer type in DPI_DMA_INSTR_HDR_S[XTYPE].
> > + */
> > +#define DPI_XTYPE_OUTBOUND (0)
> > +#define DPI_XTYPE_INBOUND (1)
> > +#define DPI_XTYPE_INTERNAL_ONLY (2)
> > +#define DPI_XTYPE_EXTERNAL_ONLY (3)
> > +#define DPI_HDR_XTYPE_MASK 0x3
> > +#define DPI_HDR_PT_MASK 0x3
> > +#define DPI_HDR_TT_MASK 0x3
> > +#define DPI_HDR_GRP_MASK 0x3FF
> > +#define DPI_HDR_FUNC_MASK 0xFFFF
> > +
> > +/* Big endian data bit position in DMA local pointer */
> > +#define DPI_LPTR_BED_BIT_POS (60)
> > +
> > +#define DPI_MIN_CMD_SIZE 8
> > +#define DPI_MAX_CMD_SIZE 64
> > +
> > +/**
> > + * Structure dpi_instr_hdr_s for CN9K
> > + *
> > + * DPI DMA Instruction Header Format
> > + */
> > +union dpi_instr_hdr_s {
> > + uint64_t u[4];
> > + struct dpi_dma_instr_hdr_s_s {
> > + uint64_t tag : 32;
> > + uint64_t tt : 2;
> > + uint64_t grp : 10;
> > + uint64_t reserved_44_47 : 4;
> > + uint64_t nfst : 4;
> > + uint64_t reserved_52_53 : 2;
> > + uint64_t nlst : 4;
> > + uint64_t reserved_58_63 : 6;
> > + /* Word 0 - End */
> > + uint64_t aura : 20;
> > + uint64_t func : 16;
> > + uint64_t pt : 2;
> > + uint64_t reserved_102 : 1;
> > + uint64_t pvfe : 1;
> > + uint64_t fl : 1;
> > + uint64_t ii : 1;
> > + uint64_t fi : 1;
> > + uint64_t ca : 1;
> > + uint64_t csel : 1;
> > + uint64_t reserved_109_111 : 3;
> > + uint64_t xtype : 2;
> > + uint64_t reserved_114_119 : 6;
> > + uint64_t fport : 2;
> > + uint64_t reserved_122_123 : 2;
> > + uint64_t lport : 2;
> > + uint64_t reserved_126_127 : 2;
> > + /* Word 1 - End */
> > + uint64_t ptr : 64;
> > + /* Word 2 - End */
> > + uint64_t reserved_192_255 : 64;
> > + /* Word 3 - End */
> > + } s;
> > +};
> > +
> > +/**
> > + * Structure dpi_cn10k_instr_hdr_s for CN10K
> > + *
> > + * DPI DMA Instruction Header Format
> > + */
> > +union dpi_cn10k_instr_hdr_s {
> > + uint64_t u[4];
> > + struct dpi_cn10k_dma_instr_hdr_s_s {
> > + uint64_t nfst : 4;
> > + uint64_t reserved_4_5 : 2;
> > + uint64_t nlst : 4;
> > + uint64_t reserved_10_11 : 2;
> > + uint64_t pvfe : 1;
> > + uint64_t reserved_13 : 1;
> > + uint64_t func : 16;
> > + uint64_t aura : 20;
> > + uint64_t xtype : 2;
> > + uint64_t reserved_52_53 : 2;
> > + uint64_t pt : 2;
> > + uint64_t fport : 2;
> > + uint64_t reserved_58_59 : 2;
> > + uint64_t lport : 2;
> > + uint64_t reserved_62_63 : 2;
> > + /* Word 0 - End */
> > + uint64_t ptr : 64;
> > + /* Word 1 - End */
> > + uint64_t tag : 32;
> > + uint64_t tt : 2;
> > + uint64_t grp : 10;
> > + uint64_t reserved_172_173 : 2;
> > + uint64_t fl : 1;
> > + uint64_t ii : 1;
> > + uint64_t fi : 1;
> > + uint64_t ca : 1;
> > + uint64_t csel : 1;
> > + uint64_t reserved_179_191 : 3;
> > + /* Word 2 - End */
> > + uint64_t reserved_192_255 : 64;
> > + /* Word 3 - End */
> > + } s;
> > +};
> > +
> > +/** @endcond */
>
> This is not required.
ack
>
> > +
> > +#endif /*__DEV_DPI_HW_H__*/
> > diff --git a/drivers/common/cnxk/meson.build
> > b/drivers/common/cnxk/meson.build
> > index d9871a6b45..d0aeb6b68c 100644
> > --- a/drivers/common/cnxk/meson.build
> > +++ b/drivers/common/cnxk/meson.build
> > @@ -19,6 +19,7 @@ sources = files(
> > 'roc_cpt.c',
> > 'roc_cpt_debug.c',
> > 'roc_dev.c',
> > + 'roc_dpi.c',
> > 'roc_hash.c',
> > 'roc_idev.c',
> > 'roc_irq.c',
> > diff --git a/drivers/common/cnxk/roc_api.h b/drivers/common/cnxk/roc_api.h
> > index b8f3667c6c..359d31327a 100644
> > --- a/drivers/common/cnxk/roc_api.h
> > +++ b/drivers/common/cnxk/roc_api.h
> > @@ -33,6 +33,7 @@
> >
> > /* HW structure definition */
> > #include "hw/cpt.h"
> > +#include "hw/dpi.h"
> > #include "hw/nix.h"
> > #include "hw/npa.h"
> > #include "hw/npc.h"
> > @@ -86,6 +87,9 @@
> > #include "roc_ie_ot.h"
> > #include "roc_se.h"
> >
> > +/* DPI */
> > +#include "roc_dpi.h"
> > +
> > /* HASH computation */
> > #include "roc_hash.h"
> >
> > diff --git a/drivers/common/cnxk/roc_dpi.c b/drivers/common/cnxk/roc_dpi.c
> > new file mode 100644
> > index 0000000000..a9613d82f1
> > --- /dev/null
> > +++ b/drivers/common/cnxk/roc_dpi.c
> > @@ -0,0 +1,193 @@
> > +/* SPDX-License-Identifier: BSD-3-Clause
> > + * Copyright(C) 2021 Marvell.
> > + */
> > +
> > +#include "roc_api.h"
> > +#include "roc_priv.h"
>
> move internal headers file after the public ones with a new line.
ack
>
> > +#include <fcntl.h>
> > +#include <sys/stat.h>
> > +#include <sys/types.h>
> > +#include <unistd.h>
> > +
> > +#define DPI_PF_MBOX_SYSFS_ENTRY "dpi_device_config"
> > +
> > +static inline int
> > +send_msg_to_pf(struct plt_pci_addr *pci_addr, const char *value, int size)
> > +{
> > + char buf[255] = {0};
> > + int res, fd;
> > +
> > + res = snprintf(
> > + buf, sizeof(buf), "/sys/bus/pci/devices/" PCI_PRI_FMT "/%s",
> > + pci_addr->domain, pci_addr->bus, DPI_PF_DBDF_DEVICE & 0x7,
> > + DPI_PF_DBDF_FUNCTION & 0x7, DPI_PF_MBOX_SYSFS_ENTRY);
> > +
> > + if ((res < 0) || ((size_t)res > sizeof(buf)))
> > + return -ERANGE;
> > +
> > + fd = open(buf, O_WRONLY);
> > + if (fd < 0)
> > + return -EACCES;
> > +
> > + res = write(fd, value, size);
> > + close(fd);
> > + if (res < 0)
> > + return -EACCES;
> > +
> > + return 0;
> > +}
> > +
> > +int
> > +roc_dpi_queue_start(struct roc_dpi *dpi)
> > +{
> > + plt_write64(0x1, dpi->rbase + DPI_VDMA_EN);
> > + return 0;
> > +}
> > +
> > +int
> > +roc_dpi_queue_stop(struct roc_dpi *dpi)
> > +{
> > + plt_write64(0x0, dpi->rbase + DPI_VDMA_EN);
> > + return 0;
> > +}
> > +
> > +int
> > +roc_dpi_queue_configure(struct roc_dpi *roc_dpi)
> > +{
> > + struct plt_pci_device *pci_dev;
> > + const struct plt_memzone *dpi_mz;
> > + dpi_mbox_msg_t mbox_msg;
> > + struct npa_pool_s pool;
> > + struct npa_aura_s aura;
> > + int rc, count, buflen;
> > + uint64_t aura_handle;
> > + plt_iova_t iova;
> > + char name[32];
> > +
> > + if (!roc_dpi) {
> > + plt_err("roc_dpi is NULL");
> > + return -EINVAL;
> > + }
> > +
> > + pci_dev = roc_dpi->pci_dev;
> > + memset(&pool, 0, sizeof(struct npa_pool_s));
> > + pool.nat_align = 1;
> > +
> > + memset(&aura, 0, sizeof(aura));
> > + rc = roc_npa_pool_create(&aura_handle, DPI_CMD_QUEUE_SIZE,
> > + DPI_CMD_QUEUE_BUFS, &aura, &pool);
> > + if (rc) {
> > + plt_err("Failed to create NPA pool, err %d\n", rc);
> > + return rc;
> > + }
> > +
> > + snprintf(name, sizeof(name), "dpimem%d", roc_dpi->vfid);
> > + buflen = DPI_CMD_QUEUE_SIZE * DPI_CMD_QUEUE_BUFS;
> > + dpi_mz = plt_memzone_reserve_aligned(name, buflen, 0,
> > + DPI_CMD_QUEUE_SIZE);
> > + if (dpi_mz == NULL) {
> > + plt_err("dpi memzone reserve failed");
> > + rc = -ENOMEM;
> > + goto err1;
> > + }
> > +
> > + roc_dpi->mz = dpi_mz;
> > + iova = dpi_mz->iova;
> > + for (count = 0; count < DPI_CMD_QUEUE_BUFS; count++) {
> > + roc_npa_aura_op_free(aura_handle, 0, iova);
> > + iova += DPI_CMD_QUEUE_SIZE;
> > + }
> > +
> > + roc_dpi->chunk_base = (void *)roc_npa_aura_op_alloc(aura_handle, 0);
> > + if (!roc_dpi->chunk_base) {
> > + plt_err("Failed to alloc buffer from NPA aura");
> > + rc = -ENOMEM;
> > + goto err2;
> > + }
> > +
> > + roc_dpi->chunk_next = (void *)roc_npa_aura_op_alloc(aura_handle, 0);
> > + if (!roc_dpi->chunk_next) {
> > + plt_err("Failed to alloc buffer from NPA aura");
> > + rc = -ENOMEM;
> > + goto err2;
> > + }
> > +
> > + roc_dpi->aura_handle = aura_handle;
> > + /* subtract 2 as they have already been alloc'ed above */
> > + roc_dpi->pool_size_m1 = (DPI_CMD_QUEUE_SIZE >> 3) - 2;
> > +
> > + plt_write64(0x0, roc_dpi->rbase + DPI_VDMA_REQQ_CTL);
> > + plt_write64(((uint64_t)(roc_dpi->chunk_base) >> 7) << 7,
> > + roc_dpi->rbase + DPI_VDMA_SADDR);
> > + mbox_msg.u[0] = 0;
> > + mbox_msg.u[1] = 0;
> > + /* DPI PF driver expects vfid starts from index 0 */
> > + mbox_msg.s.vfid = roc_dpi->vfid;
> > + mbox_msg.s.cmd = DPI_QUEUE_OPEN;
> > + mbox_msg.s.csize = DPI_CMD_QUEUE_SIZE;
> > + mbox_msg.s.aura = roc_npa_aura_handle_to_aura(aura_handle);
> > + mbox_msg.s.sso_pf_func = idev_sso_pffunc_get();
> > + mbox_msg.s.npa_pf_func = idev_npa_pffunc_get();
> > +
> > + rc = send_msg_to_pf(&pci_dev->addr, (const char *)&mbox_msg,
> > + sizeof(dpi_mbox_msg_t));
> > + if (rc < 0) {
> > + plt_err("Failed to send mbox message %d to DPI PF, err %d",
> > + mbox_msg.s.cmd, rc);
> > + goto err2;
> > + }
> > +
> > + return rc;
> > +
> > +err2:
> > + roc_npa_pool_destroy(aura_handle);
> > +err1:
> > + plt_memzone_free(dpi_mz);
> > + return rc;
> > +}
> > +
> > +int
> > +roc_dpi_dev_init(struct roc_dpi *roc_dpi)
> > +{
> > + struct plt_pci_device *pci_dev = roc_dpi->pci_dev;
> > + uint16_t vfid;
> > +
> > + roc_dpi->rbase = pci_dev->mem_resource[0].addr;
> > + vfid = ((pci_dev->addr.devid & 0x1F) << 3) |
> > + (pci_dev->addr.function & 0x7);
> > + vfid -= 1;
> > + roc_dpi->vfid = vfid;
> > + plt_spinlock_init(&roc_dpi->chunk_lock);
> > +
> > + return 0;
> > +}
> > +
> > +int
> > +roc_dpi_dev_fini(struct roc_dpi *roc_dpi)
> > +{
> > + struct plt_pci_device *pci_dev = roc_dpi->pci_dev;
> > + dpi_mbox_msg_t mbox_msg;
> > + uint64_t reg;
> > + int rc;
> > +
> > + /* Wait for SADDR to become idle */
> > + reg = plt_read64(roc_dpi->rbase + DPI_VDMA_SADDR);
> > + while (!(reg & BIT_ULL(63)))
> > + reg = plt_read64(roc_dpi->rbase + DPI_VDMA_SADDR);
> > +
> > + mbox_msg.u[0] = 0;
> > + mbox_msg.u[1] = 0;
> > + mbox_msg.s.vfid = roc_dpi->vfid;
> > + mbox_msg.s.cmd = DPI_QUEUE_CLOSE;
> > +
> > + rc = send_msg_to_pf(&pci_dev->addr, (const char *)&mbox_msg,
> > + sizeof(dpi_mbox_msg_t));
> > + if (rc < 0)
> > + plt_err("Failed to send mbox message %d to DPI PF, err %d",
> > + mbox_msg.s.cmd, rc);
> > +
> > + roc_npa_pool_destroy(roc_dpi->aura_handle);
> > + plt_memzone_free(roc_dpi->mz);
> > +
> > + return rc;
> > +}
> > diff --git a/drivers/common/cnxk/roc_dpi.h b/drivers/common/cnxk/roc_dpi.h
> > new file mode 100644
> > index 0000000000..c2e6d997ea
> > --- /dev/null
> > +++ b/drivers/common/cnxk/roc_dpi.h
> > @@ -0,0 +1,44 @@
> > +/* SPDX-License-Identifier: BSD-3-Clause
> > + * Copyright(C) 2021 Marvell.
> > + */
> > +
> > +#ifndef _ROC_DPI_H_
> > +#define _ROC_DPI_H_
> > +
> > +struct roc_dpi_args {
> > + uint8_t num_ssegs;
> > + uint8_t num_dsegs;
> > + uint8_t comp_type;
> > + uint8_t direction;
> > + uint8_t sdevice;
> > + uint8_t ddevice;
> > + uint8_t swap;
> > + uint8_t use_lock : 1;
> > + uint8_t tt : 7;
> > + uint16_t func;
> > + uint16_t grp;
> > + uint32_t tag;
> > + uint64_t comp_ptr;
> > +};
> > +
> > +struct roc_dpi {
> > + struct plt_pci_device *pci_dev;
> > + const struct plt_memzone *mz;
> > + uint8_t *rbase;
> > + uint16_t vfid;
> > + uint16_t pool_size_m1;
> > + uint16_t chunk_head;
> > + uint64_t *chunk_base;
> > + uint64_t *chunk_next;
> > + uint64_t aura_handle;
> > + plt_spinlock_t chunk_lock;
>
> segregate input parameters for roc_dpi_dev_init() and RoC managed
> variables with comments.
>
> > +} __plt_cache_aligned;
> > +
> > +int __roc_api roc_dpi_dev_init(struct roc_dpi *roc_dpi);
> > +int __roc_api roc_dpi_dev_fini(struct roc_dpi *roc_dpi);
> > +
> > +int __roc_api roc_dpi_queue_configure(struct roc_dpi *dpi);
> > +int __roc_api roc_dpi_queue_start(struct roc_dpi *dpi);
> > +int __roc_api roc_dpi_queue_stop(struct roc_dpi *dpi);
>
> For future proof and make API looks clean, add quued id.
> If there will not be any queue in the future change to
> roc_dpi_configure()... etc.
> Please change start/stop change to enable/disable.
ack. will get rid of "queue" in the name.
>
>
> > +
> > +#endif
> > diff --git a/drivers/common/cnxk/roc_dpi_priv.h
> > b/drivers/common/cnxk/roc_dpi_priv.h
> > new file mode 100644
> > index 0000000000..92953fbcfc
> > --- /dev/null
> > +++ b/drivers/common/cnxk/roc_dpi_priv.h
> > @@ -0,0 +1,40 @@
> > +/* SPDX-License-Identifier: BSD-3-Clause
> > + * Copyright(C) 2021 Marvell.
> > + */
> > +
> > +#ifndef _ROC_DPI_PRIV_H_
> > +#define _ROC_DPI_PRIV_H_
> > +
> > +#define DPI_MAX_VFS 8
> > +
> > +/* DPI PF DBDF information macros */
> > +#define DPI_PF_DBDF_DEVICE 0
> > +#define DPI_PF_DBDF_FUNCTION 0
> > +
> > +#define DPI_QUEUE_OPEN 0x1
> > +#define DPI_QUEUE_CLOSE 0x2
> > +#define DPI_REG_DUMP 0x3
> > +#define DPI_GET_REG_CFG 0x4
> > +
> > +#define DPI_CMD_QUEUE_SIZE 4096
> > +#define DPI_CMD_QUEUE_BUFS 1024
> > +
> > +typedef union dpi_mbox_msg_t {
> > + uint64_t u[2];
> > + struct dpi_mbox_message_s {
> > + /* VF ID to configure */
> > + uint64_t vfid : 4;
> > + /* Command code */
> > + uint64_t cmd : 4;
> > + /* Command buffer size in 8-byte words */
> > + uint64_t csize : 14;
> > + /* aura of the command buffer */
> > + uint64_t aura : 20;
> > + /* SSO PF function */
> > + uint64_t sso_pf_func : 16;
> > + /* NPA PF function */
> > + uint64_t npa_pf_func : 16;
> > + } s;
> > +} dpi_mbox_msg_t;
> > +
> > +#endif
> > diff --git a/drivers/common/cnxk/roc_platform.h
> > b/drivers/common/cnxk/roc_platform.h
> > index 5da23fe5f8..61d4781209 100644
> > --- a/drivers/common/cnxk/roc_platform.h
> > +++ b/drivers/common/cnxk/roc_platform.h
> > @@ -69,6 +69,7 @@
> > #define __roc_api __rte_internal
> > #define plt_iova_t rte_iova_t
> >
> > +#define plt_pci_addr rte_pci_addr
> > #define plt_pci_device rte_pci_device
> > #define plt_pci_read_config rte_pci_read_config
> > #define plt_pci_find_ext_capability rte_pci_find_ext_capability
> > diff --git a/drivers/common/cnxk/roc_priv.h b/drivers/common/cnxk/roc_priv.h
> > index f72bbd568f..782b90cf8d 100644
> > --- a/drivers/common/cnxk/roc_priv.h
> > +++ b/drivers/common/cnxk/roc_priv.h
> > @@ -41,4 +41,7 @@
> > /* NIX Inline dev */
> > #include "roc_nix_inl_priv.h"
> >
> > +/* DPI */
> > +#include "roc_dpi_priv.h"
> > +
> > #endif /* _ROC_PRIV_H_ */
> > diff --git a/drivers/common/cnxk/version.map
> > b/drivers/common/cnxk/version.map
> > index 8d4d42f476..3edc42cfd6 100644
> > --- a/drivers/common/cnxk/version.map
> > +++ b/drivers/common/cnxk/version.map
> > @@ -68,6 +68,11 @@ INTERNAL {
> > roc_cpt_lmtline_init;
> > roc_cpt_parse_hdr_dump;
> > roc_cpt_rxc_time_cfg;
> > + roc_dpi_dev_init;
> > + roc_dpi_dev_fini;
> > + roc_dpi_queue_configure;
> > + roc_dpi_queue_start;
> > + roc_dpi_queue_stop;
> > roc_error_msg_get;
> > roc_hash_sha1_gen;
> > roc_hash_sha256_gen;
> > --
> > 2.17.1
> >
