From: Danylo Vodopianov <dvo-...@napatech.com>
Added TX and RX queue setup. Handles memory allocation
and hardware Virtio queue setup.
Allocates and configures memory for hardware Virtio queues,
including handling IOMMU and VFIO mappings.
Signed-off-by: Danylo Vodopianov <dvo-...@napatech.com>
---
drivers/net/ntnic/include/ntnic_virt_queue.h | 3 +-
drivers/net/ntnic/include/ntos_drv.h | 6 +
drivers/net/ntnic/nthw/nthw_drv.h | 2 +
drivers/net/ntnic/ntnic_ethdev.c | 323 +++++++++++++++++++
4 files changed, 333 insertions(+), 1 deletion(-)
diff --git a/drivers/net/ntnic/include/ntnic_virt_queue.h
b/drivers/net/ntnic/include/ntnic_virt_queue.h
index 422ac3b950..821b23af6c 100644
--- a/drivers/net/ntnic/include/ntnic_virt_queue.h
+++ b/drivers/net/ntnic/include/ntnic_virt_queue.h
@@ -13,7 +13,8 @@
struct nthw_virt_queue;
-struct nthw_virtq_desc_buf;
+#define SPLIT_RING 0
+#define IN_ORDER 1
struct nthw_cvirtq_desc;
diff --git a/drivers/net/ntnic/include/ntos_drv.h
b/drivers/net/ntnic/include/ntos_drv.h
index 233d585303..933b012e07 100644
--- a/drivers/net/ntnic/include/ntos_drv.h
+++ b/drivers/net/ntnic/include/ntos_drv.h
@@ -47,6 +47,7 @@ struct __rte_cache_aligned ntnic_rx_queue {
struct hwq_s hwq;
struct nthw_virt_queue *vq;
+ int nb_hw_rx_descr;
nt_meta_port_type_t type;
uint32_t port; /* Rx port for this queue */
enum fpga_info_profile profile; /* Inline / Capture */
@@ -57,7 +58,12 @@ struct __rte_cache_aligned ntnic_tx_queue {
struct flow_queue_id_s queue; /* queue info - user id and hw queue
index */
struct hwq_s hwq;
struct nthw_virt_queue *vq;
+ int nb_hw_tx_descr;
+ /* Used for bypass in NTDVIO0 header on Tx - pre calculated */
+ int target_id;
nt_meta_port_type_t type;
+ /* only used for exception tx queue from OVS SW switching */
+ int rss_target_id;
uint32_t port; /* Tx port for this queue */
int enabled; /* Enabling/disabling of this queue */
diff --git a/drivers/net/ntnic/nthw/nthw_drv.h
b/drivers/net/ntnic/nthw/nthw_drv.h
index eaa2b19015..69e0360f5f 100644
--- a/drivers/net/ntnic/nthw/nthw_drv.h
+++ b/drivers/net/ntnic/nthw/nthw_drv.h
@@ -71,6 +71,8 @@ typedef struct fpga_info_s {
struct nthw_pcie3 *mp_nthw_pcie3;
struct nthw_tsm *mp_nthw_tsm;
+ nthw_dbs_t *mp_nthw_dbs;
+
uint8_t *bar0_addr; /* Needed for register read/write */
size_t bar0_size;
diff --git a/drivers/net/ntnic/ntnic_ethdev.c b/drivers/net/ntnic/ntnic_ethdev.c
index 78a689d444..57827d73d5 100644
--- a/drivers/net/ntnic/ntnic_ethdev.c
+++ b/drivers/net/ntnic/ntnic_ethdev.c
@@ -31,10 +31,16 @@
#define MAX_TOTAL_QUEUES 128
+#define SG_NB_HW_RX_DESCRIPTORS 1024
+#define SG_NB_HW_TX_DESCRIPTORS 1024
+#define SG_HW_RX_PKT_BUFFER_SIZE (1024 << 1)
+#define SG_HW_TX_PKT_BUFFER_SIZE (1024 << 1)
+
/* Max RSS queues */
#define MAX_QUEUES 125
#define ONE_G_SIZE 0x40000000
+#define ONE_G_MASK (ONE_G_SIZE - 1)
#define ETH_DEV_NTNIC_HELP_ARG "help"
#define ETH_DEV_NTHW_RXQUEUES_ARG "rxqs"
@@ -187,6 +193,157 @@ eth_dev_infos_get(struct rte_eth_dev *eth_dev, struct
rte_eth_dev_info *dev_info
return 0;
}
+static int allocate_hw_virtio_queues(struct rte_eth_dev *eth_dev, int vf_num,
struct hwq_s *hwq,
+ int num_descr, int buf_size)
+{
+ int i, res;
+ uint32_t size;
+ uint64_t iova_addr;
+
+ NT_LOG(DBG, NTNIC, "***** Configure IOMMU for HW queues on VF %i
*****\n", vf_num);
+
+ /* Just allocate 1MB to hold all combined descr rings */
+ uint64_t tot_alloc_size = 0x100000 + buf_size * num_descr;
+
+ void *virt =
+ rte_malloc_socket("VirtQDescr", tot_alloc_size,
nt_util_align_size(tot_alloc_size),
+ eth_dev->data->numa_node);
+
+ if (!virt)
+ return -1;
+
+ uint64_t gp_offset = (uint64_t)virt & ONE_G_MASK;
+ rte_iova_t hpa = rte_malloc_virt2iova(virt);
+
+ NT_LOG(DBG, NTNIC, "Allocated virtio descr rings : virt "
+ "%p [0x%" PRIX64 "],hpa %" PRIX64 " [0x%" PRIX64 "]\n",
+ virt, gp_offset, hpa, hpa & ONE_G_MASK);
+
+ /*
+ * Same offset on both HPA and IOVA
+ * Make sure 1G boundary is never crossed
+ */
+ if (((hpa & ONE_G_MASK) != gp_offset) ||
+ (((uint64_t)virt + tot_alloc_size) & ~ONE_G_MASK) !=
+ ((uint64_t)virt & ~ONE_G_MASK)) {
+ NT_LOG(ERR, NTNIC,
"*********************************************************\n");
+ NT_LOG(ERR, NTNIC, "ERROR, no optimal IOMMU mapping available
hpa: %016" PRIX64
+ "(%016" PRIX64 "), gp_offset: %016" PRIX64 " size: %"
PRIu64 "\n",
+ hpa, hpa & ONE_G_MASK, gp_offset, tot_alloc_size);
+ NT_LOG(ERR, NTNIC,
"*********************************************************\n");
+
+ rte_free(virt);
+
+ /* Just allocate 1MB to hold all combined descr rings */
+ size = 0x100000;
+ void *virt = rte_malloc_socket("VirtQDescr", size, 4096,
eth_dev->data->numa_node);
+
+ if (!virt)
+ return -1;
+
+ res = nt_vfio_dma_map(vf_num, virt, &iova_addr, size);
+
+ NT_LOG(DBG, NTNIC, "VFIO MMAP res %i, vf_num %i\n", res,
vf_num);
+
+ if (res != 0)
+ return -1;
+
+ hwq->vf_num = vf_num;
+ hwq->virt_queues_ctrl.virt_addr = virt;
+ hwq->virt_queues_ctrl.phys_addr = (void *)iova_addr;
+ hwq->virt_queues_ctrl.len = size;
+
+ NT_LOG(DBG, NTNIC,
+ "Allocated for virtio descr rings combined 1MB : %p,
IOVA %016" PRIX64 "\n",
+ virt, iova_addr);
+
+ size = num_descr * sizeof(struct nthw_memory_descriptor);
+ hwq->pkt_buffers =
+ rte_zmalloc_socket("rx_pkt_buffers", size, 64,
eth_dev->data->numa_node);
+
+ if (!hwq->pkt_buffers) {
+ NT_LOG(ERR, NTNIC,
+ "Failed to allocated buffer array for hw-queue
%p, total size %i, elements %i\n",
+ hwq->pkt_buffers, size, num_descr);
+ rte_free(virt);
+ return -1;
+ }
+
+ size = buf_size * num_descr;
+ void *virt_addr =
+ rte_malloc_socket("pkt_buffer_pkts", size, 4096,
eth_dev->data->numa_node);
+
+ if (!virt_addr) {
+ NT_LOG(ERR, NTNIC,
+ "Failed allocate packet buffers for hw-queue
%p, buf size %i, elements %i\n",
+ hwq->pkt_buffers, buf_size, num_descr);
+ rte_free(hwq->pkt_buffers);
+ rte_free(virt);
+ return -1;
+ }
+
+ res = nt_vfio_dma_map(vf_num, virt_addr, &iova_addr, size);
+
+ NT_LOG(DBG, NTNIC,
+ "VFIO MMAP res %i, virt %p, iova %016" PRIX64 ", vf_num
%i, num pkt bufs %i, tot size %i\n",
+ res, virt_addr, iova_addr, vf_num, num_descr, size);
+
+ if (res != 0)
+ return -1;
+
+ for (i = 0; i < num_descr; i++) {
+ hwq->pkt_buffers[i].virt_addr =
+ (void *)((char *)virt_addr + ((uint64_t)(i) *
buf_size));
+ hwq->pkt_buffers[i].phys_addr =
+ (void *)(iova_addr + ((uint64_t)(i) *
buf_size));
+ hwq->pkt_buffers[i].len = buf_size;
+ }
+
+ return 0;
+ } /* End of: no optimal IOMMU mapping available */
+
+ res = nt_vfio_dma_map(vf_num, virt, &iova_addr, ONE_G_SIZE);
+
+ if (res != 0) {
+ NT_LOG(ERR, NTNIC, "VFIO MMAP FAILED! res %i, vf_num %i\n",
res, vf_num);
+ return -1;
+ }
+
+ hwq->vf_num = vf_num;
+ hwq->virt_queues_ctrl.virt_addr = virt;
+ hwq->virt_queues_ctrl.phys_addr = (void *)(iova_addr);
+ hwq->virt_queues_ctrl.len = 0x100000;
+ iova_addr += 0x100000;
+
+ NT_LOG(DBG, NTNIC,
+ "VFIO MMAP: virt_addr=%p phys_addr=%p size=%" PRIX32 " hpa=%"
PRIX64 "\n",
+ hwq->virt_queues_ctrl.virt_addr,
hwq->virt_queues_ctrl.phys_addr,
+ hwq->virt_queues_ctrl.len,
rte_malloc_virt2iova(hwq->virt_queues_ctrl.virt_addr));
+
+ size = num_descr * sizeof(struct nthw_memory_descriptor);
+ hwq->pkt_buffers =
+ rte_zmalloc_socket("rx_pkt_buffers", size, 64,
eth_dev->data->numa_node);
+
+ if (!hwq->pkt_buffers) {
+ NT_LOG(ERR, NTNIC,
+ "Failed to allocated buffer array for hw-queue %p,
total size %i, elements %i\n",
+ hwq->pkt_buffers, size, num_descr);
+ rte_free(virt);
+ return -1;
+ }
+
+ void *virt_addr = (void *)((uint64_t)virt + 0x100000);
+
+ for (i = 0; i < num_descr; i++) {
+ hwq->pkt_buffers[i].virt_addr =
+ (void *)((char *)virt_addr + ((uint64_t)(i) *
buf_size));
+ hwq->pkt_buffers[i].phys_addr = (void *)(iova_addr +
((uint64_t)(i) * buf_size));
+ hwq->pkt_buffers[i].len = buf_size;
+ }
+
+ return 0;
+}
+
static void release_hw_virtio_queues(struct hwq_s *hwq)
{
if (!hwq || hwq->vf_num == 0)
@@ -245,6 +402,170 @@ static int allocate_queue(int num)
return next_free;
}
+static int eth_rx_scg_queue_setup(struct rte_eth_dev *eth_dev,
+ uint16_t rx_queue_id,
+ uint16_t nb_rx_desc __rte_unused,
+ unsigned int socket_id __rte_unused,
+ const struct rte_eth_rxconf *rx_conf __rte_unused,
+ struct rte_mempool *mb_pool)
+{
+ NT_LOG_DBGX(DBG, NTNIC, "\n");
+ struct rte_pktmbuf_pool_private *mbp_priv;
+ struct pmd_internals *internals = (struct pmd_internals
*)eth_dev->data->dev_private;
+ struct ntnic_rx_queue *rx_q = &internals->rxq_scg[rx_queue_id];
+ struct drv_s *p_drv = internals->p_drv;
+ struct ntdrv_4ga_s *p_nt_drv = &p_drv->ntdrv;
+
+ if (sg_ops == NULL) {
+ NT_LOG_DBGX(DBG, NTNIC, "SG module is not initialized\n");
+ return 0;
+ }
+
+ if (internals->type == PORT_TYPE_OVERRIDE) {
+ rx_q->mb_pool = mb_pool;
+ eth_dev->data->rx_queues[rx_queue_id] = rx_q;
+ mbp_priv = rte_mempool_get_priv(rx_q->mb_pool);
+ rx_q->buf_size = (uint16_t)(mbp_priv->mbuf_data_room_size -
RTE_PKTMBUF_HEADROOM);
+ rx_q->enabled = 1;
+ return 0;
+ }
+
+ NT_LOG(DBG, NTNIC, "(%i) NTNIC RX OVS-SW queue setup: queue id %i, hw
queue index %i\n",
+ internals->port, rx_queue_id, rx_q->queue.hw_id);
+
+ rx_q->mb_pool = mb_pool;
+
+ eth_dev->data->rx_queues[rx_queue_id] = rx_q;
+
+ mbp_priv = rte_mempool_get_priv(rx_q->mb_pool);
+ rx_q->buf_size = (uint16_t)(mbp_priv->mbuf_data_room_size -
RTE_PKTMBUF_HEADROOM);
+ rx_q->enabled = 1;
+
+ if (allocate_hw_virtio_queues(eth_dev, EXCEPTION_PATH_HID, &rx_q->hwq,
+ SG_NB_HW_RX_DESCRIPTORS, SG_HW_RX_PKT_BUFFER_SIZE) < 0)
+ return -1;
+
+ rx_q->nb_hw_rx_descr = SG_NB_HW_RX_DESCRIPTORS;
+
+ rx_q->profile = p_drv->ntdrv.adapter_info.fpga_info.profile;
+
+ rx_q->vq =
+
sg_ops->nthw_setup_mngd_rx_virt_queue(p_nt_drv->adapter_info.fpga_info.mp_nthw_dbs,
+ rx_q->queue.hw_id, /* index */
+ rx_q->nb_hw_rx_descr,
+ EXCEPTION_PATH_HID, /* host_id */
+ 1, /* header NT DVIO header for exception path */
+ &rx_q->hwq.virt_queues_ctrl,
+ rx_q->hwq.pkt_buffers,
+ SPLIT_RING,
+ -1);
+
+ NT_LOG(DBG, NTNIC, "(%i) NTNIC RX OVS-SW queues successfully setup\n",
internals->port);
+
+ return 0;
+}
+
+static int eth_tx_scg_queue_setup(struct rte_eth_dev *eth_dev,
+ uint16_t tx_queue_id,
+ uint16_t nb_tx_desc __rte_unused,
+ unsigned int socket_id __rte_unused,
+ const struct rte_eth_txconf *tx_conf __rte_unused)
+{
+ const struct port_ops *port_ops = get_port_ops();
+
+ if (port_ops == NULL) {
+ NT_LOG_DBGX(ERR, NTNIC, "Link management module
uninitialized\n");
+ return -1;
+ }
+
+ NT_LOG_DBGX(DBG, NTNIC, "\n");
+ struct pmd_internals *internals = (struct pmd_internals
*)eth_dev->data->dev_private;
+ struct drv_s *p_drv = internals->p_drv;
+ struct ntdrv_4ga_s *p_nt_drv = &p_drv->ntdrv;
+ struct ntnic_tx_queue *tx_q = &internals->txq_scg[tx_queue_id];
+
+ if (internals->type == PORT_TYPE_OVERRIDE) {
+ eth_dev->data->tx_queues[tx_queue_id] = tx_q;
+ return 0;
+ }
+
+ if (sg_ops == NULL) {
+ NT_LOG_DBGX(DBG, NTNIC, "SG module is not initialized\n");
+ return 0;
+ }
+
+ NT_LOG(DBG, NTNIC, "(%i) NTNIC TX OVS-SW queue setup: queue id %i, hw
queue index %i\n",
+ tx_q->port, tx_queue_id, tx_q->queue.hw_id);
+
+ if (tx_queue_id > internals->nb_tx_queues) {
+ NT_LOG(ERR, NTNIC, "Error invalid tx queue id\n");
+ return -1;
+ }
+
+ eth_dev->data->tx_queues[tx_queue_id] = tx_q;
+
+ /* Calculate target ID for HW - to be used in NTDVIO0 header
bypass_port */
+ if (tx_q->rss_target_id >= 0) {
+ /* bypass to a multiqueue port - qsl-hsh index */
+ tx_q->target_id = tx_q->rss_target_id + 0x90;
+
+ } else if (internals->vpq[tx_queue_id].hw_id > -1) {
+ /* virtual port - queue index */
+ tx_q->target_id = internals->vpq[tx_queue_id].hw_id;
+
+ } else {
+ /* Phy port - phy port identifier */
+ /* output/bypass to MAC */
+ tx_q->target_id = (int)(tx_q->port + 0x80);
+ }
+
+ if (allocate_hw_virtio_queues(eth_dev, EXCEPTION_PATH_HID, &tx_q->hwq,
+ SG_NB_HW_TX_DESCRIPTORS, SG_HW_TX_PKT_BUFFER_SIZE) < 0)
{
+ return -1;
+ }
+
+ tx_q->nb_hw_tx_descr = SG_NB_HW_TX_DESCRIPTORS;
+
+ tx_q->profile = p_drv->ntdrv.adapter_info.fpga_info.profile;
+
+ uint32_t port, header;
+ port = tx_q->port; /* transmit port */
+ header = 0; /* header type VirtIO-Net */
+
+ tx_q->vq =
+
sg_ops->nthw_setup_mngd_tx_virt_queue(p_nt_drv->adapter_info.fpga_info.mp_nthw_dbs,
+ tx_q->queue.hw_id, /* index */
+ tx_q->nb_hw_tx_descr, /* queue size */
+ EXCEPTION_PATH_HID, /* host_id always VF4 */
+ port,
+ /*
+ * in_port - in vswitch mode has
+ * to move tx port from OVS excep.
+ * away from VM tx port,
+ * because of QoS is matched by port id!
+ */
+ tx_q->port + 128,
+ header,
+ &tx_q->hwq.virt_queues_ctrl,
+ tx_q->hwq.pkt_buffers,
+ SPLIT_RING,
+ -1,
+ IN_ORDER);
+
+ tx_q->enabled = 1;
+
+ NT_LOG(DBG, NTNIC, "(%i) NTNIC TX OVS-SW queues successfully setup\n",
internals->port);
+
+ if (internals->type == PORT_TYPE_PHYSICAL) {
+ struct adapter_info_s *p_adapter_info =
&internals->p_drv->ntdrv.adapter_info;
+ NT_LOG(DBG, NTNIC, "Port %i is ready for data. Enable port\n",
+ internals->n_intf_no);
+ port_ops->set_adm_state(p_adapter_info, internals->n_intf_no,
true);
+ }
+
+ return 0;
+}
+
static int eth_rx_queue_start(struct rte_eth_dev *eth_dev, uint16_t
rx_queue_id)
{
eth_dev->data->rx_queue_state[rx_queue_id] =
RTE_ETH_QUEUE_STATE_STARTED;
@@ -580,9 +901,11 @@ static const struct eth_dev_ops nthw_eth_dev_ops = {
.link_update = eth_link_update,
.dev_infos_get = eth_dev_infos_get,
.fw_version_get = eth_fw_version_get,
+ .rx_queue_setup = eth_rx_scg_queue_setup,
.rx_queue_start = eth_rx_queue_start,
.rx_queue_stop = eth_rx_queue_stop,
.rx_queue_release = eth_rx_queue_release,
+ .tx_queue_setup = eth_tx_scg_queue_setup,
.tx_queue_start = eth_tx_queue_start,
.tx_queue_stop = eth_tx_queue_stop,
.tx_queue_release = eth_tx_queue_release,
--
2.45.0
