From: Neel Patel <neel.pa...@amd.com>
They can be batched together this way, reducing the number
of PCIe transactions. This improves transmit PPS by up to 50% in
some configurations.
Signed-off-by: Andrew Boyer <andrew.bo...@amd.com>
Signed-off-by: Neel Patel <neel.pa...@amd.com>
---
drivers/net/ionic/ionic_dev.c | 9 +++--
drivers/net/ionic/ionic_dev.h | 6 ++-
drivers/net/ionic/ionic_lif.c | 26 +++++++++----
drivers/net/ionic/ionic_rxtx.h | 56 +++++++++++++++++++++++++++
drivers/net/ionic/ionic_rxtx_sg.c | 18 ++++-----
drivers/net/ionic/ionic_rxtx_simple.c | 18 ++++-----
6 files changed, 101 insertions(+), 32 deletions(-)
diff --git a/drivers/net/ionic/ionic_dev.c b/drivers/net/ionic/ionic_dev.c
index 70c14882ed..7f15914f74 100644
--- a/drivers/net/ionic/ionic_dev.c
+++ b/drivers/net/ionic/ionic_dev.c
@@ -369,17 +369,19 @@ ionic_q_init(struct ionic_queue *q, uint32_t index,
uint16_t num_descs)
q->index = index;
q->num_descs = num_descs;
q->size_mask = num_descs - 1;
- q->head_idx = 0;
- q->tail_idx = 0;
+ ionic_q_reset(q);
return 0;
}
void
-ionic_q_map(struct ionic_queue *q, void *base, rte_iova_t base_pa)
+ionic_q_map(struct ionic_queue *q, void *base, rte_iova_t base_pa,
+ void *cmb_base, rte_iova_t cmb_base_pa)
{
q->base = base;
q->base_pa = base_pa;
+ q->cmb_base = cmb_base;
+ q->cmb_base_pa = cmb_base_pa;
}
void
@@ -393,5 +395,6 @@ void
ionic_q_reset(struct ionic_queue *q)
{
q->head_idx = 0;
+ q->cmb_head_idx = 0;
q->tail_idx = 0;
}
diff --git a/drivers/net/ionic/ionic_dev.h b/drivers/net/ionic/ionic_dev.h
index 971c261b27..3a366247f1 100644
--- a/drivers/net/ionic/ionic_dev.h
+++ b/drivers/net/ionic/ionic_dev.h
@@ -145,11 +145,13 @@ struct ionic_queue {
uint16_t num_descs;
uint16_t num_segs;
uint16_t head_idx;
+ uint16_t cmb_head_idx;
uint16_t tail_idx;
uint16_t size_mask;
uint8_t type;
uint8_t hw_type;
void *base;
+ void *cmb_base;
void *sg_base;
struct ionic_doorbell __iomem *db;
void **info;
@@ -158,6 +160,7 @@ struct ionic_queue {
uint32_t hw_index;
rte_iova_t base_pa;
rte_iova_t sg_base_pa;
+ rte_iova_t cmb_base_pa;
};
#define IONIC_INTR_NONE (-1)
@@ -244,7 +247,8 @@ uint32_t ionic_cq_service(struct ionic_cq *cq, uint32_t
work_to_do,
int ionic_q_init(struct ionic_queue *q, uint32_t index, uint16_t num_descs);
void ionic_q_reset(struct ionic_queue *q);
-void ionic_q_map(struct ionic_queue *q, void *base, rte_iova_t base_pa);
+void ionic_q_map(struct ionic_queue *q, void *base, rte_iova_t base_pa,
+ void *cmb_base, rte_iova_t cmb_base_pa);
void ionic_q_sg_map(struct ionic_queue *q, void *base, rte_iova_t base_pa);
static inline uint16_t
diff --git a/drivers/net/ionic/ionic_lif.c b/drivers/net/ionic/ionic_lif.c
index fe2112c057..2713f8aa24 100644
--- a/drivers/net/ionic/ionic_lif.c
+++ b/drivers/net/ionic/ionic_lif.c
@@ -572,10 +572,11 @@ ionic_qcq_alloc(struct ionic_lif *lif,
{
struct ionic_qcq *new;
uint32_t q_size, cq_size, sg_size, total_size;
- void *q_base, *cq_base, *sg_base;
+ void *q_base, *cmb_q_base, *cq_base, *sg_base;
rte_iova_t q_base_pa = 0;
rte_iova_t cq_base_pa = 0;
rte_iova_t sg_base_pa = 0;
+ rte_iova_t cmb_q_base_pa = 0;
size_t page_size = rte_mem_page_size();
int err;
@@ -666,19 +667,22 @@ ionic_qcq_alloc(struct ionic_lif *lif,
IONIC_PRINT(ERR, "Cannot reserve queue from NIC mem");
return -ENOMEM;
}
- q_base = (void *)
+ cmb_q_base = (void *)
((uintptr_t)lif->adapter->bars.bar[2].vaddr +
(uintptr_t)lif->adapter->cmb_offset);
/* CMB PA is a relative address */
- q_base_pa = lif->adapter->cmb_offset;
+ cmb_q_base_pa = lif->adapter->cmb_offset;
lif->adapter->cmb_offset += q_size;
+ } else {
+ cmb_q_base = NULL;
+ cmb_q_base_pa = 0;
}
IONIC_PRINT(DEBUG, "Q-Base-PA = %#jx CQ-Base-PA = %#jx "
"SG-base-PA = %#jx",
q_base_pa, cq_base_pa, sg_base_pa);
- ionic_q_map(&new->q, q_base, q_base_pa);
+ ionic_q_map(&new->q, q_base, q_base_pa, cmb_q_base, cmb_q_base_pa);
ionic_cq_map(&new->cq, cq_base, cq_base_pa);
*qcq = new;
@@ -1583,7 +1587,6 @@ ionic_lif_txq_init(struct ionic_tx_qcq *txq)
.flags = rte_cpu_to_le_16(IONIC_QINIT_F_ENA),
.intr_index = rte_cpu_to_le_16(IONIC_INTR_NONE),
.ring_size = rte_log2_u32(q->num_descs),
- .ring_base = rte_cpu_to_le_64(q->base_pa),
.cq_ring_base = rte_cpu_to_le_64(cq->base_pa),
.sg_ring_base = rte_cpu_to_le_64(q->sg_base_pa),
},
@@ -1592,8 +1595,12 @@ ionic_lif_txq_init(struct ionic_tx_qcq *txq)
if (txq->flags & IONIC_QCQ_F_SG)
ctx.cmd.q_init.flags |= rte_cpu_to_le_16(IONIC_QINIT_F_SG);
- if (txq->flags & IONIC_QCQ_F_CMB)
+ if (txq->flags & IONIC_QCQ_F_CMB) {
ctx.cmd.q_init.flags |= rte_cpu_to_le_16(IONIC_QINIT_F_CMB);
+ ctx.cmd.q_init.ring_base = rte_cpu_to_le_64(q->cmb_base_pa);
+ } else {
+ ctx.cmd.q_init.ring_base = rte_cpu_to_le_64(q->base_pa);
+ }
IONIC_PRINT(DEBUG, "txq_init.index %d", q->index);
IONIC_PRINT(DEBUG, "txq_init.ring_base 0x%" PRIx64 "", q->base_pa);
@@ -1638,7 +1645,6 @@ ionic_lif_rxq_init(struct ionic_rx_qcq *rxq)
.flags = rte_cpu_to_le_16(IONIC_QINIT_F_ENA),
.intr_index = rte_cpu_to_le_16(IONIC_INTR_NONE),
.ring_size = rte_log2_u32(q->num_descs),
- .ring_base = rte_cpu_to_le_64(q->base_pa),
.cq_ring_base = rte_cpu_to_le_64(cq->base_pa),
.sg_ring_base = rte_cpu_to_le_64(q->sg_base_pa),
},
@@ -1647,8 +1653,12 @@ ionic_lif_rxq_init(struct ionic_rx_qcq *rxq)
if (rxq->flags & IONIC_QCQ_F_SG)
ctx.cmd.q_init.flags |= rte_cpu_to_le_16(IONIC_QINIT_F_SG);
- if (rxq->flags & IONIC_QCQ_F_CMB)
+ if (rxq->flags & IONIC_QCQ_F_CMB) {
ctx.cmd.q_init.flags |= rte_cpu_to_le_16(IONIC_QINIT_F_CMB);
+ ctx.cmd.q_init.ring_base = rte_cpu_to_le_64(q->cmb_base_pa);
+ } else {
+ ctx.cmd.q_init.ring_base = rte_cpu_to_le_64(q->base_pa);
+ }
IONIC_PRINT(DEBUG, "rxq_init.index %d", q->index);
IONIC_PRINT(DEBUG, "rxq_init.ring_base 0x%" PRIx64 "", q->base_pa);
diff --git a/drivers/net/ionic/ionic_rxtx.h b/drivers/net/ionic/ionic_rxtx.h
index 8537141597..5348395956 100644
--- a/drivers/net/ionic/ionic_rxtx.h
+++ b/drivers/net/ionic/ionic_rxtx.h
@@ -77,4 +77,60 @@ uint16_t ionic_xmit_pkts_sg(void *tx_queue, struct rte_mbuf
**tx_pkts,
int ionic_rx_fill_sg(struct ionic_rx_qcq *rxq);
+static inline void
+ionic_rxq_flush(struct ionic_queue *q)
+{
+ struct ionic_rxq_desc *desc_base = q->base;
+ struct ionic_rxq_desc *cmb_desc_base = q->cmb_base;
+
+ if (q->cmb_base) {
+ if (q->head_idx < q->cmb_head_idx) {
+ /* copy [cmb_head, num_descs) */
+ rte_memcpy((void *)&cmb_desc_base[q->cmb_head_idx],
+ (void *)&desc_base[q->cmb_head_idx],
+ (q->num_descs - q->cmb_head_idx) *
sizeof(*desc_base));
+ /* copy [0, head) */
+ rte_memcpy((void *)&cmb_desc_base[0],
+ (void *)&desc_base[0],
+ q->head_idx * sizeof(*desc_base));
+ } else {
+ /* copy [cmb_head, head) */
+ rte_memcpy((void *)&cmb_desc_base[q->cmb_head_idx],
+ (void *)&desc_base[q->cmb_head_idx],
+ (q->head_idx - q->cmb_head_idx) *
sizeof(*desc_base));
+ }
+ q->cmb_head_idx = q->head_idx;
+ }
+
+ ionic_q_flush(q);
+}
+
+static inline void
+ionic_txq_flush(struct ionic_queue *q)
+{
+ struct ionic_txq_desc *desc_base = q->base;
+ struct ionic_txq_desc *cmb_desc_base = q->cmb_base;
+
+ if (q->cmb_base) {
+ if (q->head_idx < q->cmb_head_idx) {
+ /* copy [cmb_head, num_descs) */
+ rte_memcpy((void *)&cmb_desc_base[q->cmb_head_idx],
+ (void *)&desc_base[q->cmb_head_idx],
+ (q->num_descs - q->cmb_head_idx) *
sizeof(*desc_base));
+ /* copy [0, head) */
+ rte_memcpy((void *)&cmb_desc_base[0],
+ (void *)&desc_base[0],
+ q->head_idx * sizeof(*desc_base));
+ } else {
+ /* copy [cmb_head, head) */
+ rte_memcpy((void *)&cmb_desc_base[q->cmb_head_idx],
+ (void *)&desc_base[q->cmb_head_idx],
+ (q->head_idx - q->cmb_head_idx) *
sizeof(*desc_base));
+ }
+ q->cmb_head_idx = q->head_idx;
+ }
+
+ ionic_q_flush(q);
+}
+
#endif /* _IONIC_RXTX_H_ */
diff --git a/drivers/net/ionic/ionic_rxtx_sg.c
b/drivers/net/ionic/ionic_rxtx_sg.c
index 1392342463..92e1d6e259 100644
--- a/drivers/net/ionic/ionic_rxtx_sg.c
+++ b/drivers/net/ionic/ionic_rxtx_sg.c
@@ -166,6 +166,7 @@ ionic_xmit_pkts_sg(void *tx_queue, struct rte_mbuf
**tx_pkts,
{
struct ionic_tx_qcq *txq = tx_queue;
struct ionic_queue *q = &txq->qcq.q;
+ struct ionic_txq_desc *desc_base = q->base;
struct ionic_tx_stats *stats = &txq->stats;
struct rte_mbuf *mbuf;
uint32_t bytes_tx = 0;
@@ -173,9 +174,7 @@ ionic_xmit_pkts_sg(void *tx_queue, struct rte_mbuf
**tx_pkts,
uint64_t then, now, hz, delta;
int err;
- struct ionic_txq_desc *desc_base = q->base;
- if (!(txq->flags & IONIC_QCQ_F_CMB))
- rte_prefetch0(&desc_base[q->head_idx]);
+ rte_prefetch0(&desc_base[q->head_idx]);
rte_prefetch0(IONIC_INFO_PTR(q, q->head_idx));
if (nb_pkts) {
@@ -196,8 +195,7 @@ ionic_xmit_pkts_sg(void *tx_queue, struct rte_mbuf
**tx_pkts,
while (nb_tx < nb_pkts) {
uint16_t next_idx = Q_NEXT_TO_POST(q, 1);
- if (!(txq->flags & IONIC_QCQ_F_CMB))
- rte_prefetch0(&desc_base[next_idx]);
+ rte_prefetch0(&desc_base[next_idx]);
rte_prefetch0(IONIC_INFO_PTR(q, next_idx));
if (nb_tx + 1 < nb_pkts) {
@@ -222,7 +220,7 @@ ionic_xmit_pkts_sg(void *tx_queue, struct rte_mbuf
**tx_pkts,
if (nb_tx > 0) {
rte_wmb();
- ionic_q_flush(q);
+ ionic_txq_flush(q);
txq->last_wdog_cycles = rte_get_timer_cycles();
@@ -458,8 +456,7 @@ ionic_rxq_service_sg(struct ionic_rx_qcq *rxq, uint32_t
work_to_do,
/* Prefetch 4 x 16B comp */
rte_prefetch0(&cq_desc_base[Q_NEXT_TO_SRVC(cq, 4)]);
/* Prefetch 4 x 16B descriptors */
- if (!(rxq->flags & IONIC_QCQ_F_CMB))
- rte_prefetch0(&q_desc_base[Q_NEXT_TO_POST(q, 4)]);
+ rte_prefetch0(&q_desc_base[Q_NEXT_TO_POST(q, 4)]);
/* Clean one descriptor */
ionic_rx_clean_one_sg(rxq, cq_desc, rx_svc);
@@ -478,7 +475,8 @@ ionic_rxq_service_sg(struct ionic_rx_qcq *rxq, uint32_t
work_to_do,
/* Update the queue indices and ring the doorbell */
if (work_done) {
- ionic_q_flush(q);
+ ionic_rxq_flush(q);
+
rxq->last_wdog_cycles = rte_get_timer_cycles();
rxq->wdog_ms = IONIC_Q_WDOG_MS;
} else {
@@ -542,7 +540,7 @@ ionic_rx_fill_sg(struct ionic_rx_qcq *rxq)
q->head_idx = Q_NEXT_TO_POST(q, 1);
}
- ionic_q_flush(q);
+ ionic_rxq_flush(q);
return err;
}
diff --git a/drivers/net/ionic/ionic_rxtx_simple.c
b/drivers/net/ionic/ionic_rxtx_simple.c
index 00152c885a..f12f66f40c 100644
--- a/drivers/net/ionic/ionic_rxtx_simple.c
+++ b/drivers/net/ionic/ionic_rxtx_simple.c
@@ -139,6 +139,7 @@ ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
{
struct ionic_tx_qcq *txq = tx_queue;
struct ionic_queue *q = &txq->qcq.q;
+ struct ionic_txq_desc *desc_base = q->base;
struct ionic_tx_stats *stats = &txq->stats;
struct rte_mbuf *mbuf;
uint32_t bytes_tx = 0;
@@ -146,9 +147,7 @@ ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
uint64_t then, now, hz, delta;
int err;
- struct ionic_txq_desc *desc_base = q->base;
- if (!(txq->flags & IONIC_QCQ_F_CMB))
- rte_prefetch0(&desc_base[q->head_idx]);
+ rte_prefetch0(&desc_base[q->head_idx]);
rte_prefetch0(&q->info[q->head_idx]);
if (nb_pkts) {
@@ -169,8 +168,7 @@ ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
while (nb_tx < nb_pkts) {
uint16_t next_idx = Q_NEXT_TO_POST(q, 1);
- if (!(txq->flags & IONIC_QCQ_F_CMB))
- rte_prefetch0(&desc_base[next_idx]);
+ rte_prefetch0(&desc_base[next_idx]);
rte_prefetch0(&q->info[next_idx]);
if (nb_tx + 1 < nb_pkts) {
@@ -195,7 +193,7 @@ ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
if (nb_tx > 0) {
rte_wmb();
- ionic_q_flush(q);
+ ionic_txq_flush(q);
txq->last_wdog_cycles = rte_get_timer_cycles();
@@ -379,8 +377,7 @@ ionic_rxq_service(struct ionic_rx_qcq *rxq, uint32_t
work_to_do,
/* Prefetch 4 x 16B comp */
rte_prefetch0(&cq_desc_base[Q_NEXT_TO_SRVC(cq, 4)]);
/* Prefetch 4 x 16B descriptors */
- if (!(rxq->flags & IONIC_QCQ_F_CMB))
- rte_prefetch0(&q_desc_base[Q_NEXT_TO_POST(q, 4)]);
+ rte_prefetch0(&q_desc_base[Q_NEXT_TO_POST(q, 4)]);
/* Clean one descriptor */
ionic_rx_clean_one(rxq, cq_desc, rx_svc);
@@ -399,7 +396,8 @@ ionic_rxq_service(struct ionic_rx_qcq *rxq, uint32_t
work_to_do,
/* Update the queue indices and ring the doorbell */
if (work_done) {
- ionic_q_flush(q);
+ ionic_rxq_flush(q);
+
rxq->last_wdog_cycles = rte_get_timer_cycles();
rxq->wdog_ms = IONIC_Q_WDOG_MS;
} else {
@@ -463,7 +461,7 @@ ionic_rx_fill(struct ionic_rx_qcq *rxq)
q->head_idx = Q_NEXT_TO_POST(q, 1);
}
- ionic_q_flush(q);
+ ionic_rxq_flush(q);
return err;
}
--
2.17.1
