Optimizations and cleanup:
- flatten packet send path
- flatten mbuf free path
- disable CQ entry writing and use CQ messages instead
- use rte_mempool_put_bulk() to bulk return freed mbufs
- remove unnecessary fields vnic_bufs struct, use contiguous array of cache
aligned divisible elements. No next pointers.
- use local variables inside per packet loop instead of fields in structs.
- factor book keeping out of the per packet tx loop where possible
(removed several conditionals)
- put Tx and Rx code in 1 file (enic_rxtx.c)
Reviewed-by: Nelson Escobar <neescoba at cisco.com>
Signed-off-by: John Daley <johndale at cisco.com>
---
drivers/net/enic/Makefile | 2 +-
drivers/net/enic/base/enic_vnic_wq.h | 79 ------
drivers/net/enic/base/vnic_cq.h | 37 +--
drivers/net/enic/base/vnic_rq.h | 2 +-
drivers/net/enic/base/vnic_wq.c | 89 +++---
drivers/net/enic/base/vnic_wq.h | 113 +-------
drivers/net/enic/enic.h | 27 +-
drivers/net/enic/enic_ethdev.c | 67 +----
drivers/net/enic/enic_main.c | 132 +++------
drivers/net/enic/enic_res.h | 81 +-----
drivers/net/enic/enic_rx.c | 361 -------------------------
drivers/net/enic/enic_rxtx.c | 505 +++++++++++++++++++++++++++++++++++
12 files changed, 635 insertions(+), 860 deletions(-)
delete mode 100644 drivers/net/enic/base/enic_vnic_wq.h
delete mode 100644 drivers/net/enic/enic_rx.c
create mode 100644 drivers/net/enic/enic_rxtx.c
diff --git a/drivers/net/enic/Makefile b/drivers/net/enic/Makefile
index f316274..3926b79 100644
--- a/drivers/net/enic/Makefile
+++ b/drivers/net/enic/Makefile
@@ -53,7 +53,7 @@ VPATH += $(SRCDIR)/src
#
SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_ethdev.c
SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_main.c
-SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_rx.c
+SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_rxtx.c
SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_clsf.c
SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_res.c
SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += base/vnic_cq.c
diff --git a/drivers/net/enic/base/enic_vnic_wq.h
b/drivers/net/enic/base/enic_vnic_wq.h
deleted file mode 100644
index b019109..0000000
--- a/drivers/net/enic/base/enic_vnic_wq.h
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Copyright 2008-2015 Cisco Systems, Inc. All rights reserved.
- * Copyright 2007 Nuova Systems, Inc. All rights reserved.
- *
- * Copyright (c) 2015, Cisco Systems, Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
- * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
- * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
- * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
- * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-#ifndef _ENIC_VNIC_WQ_H_
-#define _ENIC_VNIC_WQ_H_
-
-#include "vnic_dev.h"
-#include "vnic_cq.h"
-
-static inline void enic_vnic_post_wq_index(struct vnic_wq *wq)
-{
- struct vnic_wq_buf *buf = wq->to_use;
-
- /* Adding write memory barrier prevents compiler and/or CPU
- * reordering, thus avoiding descriptor posting before
- * descriptor is initialized. Otherwise, hardware can read
- * stale descriptor fields.
- */
- wmb();
- iowrite32(buf->index, &wq->ctrl->posted_index);
-}
-
-static inline void enic_vnic_post_wq(struct vnic_wq *wq,
- void *os_buf, dma_addr_t dma_addr,
- unsigned int len, int sop,
- uint8_t desc_skip_cnt, uint8_t cq_entry,
- uint8_t compressed_send, uint64_t wrid)
-{
- struct vnic_wq_buf *buf = wq->to_use;
-
- buf->sop = sop;
- buf->cq_entry = cq_entry;
- buf->compressed_send = compressed_send;
- buf->desc_skip_cnt = desc_skip_cnt;
- buf->os_buf = os_buf;
- buf->dma_addr = dma_addr;
- buf->len = len;
- buf->wr_id = wrid;
-
- buf = buf->next;
- wq->ring.desc_avail -= desc_skip_cnt;
- wq->to_use = buf;
-
- if (cq_entry)
- enic_vnic_post_wq_index(wq);
-}
-
-#endif /* _ENIC_VNIC_WQ_H_ */
diff --git a/drivers/net/enic/base/vnic_cq.h b/drivers/net/enic/base/vnic_cq.h
index 922391b..ffc1aaa 100644
--- a/drivers/net/enic/base/vnic_cq.h
+++ b/drivers/net/enic/base/vnic_cq.h
@@ -96,41 +96,46 @@ static inline unsigned int vnic_cq_service(struct vnic_cq
*cq,
u8 type, u16 q_number, u16 completed_index, void *opaque),
void *opaque)
{
- struct cq_desc *cq_desc;
+ struct cq_desc *cq_desc, *cq_desc_last;
unsigned int work_done = 0;
u16 q_number, completed_index;
- u8 type, color;
- struct rte_mbuf **rx_pkts = opaque;
- unsigned int ret;
+ u8 type, color, type_color;
cq_desc = (struct cq_desc *)((u8 *)cq->ring.descs +
cq->ring.desc_size * cq->to_clean);
- cq_desc_dec(cq_desc, &type, &color,
- &q_number, &completed_index);
+
+ type_color = cq_desc->type_color;
+ color = (type_color >> CQ_DESC_COLOR_SHIFT) & CQ_DESC_COLOR_MASK;
+ if (color == cq->last_color)
+ return 0;
while (color != cq->last_color) {
- if (opaque)
- opaque = (void *)&(rx_pkts[work_done]);
+ cq_desc_last = cq_desc;
- ret = (*q_service)(cq->vdev, cq_desc, type,
- q_number, completed_index, opaque);
cq->to_clean++;
if (cq->to_clean == cq->ring.desc_count) {
cq->to_clean = 0;
cq->last_color = cq->last_color ? 0 : 1;
}
+ work_done++;
+ if (work_done >= work_to_do)
+ break;
+
cq_desc = (struct cq_desc *)((u8 *)cq->ring.descs +
cq->ring.desc_size * cq->to_clean);
- cq_desc_dec(cq_desc, &type, &color,
- &q_number, &completed_index);
- if (ret)
- work_done++;
- if (work_done >= work_to_do)
- break;
+ type_color = cq_desc->type_color;
+ color = (type_color >> CQ_DESC_COLOR_SHIFT)
+ & CQ_DESC_COLOR_MASK;
+
}
+ cq_desc_dec(cq_desc_last, &type, &color,
+ &q_number, &completed_index);
+
+ (*q_service)(cq->vdev, cq_desc, type,
+ q_number, completed_index, opaque);
return work_done;
}
diff --git a/drivers/net/enic/base/vnic_rq.h b/drivers/net/enic/base/vnic_rq.h
index e083ccc..424415c 100644
--- a/drivers/net/enic/base/vnic_rq.h
+++ b/drivers/net/enic/base/vnic_rq.h
@@ -74,7 +74,7 @@ struct vnic_rq {
struct vnic_dev_ring ring;
struct rte_mbuf **mbuf_ring; /* array of allocated mbufs */
unsigned int mbuf_next_idx; /* next mb to consume */
- void *os_buf_head;
+ void *mb_head;
unsigned int pkts_outstanding;
uint16_t rx_nb_hold;
uint16_t rx_free_thresh;
diff --git a/drivers/net/enic/base/vnic_wq.c b/drivers/net/enic/base/vnic_wq.c
index a3ef417..ccbbd61 100644
--- a/drivers/net/enic/base/vnic_wq.c
+++ b/drivers/net/enic/base/vnic_wq.c
@@ -59,71 +59,30 @@ int vnic_wq_alloc_ring(struct vnic_dev *vdev, struct
vnic_wq *wq,
static int vnic_wq_alloc_bufs(struct vnic_wq *wq)
{
- struct vnic_wq_buf *buf;
- unsigned int i, j, count = wq->ring.desc_count;
- unsigned int blks = VNIC_WQ_BUF_BLKS_NEEDED(count);
-
- for (i = 0; i < blks; i++) {
- wq->bufs[i] = kzalloc(VNIC_WQ_BUF_BLK_SZ(count), GFP_ATOMIC);
- if (!wq->bufs[i])
- return -ENOMEM;
- }
-
- for (i = 0; i < blks; i++) {
- buf = wq->bufs[i];
- for (j = 0; j < VNIC_WQ_BUF_BLK_ENTRIES(count); j++) {
- buf->index = i * VNIC_WQ_BUF_BLK_ENTRIES(count) + j;
- buf->desc = (u8 *)wq->ring.descs +
- wq->ring.desc_size * buf->index;
- if (buf->index + 1 == count) {
- buf->next = wq->bufs[0];
- break;
- } else if (j + 1 == VNIC_WQ_BUF_BLK_ENTRIES(count)) {
- buf->next = wq->bufs[i + 1];
- } else {
- buf->next = buf + 1;
- buf++;
- }
- }
- }
-
- wq->to_use = wq->to_clean = wq->bufs[0];
-
+ unsigned int count = wq->ring.desc_count;
+ /* Allocate the mbuf ring */
+ wq->bufs = (struct vnic_wq_buf *)rte_zmalloc_socket("wq->bufs",
+ sizeof(struct vnic_wq_buf) * count,
+ RTE_CACHE_LINE_SIZE, wq->socket_id);
+ wq->head_idx = 0;
+ wq->tail_idx = 0;
+ if (wq->bufs == NULL)
+ return -ENOMEM;
return 0;
}
void vnic_wq_free(struct vnic_wq *wq)
{
struct vnic_dev *vdev;
- unsigned int i;
vdev = wq->vdev;
vnic_dev_free_desc_ring(vdev, &wq->ring);
- for (i = 0; i < VNIC_WQ_BUF_BLKS_MAX; i++) {
- if (wq->bufs[i]) {
- kfree(wq->bufs[i]);
- wq->bufs[i] = NULL;
- }
- }
-
+ rte_free(wq->bufs);
wq->ctrl = NULL;
}
-int vnic_wq_mem_size(struct vnic_wq *wq, unsigned int desc_count,
- unsigned int desc_size)
-{
- int mem_size = 0;
-
- mem_size += vnic_dev_desc_ring_size(&wq->ring, desc_count, desc_size);
-
- mem_size += VNIC_WQ_BUF_BLKS_NEEDED(wq->ring.desc_count) *
- VNIC_WQ_BUF_BLK_SZ(wq->ring.desc_count);
-
- return mem_size;
-}
-
int vnic_wq_alloc(struct vnic_dev *vdev, struct vnic_wq *wq, unsigned int
index,
unsigned int desc_count, unsigned int desc_size)
@@ -172,9 +131,8 @@ void vnic_wq_init_start(struct vnic_wq *wq, unsigned int
cq_index,
iowrite32(error_interrupt_offset, &wq->ctrl->error_interrupt_offset);
iowrite32(0, &wq->ctrl->error_status);
- wq->to_use = wq->to_clean =
- &wq->bufs[fetch_index / VNIC_WQ_BUF_BLK_ENTRIES(count)]
- [fetch_index % VNIC_WQ_BUF_BLK_ENTRIES(count)];
+ wq->head_idx = fetch_index;
+ wq->tail_idx = wq->head_idx;
}
void vnic_wq_init(struct vnic_wq *wq, unsigned int cq_index,
@@ -184,6 +142,7 @@ void vnic_wq_init(struct vnic_wq *wq, unsigned int cq_index,
vnic_wq_init_start(wq, cq_index, 0, 0,
error_interrupt_enable,
error_interrupt_offset);
+ wq->last_completed_index = 0;
}
void vnic_wq_error_out(struct vnic_wq *wq, unsigned int error)
@@ -219,22 +178,34 @@ int vnic_wq_disable(struct vnic_wq *wq)
return -ETIMEDOUT;
}
+static inline uint32_t
+buf_idx_incr(uint32_t n_descriptors, uint32_t idx)
+{
+ idx++;
+ if (unlikely(idx == n_descriptors))
+ idx = 0;
+ return idx;
+}
+
void vnic_wq_clean(struct vnic_wq *wq,
- void (*buf_clean)(struct vnic_wq *wq, struct vnic_wq_buf *buf))
+ void (*buf_clean)(struct vnic_wq_buf *buf))
{
struct vnic_wq_buf *buf;
+ unsigned int to_clean = wq->tail_idx;
- buf = wq->to_clean;
+ buf = &wq->bufs[to_clean];
while (vnic_wq_desc_used(wq) > 0) {
- (*buf_clean)(wq, buf);
+ (*buf_clean)(buf);
+ to_clean = buf_idx_incr(wq->ring.desc_count, to_clean);
- buf = wq->to_clean = buf->next;
+ buf = &wq->bufs[to_clean];
wq->ring.desc_avail++;
}
- wq->to_use = wq->to_clean = wq->bufs[0];
+ wq->head_idx = 0;
+ wq->tail_idx = 0;
iowrite32(0, &wq->ctrl->fetch_index);
iowrite32(0, &wq->ctrl->posted_index);
diff --git a/drivers/net/enic/base/vnic_wq.h b/drivers/net/enic/base/vnic_wq.h
index c23de62..37c3ff9 100644
--- a/drivers/net/enic/base/vnic_wq.h
+++ b/drivers/net/enic/base/vnic_wq.h
@@ -64,42 +64,23 @@ struct vnic_wq_ctrl {
u32 pad9;
};
+/* 16 bytes */
struct vnic_wq_buf {
- struct vnic_wq_buf *next;
- dma_addr_t dma_addr;
- void *os_buf;
- unsigned int len;
- unsigned int index;
- int sop;
- void *desc;
- uint64_t wr_id; /* Cookie */
- uint8_t cq_entry; /* Gets completion event from hw */
- uint8_t desc_skip_cnt; /* Num descs to occupy */
- uint8_t compressed_send; /* Both hdr and payload in one desc */
+ struct rte_mempool *pool;
+ void *mb;
};
-/* Break the vnic_wq_buf allocations into blocks of 32/64 entries */
-#define VNIC_WQ_BUF_MIN_BLK_ENTRIES 32
-#define VNIC_WQ_BUF_DFLT_BLK_ENTRIES 64
-#define VNIC_WQ_BUF_BLK_ENTRIES(entries) \
- ((unsigned int)((entries < VNIC_WQ_BUF_DFLT_BLK_ENTRIES) ? \
- VNIC_WQ_BUF_MIN_BLK_ENTRIES : VNIC_WQ_BUF_DFLT_BLK_ENTRIES))
-#define VNIC_WQ_BUF_BLK_SZ(entries) \
- (VNIC_WQ_BUF_BLK_ENTRIES(entries) * sizeof(struct vnic_wq_buf))
-#define VNIC_WQ_BUF_BLKS_NEEDED(entries) \
- DIV_ROUND_UP(entries, VNIC_WQ_BUF_BLK_ENTRIES(entries))
-#define VNIC_WQ_BUF_BLKS_MAX VNIC_WQ_BUF_BLKS_NEEDED(4096)
-
struct vnic_wq {
unsigned int index;
struct vnic_dev *vdev;
struct vnic_wq_ctrl __iomem *ctrl; /* memory-mapped */
struct vnic_dev_ring ring;
- struct vnic_wq_buf *bufs[VNIC_WQ_BUF_BLKS_MAX];
- struct vnic_wq_buf *to_use;
- struct vnic_wq_buf *to_clean;
- unsigned int pkts_outstanding;
+ struct vnic_wq_buf *bufs;
+ unsigned int head_idx;
+ unsigned int tail_idx;
unsigned int socket_id;
+ const struct rte_memzone *cqmsg_rz;
+ uint16_t last_completed_index;
};
static inline unsigned int vnic_wq_desc_avail(struct vnic_wq *wq)
@@ -114,11 +95,6 @@ static inline unsigned int vnic_wq_desc_used(struct vnic_wq
*wq)
return wq->ring.desc_count - wq->ring.desc_avail - 1;
}
-static inline void *vnic_wq_next_desc(struct vnic_wq *wq)
-{
- return wq->to_use->desc;
-}
-
#define PI_LOG2_CACHE_LINE_SIZE 5
#define PI_INDEX_BITS 12
#define PI_INDEX_MASK ((1U << PI_INDEX_BITS) - 1)
@@ -191,75 +167,6 @@ static inline u64 vnic_cached_posted_index(dma_addr_t
addr, unsigned int len,
PI_PREFETCH_ADDR_MASK) << PI_PREFETCH_ADDR_OFF);
}
-static inline void vnic_wq_post(struct vnic_wq *wq,
- void *os_buf, dma_addr_t dma_addr,
- unsigned int len, int sop, int eop,
- uint8_t desc_skip_cnt, uint8_t cq_entry,
- uint8_t compressed_send, uint64_t wrid)
-{
- struct vnic_wq_buf *buf = wq->to_use;
-
- buf->sop = sop;
- buf->cq_entry = cq_entry;
- buf->compressed_send = compressed_send;
- buf->desc_skip_cnt = desc_skip_cnt;
- buf->os_buf = os_buf;
- buf->dma_addr = dma_addr;
- buf->len = len;
- buf->wr_id = wrid;
-
- buf = buf->next;
- if (eop) {
-#ifdef DO_PREFETCH
- uint64_t wr = vnic_cached_posted_index(dma_addr, len,
- buf->index);
-#endif
- /* Adding write memory barrier prevents compiler and/or CPU
- * reordering, thus avoiding descriptor posting before
- * descriptor is initialized. Otherwise, hardware can read
- * stale descriptor fields.
- */
- wmb();
-#ifdef DO_PREFETCH
- /* Intel chipsets seem to limit the rate of PIOs that we can
- * push on the bus. Thus, it is very important to do a single
- * 64 bit write here. With two 32-bit writes, my maximum
- * pkt/sec rate was cut almost in half. -AJF
- */
- iowrite64((uint64_t)wr, &wq->ctrl->posted_index);
-#else
- iowrite32(buf->index, &wq->ctrl->posted_index);
-#endif
- }
- wq->to_use = buf;
-
- wq->ring.desc_avail -= desc_skip_cnt;
-}
-
-static inline void vnic_wq_service(struct vnic_wq *wq,
- struct cq_desc *cq_desc, u16 completed_index,
- void (*buf_service)(struct vnic_wq *wq,
- struct cq_desc *cq_desc, struct vnic_wq_buf *buf, void *opaque),
- void *opaque)
-{
- struct vnic_wq_buf *buf;
-
- buf = wq->to_clean;
- while (1) {
-
- (*buf_service)(wq, cq_desc, buf, opaque);
-
- wq->ring.desc_avail++;
-
- wq->to_clean = buf->next;
-
- if (buf->index == completed_index)
- break;
-
- buf = wq->to_clean;
- }
-}
-
void vnic_wq_free(struct vnic_wq *wq);
int vnic_wq_alloc(struct vnic_dev *vdev, struct vnic_wq *wq, unsigned int
index,
unsigned int desc_count, unsigned int desc_size);
@@ -275,8 +182,6 @@ unsigned int vnic_wq_error_status(struct vnic_wq *wq);
void vnic_wq_enable(struct vnic_wq *wq);
int vnic_wq_disable(struct vnic_wq *wq);
void vnic_wq_clean(struct vnic_wq *wq,
- void (*buf_clean)(struct vnic_wq *wq, struct vnic_wq_buf *buf));
-int vnic_wq_mem_size(struct vnic_wq *wq, unsigned int desc_count,
- unsigned int desc_size);
+ void (*buf_clean)(struct vnic_wq_buf *buf));
#endif /* _VNIC_WQ_H_ */
diff --git a/drivers/net/enic/enic.h b/drivers/net/enic/enic.h
index 8c914f5..43b82a6 100644
--- a/drivers/net/enic/enic.h
+++ b/drivers/net/enic/enic.h
@@ -155,6 +155,30 @@ static inline struct enic *pmd_priv(struct rte_eth_dev
*eth_dev)
return (struct enic *)eth_dev->data->dev_private;
}
+static inline uint32_t
+enic_ring_add(uint32_t n_descriptors, uint32_t i0, uint32_t i1)
+{
+ uint32_t d = i0 + i1;
+ d -= (d >= n_descriptors) ? n_descriptors : 0;
+ return d;
+}
+
+static inline uint32_t
+enic_ring_sub(uint32_t n_descriptors, uint32_t i0, uint32_t i1)
+{
+ int32_t d = i1 - i0;
+ return (uint32_t)((d < 0) ? ((int32_t)n_descriptors + d) : d);
+}
+
+static inline uint32_t
+enic_ring_incr(uint32_t n_descriptors, uint32_t idx)
+{
+ idx++;
+ if (unlikely(idx == n_descriptors))
+ idx = 0;
+ return idx;
+}
+
#define RTE_LIBRTE_ENIC_ASSERT_ENABLE
#ifdef RTE_LIBRTE_ENIC_ASSERT_ENABLE
#define ASSERT(x) do { \
@@ -209,5 +233,6 @@ extern int enic_clsf_init(struct enic *enic);
extern void enic_clsf_destroy(struct enic *enic);
uint16_t enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
uint16_t nb_pkts);
-
+uint16_t enic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
+ uint16_t nb_pkts);
#endif /* _ENIC_H_ */
diff --git a/drivers/net/enic/enic_ethdev.c b/drivers/net/enic/enic_ethdev.c
index 6bea940..697ff82 100644
--- a/drivers/net/enic/enic_ethdev.c
+++ b/drivers/net/enic/enic_ethdev.c
@@ -519,71 +519,6 @@ static void enicpmd_remove_mac_addr(struct rte_eth_dev
*eth_dev, __rte_unused ui
enic_del_mac_address(enic);
}
-
-static uint16_t enicpmd_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
- uint16_t nb_pkts)
-{
- uint16_t index;
- unsigned int frags;
- unsigned int pkt_len;
- unsigned int seg_len;
- unsigned int inc_len;
- unsigned int nb_segs;
- struct rte_mbuf *tx_pkt, *next_tx_pkt;
- struct vnic_wq *wq = (struct vnic_wq *)tx_queue;
- struct enic *enic = vnic_dev_priv(wq->vdev);
- unsigned short vlan_id;
- unsigned short ol_flags;
- uint8_t last_seg, eop;
- unsigned int host_tx_descs = 0;
-
- for (index = 0; index < nb_pkts; index++) {
- tx_pkt = *tx_pkts++;
- inc_len = 0;
- nb_segs = tx_pkt->nb_segs;
- if (nb_segs > vnic_wq_desc_avail(wq)) {
- if (index > 0)
- enic_post_wq_index(wq);
-
- /* wq cleanup and try again */
- if (!enic_cleanup_wq(enic, wq) ||
- (nb_segs > vnic_wq_desc_avail(wq))) {
- return index;
- }
- }
-
- pkt_len = tx_pkt->pkt_len;
- vlan_id = tx_pkt->vlan_tci;
- ol_flags = tx_pkt->ol_flags;
- for (frags = 0; inc_len < pkt_len; frags++) {
- if (!tx_pkt)
- break;
- next_tx_pkt = tx_pkt->next;
- seg_len = tx_pkt->data_len;
- inc_len += seg_len;
-
- host_tx_descs++;
- last_seg = 0;
- eop = 0;
- if ((pkt_len == inc_len) || !next_tx_pkt) {
- eop = 1;
- /* post if last packet in batch or > thresh */
- if ((index == (nb_pkts - 1)) ||
- (host_tx_descs > ENIC_TX_POST_THRESH)) {
- last_seg = 1;
- host_tx_descs = 0;
- }
- }
- enic_send_pkt(enic, wq, tx_pkt, (unsigned short)seg_len,
- !frags, eop, last_seg, ol_flags, vlan_id);
- tx_pkt = next_tx_pkt;
- }
- }
-
- enic_cleanup_wq(enic, wq);
- return index;
-}
-
static const struct eth_dev_ops enicpmd_eth_dev_ops = {
.dev_configure = enicpmd_dev_configure,
.dev_start = enicpmd_dev_start,
@@ -642,7 +577,7 @@ static int eth_enicpmd_dev_init(struct rte_eth_dev *eth_dev)
enic->rte_dev = eth_dev;
eth_dev->dev_ops = &enicpmd_eth_dev_ops;
eth_dev->rx_pkt_burst = &enic_recv_pkts;
- eth_dev->tx_pkt_burst = &enicpmd_xmit_pkts;
+ eth_dev->tx_pkt_burst = &enic_xmit_pkts;
pdev = eth_dev->pci_dev;
rte_eth_copy_pci_info(eth_dev, pdev);
diff --git a/drivers/net/enic/enic_main.c b/drivers/net/enic/enic_main.c
index 646d87f..ba73604 100644
--- a/drivers/net/enic/enic_main.c
+++ b/drivers/net/enic/enic_main.c
@@ -40,11 +40,11 @@
#include <libgen.h>
#include <rte_pci.h>
-#include <rte_memzone.h>
#include <rte_malloc.h>
#include <rte_mbuf.h>
#include <rte_string_fns.h>
#include <rte_ethdev.h>
+#include <rte_memzone.h>
#include "enic_compat.h"
#include "enic.h"
@@ -58,7 +58,6 @@
#include "vnic_cq.h"
#include "vnic_intr.h"
#include "vnic_nic.h"
-#include "enic_vnic_wq.h"
static inline struct rte_mbuf *
rte_rxmbuf_alloc(struct rte_mempool *mp)
@@ -109,38 +108,17 @@ enic_rxmbuf_queue_release(struct enic *enic, struct
vnic_rq *rq)
}
}
-
void enic_set_hdr_split_size(struct enic *enic, u16 split_hdr_size)
{
vnic_set_hdr_split_size(enic->vdev, split_hdr_size);
}
-static void enic_free_wq_buf(__rte_unused struct vnic_wq *wq, struct
vnic_wq_buf *buf)
+static void enic_free_wq_buf(struct vnic_wq_buf *buf)
{
- struct rte_mbuf *mbuf = (struct rte_mbuf *)buf->os_buf;
+ struct rte_mbuf *mbuf = (struct rte_mbuf *)buf->mb;
rte_mempool_put(mbuf->pool, mbuf);
- buf->os_buf = NULL;
-}
-
-static void enic_wq_free_buf(struct vnic_wq *wq,
- __rte_unused struct cq_desc *cq_desc,
- struct vnic_wq_buf *buf,
- __rte_unused void *opaque)
-{
- enic_free_wq_buf(wq, buf);
-}
-
-static int enic_wq_service(struct vnic_dev *vdev, struct cq_desc *cq_desc,
- __rte_unused u8 type, u16 q_number, u16 completed_index, void *opaque)
-{
- struct enic *enic = vnic_dev_priv(vdev);
-
- vnic_wq_service(&enic->wq[q_number], cq_desc,
- completed_index, enic_wq_free_buf,
- opaque);
-
- return 0;
+ buf->mb = NULL;
}
static void enic_log_q_error(struct enic *enic)
@@ -163,64 +141,6 @@ static void enic_log_q_error(struct enic *enic)
}
}
-unsigned int enic_cleanup_wq(struct enic *enic, struct vnic_wq *wq)
-{
- unsigned int cq = enic_cq_wq(enic, wq->index);
-
- /* Return the work done */
- return vnic_cq_service(&enic->cq[cq],
- -1 /*wq_work_to_do*/, enic_wq_service, NULL);
-}
-
-void enic_post_wq_index(struct vnic_wq *wq)
-{
- enic_vnic_post_wq_index(wq);
-}
-
-void enic_send_pkt(struct enic *enic, struct vnic_wq *wq,
- struct rte_mbuf *tx_pkt, unsigned short len,
- uint8_t sop, uint8_t eop, uint8_t cq_entry,
- uint16_t ol_flags, uint16_t vlan_tag)
-{
- struct wq_enet_desc *desc = vnic_wq_next_desc(wq);
- uint16_t mss = 0;
- uint8_t vlan_tag_insert = 0;
- uint64_t bus_addr = (dma_addr_t)
- (tx_pkt->buf_physaddr + tx_pkt->data_off);
-
- if (sop) {
- if (ol_flags & PKT_TX_VLAN_PKT)
- vlan_tag_insert = 1;
-
- if (enic->hw_ip_checksum) {
- if (ol_flags & PKT_TX_IP_CKSUM)
- mss |= ENIC_CALC_IP_CKSUM;
-
- if (ol_flags & PKT_TX_TCP_UDP_CKSUM)
- mss |= ENIC_CALC_TCP_UDP_CKSUM;
- }
- }
-
- wq_enet_desc_enc(desc,
- bus_addr,
- len,
- mss,
- 0 /* header_length */,
- 0 /* offload_mode WQ_ENET_OFFLOAD_MODE_CSUM */,
- eop,
- cq_entry,
- 0 /* fcoe_encap */,
- vlan_tag_insert,
- vlan_tag,
- 0 /* loopback */);
-
- enic_vnic_post_wq(wq, (void *)tx_pkt, bus_addr, len,
- sop,
- 1 /*desc_skip_cnt*/,
- cq_entry,
- 0 /*compressed send*/,
- 0 /*wrid*/);
-}
void enic_dev_stats_clear(struct enic *enic)
{
@@ -297,12 +217,28 @@ void enic_init_vnic_resources(struct enic *enic)
unsigned int error_interrupt_enable = 1;
unsigned int error_interrupt_offset = 0;
unsigned int index = 0;
+ unsigned int cq_idx;
+
+ vnic_dev_stats_clear(enic->vdev);
for (index = 0; index < enic->rq_count; index++) {
vnic_rq_init(&enic->rq[index],
enic_cq_rq(enic, index),
error_interrupt_enable,
error_interrupt_offset);
+
+ cq_idx = enic_cq_rq(enic, index);
+ vnic_cq_init(&enic->cq[cq_idx],
+ 0 /* flow_control_enable */,
+ 1 /* color_enable */,
+ 0 /* cq_head */,
+ 0 /* cq_tail */,
+ 1 /* cq_tail_color */,
+ 0 /* interrupt_enable */,
+ 1 /* cq_entry_enable */,
+ 0 /* cq_message_enable */,
+ 0 /* interrupt offset */,
+ 0 /* cq_message_addr */);
}
for (index = 0; index < enic->wq_count; index++) {
@@ -310,22 +246,19 @@ void enic_init_vnic_resources(struct enic *enic)
enic_cq_wq(enic, index),
error_interrupt_enable,
error_interrupt_offset);
- }
-
- vnic_dev_stats_clear(enic->vdev);
- for (index = 0; index < enic->cq_count; index++) {
- vnic_cq_init(&enic->cq[index],
+ cq_idx = enic_cq_wq(enic, index);
+ vnic_cq_init(&enic->cq[cq_idx],
0 /* flow_control_enable */,
1 /* color_enable */,
0 /* cq_head */,
0 /* cq_tail */,
1 /* cq_tail_color */,
0 /* interrupt_enable */,
- 1 /* cq_entry_enable */,
- 0 /* cq_message_enable */,
+ 0 /* cq_entry_enable */,
+ 1 /* cq_message_enable */,
0 /* interrupt offset */,
- 0 /* cq_message_addr */);
+ (u64)enic->wq[index].cqmsg_rz->phys_addr);
}
vnic_intr_init(&enic->intr,
@@ -569,6 +502,7 @@ void enic_free_wq(void *txq)
struct vnic_wq *wq = (struct vnic_wq *)txq;
struct enic *enic = vnic_dev_priv(wq->vdev);
+ rte_memzone_free(wq->cqmsg_rz);
vnic_wq_free(wq);
vnic_cq_free(&enic->cq[enic->rq_count + wq->index]);
}
@@ -579,6 +513,8 @@ int enic_alloc_wq(struct enic *enic, uint16_t queue_idx,
int err;
struct vnic_wq *wq = &enic->wq[queue_idx];
unsigned int cq_index = enic_cq_wq(enic, queue_idx);
+ char name[NAME_MAX];
+ static int instance;
wq->socket_id = socket_id;
if (nb_desc) {
@@ -614,6 +550,18 @@ int enic_alloc_wq(struct enic *enic, uint16_t queue_idx,
dev_err(enic, "error in allocation of cq for wq\n");
}
+ /* setup up CQ message */
+ snprintf((char *)name, sizeof(name),
+ "vnic_cqmsg-%s-%d-%d", enic->bdf_name, queue_idx,
+ instance++);
+
+ wq->cqmsg_rz = rte_memzone_reserve_aligned((const char *)name,
+ sizeof(uint32_t),
+ SOCKET_ID_ANY, 0,
+ ENIC_ALIGN);
+ if (!wq->cqmsg_rz)
+ return -ENOMEM;
+
return err;
}
diff --git a/drivers/net/enic/enic_res.h b/drivers/net/enic/enic_res.h
index 00fa71d..3e1bdf5 100644
--- a/drivers/net/enic/enic_res.h
+++ b/drivers/net/enic/enic_res.h
@@ -53,89 +53,10 @@
#define ENIC_NON_TSO_MAX_DESC 16
#define ENIC_DEFAULT_RX_FREE_THRESH 32
-#define ENIC_TX_POST_THRESH (ENIC_MIN_WQ_DESCS / 2)
+#define ENIC_TX_XMIT_MAX 64
#define ENIC_SETTING(enic, f) ((enic->config.flags & VENETF_##f) ? 1 : 0)
-static inline void enic_queue_wq_desc_ex(struct vnic_wq *wq,
- void *os_buf, dma_addr_t dma_addr, unsigned int len,
- unsigned int mss_or_csum_offset, unsigned int hdr_len,
- int vlan_tag_insert, unsigned int vlan_tag,
- int offload_mode, int cq_entry, int sop, int eop, int loopback)
-{
- struct wq_enet_desc *desc = vnic_wq_next_desc(wq);
- u8 desc_skip_cnt = 1;
- u8 compressed_send = 0;
- u64 wrid = 0;
-
- wq_enet_desc_enc(desc,
- (u64)dma_addr | VNIC_PADDR_TARGET,
- (u16)len,
- (u16)mss_or_csum_offset,
- (u16)hdr_len, (u8)offload_mode,
- (u8)eop, (u8)cq_entry,
- 0, /* fcoe_encap */
- (u8)vlan_tag_insert,
- (u16)vlan_tag,
- (u8)loopback);
-
- vnic_wq_post(wq, os_buf, dma_addr, len, sop, eop, desc_skip_cnt,
- (u8)cq_entry, compressed_send, wrid);
-}
-
-static inline void enic_queue_wq_desc_cont(struct vnic_wq *wq,
- void *os_buf, dma_addr_t dma_addr, unsigned int len,
- int eop, int loopback)
-{
- enic_queue_wq_desc_ex(wq, os_buf, dma_addr, len,
- 0, 0, 0, 0, 0,
- eop, 0 /* !SOP */, eop, loopback);
-}
-
-static inline void enic_queue_wq_desc(struct vnic_wq *wq, void *os_buf,
- dma_addr_t dma_addr, unsigned int len, int vlan_tag_insert,
- unsigned int vlan_tag, int eop, int loopback)
-{
- enic_queue_wq_desc_ex(wq, os_buf, dma_addr, len,
- 0, 0, vlan_tag_insert, vlan_tag,
- WQ_ENET_OFFLOAD_MODE_CSUM,
- eop, 1 /* SOP */, eop, loopback);
-}
-
-static inline void enic_queue_wq_desc_csum(struct vnic_wq *wq,
- void *os_buf, dma_addr_t dma_addr, unsigned int len,
- int ip_csum, int tcpudp_csum, int vlan_tag_insert,
- unsigned int vlan_tag, int eop, int loopback)
-{
- enic_queue_wq_desc_ex(wq, os_buf, dma_addr, len,
- (ip_csum ? 1 : 0) + (tcpudp_csum ? 2 : 0),
- 0, vlan_tag_insert, vlan_tag,
- WQ_ENET_OFFLOAD_MODE_CSUM,
- eop, 1 /* SOP */, eop, loopback);
-}
-
-static inline void enic_queue_wq_desc_csum_l4(struct vnic_wq *wq,
- void *os_buf, dma_addr_t dma_addr, unsigned int len,
- unsigned int csum_offset, unsigned int hdr_len,
- int vlan_tag_insert, unsigned int vlan_tag, int eop, int loopback)
-{
- enic_queue_wq_desc_ex(wq, os_buf, dma_addr, len,
- csum_offset, hdr_len, vlan_tag_insert, vlan_tag,
- WQ_ENET_OFFLOAD_MODE_CSUM_L4,
- eop, 1 /* SOP */, eop, loopback);
-}
-
-static inline void enic_queue_wq_desc_tso(struct vnic_wq *wq,
- void *os_buf, dma_addr_t dma_addr, unsigned int len,
- unsigned int mss, unsigned int hdr_len, int vlan_tag_insert,
- unsigned int vlan_tag, int eop, int loopback)
-{
- enic_queue_wq_desc_ex(wq, os_buf, dma_addr, len,
- mss, hdr_len, vlan_tag_insert, vlan_tag,
- WQ_ENET_OFFLOAD_MODE_TSO,
- eop, 1 /* SOP */, eop, loopback);
-}
-
struct enic;
int enic_get_vnic_config(struct enic *);
diff --git a/drivers/net/enic/enic_rx.c b/drivers/net/enic/enic_rx.c
deleted file mode 100644
index 39bb55c..0000000
--- a/drivers/net/enic/enic_rx.c
+++ /dev/null
@@ -1,361 +0,0 @@
-/*
- * Copyright 2008-2014 Cisco Systems, Inc. All rights reserved.
- * Copyright 2007 Nuova Systems, Inc. All rights reserved.
- *
- * Copyright (c) 2014, Cisco Systems, Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
- * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
- * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
- * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
- * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-#include <rte_mbuf.h>
-#include <rte_ethdev.h>
-#include <rte_prefetch.h>
-
-#include "enic_compat.h"
-#include "rq_enet_desc.h"
-#include "enic.h"
-
-#define RTE_PMD_USE_PREFETCH
-
-#ifdef RTE_PMD_USE_PREFETCH
-/*
- * Prefetch a cache line into all cache levels.
- */
-#define rte_enic_prefetch(p) rte_prefetch0(p)
-#else
-#define rte_enic_prefetch(p) do {} while (0)
-#endif
-
-#ifdef RTE_PMD_PACKET_PREFETCH
-#define rte_packet_prefetch(p) rte_prefetch1(p)
-#else
-#define rte_packet_prefetch(p) do {} while (0)
-#endif
-
-static inline struct rte_mbuf *
-rte_rxmbuf_alloc(struct rte_mempool *mp)
-{
- struct rte_mbuf *m;
-
- m = __rte_mbuf_raw_alloc(mp);
- __rte_mbuf_sanity_check_raw(m, 0);
- return m;
-}
-
-static inline uint16_t
-enic_cq_rx_desc_ciflags(struct cq_enet_rq_desc *crd)
-{
- return le16_to_cpu(crd->completed_index_flags) & ~CQ_DESC_COMP_NDX_MASK;
-}
-
-static inline uint16_t
-enic_cq_rx_desc_bwflags(struct cq_enet_rq_desc *crd)
-{
- return(le16_to_cpu(crd->bytes_written_flags) &
- ~CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK);
-}
-
-static inline uint8_t
-enic_cq_rx_desc_packet_error(uint16_t bwflags)
-{
- return((bwflags & CQ_ENET_RQ_DESC_FLAGS_TRUNCATED) ==
- CQ_ENET_RQ_DESC_FLAGS_TRUNCATED);
-}
-
-static inline uint8_t
-enic_cq_rx_desc_eop(uint16_t ciflags)
-{
- return (ciflags & CQ_ENET_RQ_DESC_FLAGS_EOP)
- == CQ_ENET_RQ_DESC_FLAGS_EOP;
-}
-
-static inline uint8_t
-enic_cq_rx_desc_csum_not_calc(struct cq_enet_rq_desc *cqrd)
-{
- return ((le16_to_cpu(cqrd->q_number_rss_type_flags) &
- CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC) ==
- CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC);
-}
-
-static inline uint8_t
-enic_cq_rx_desc_ipv4_csum_ok(struct cq_enet_rq_desc *cqrd)
-{
- return ((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK) ==
- CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK);
-}
-
-static inline uint8_t
-enic_cq_rx_desc_tcp_udp_csum_ok(struct cq_enet_rq_desc *cqrd)
-{
- return((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK) ==
- CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK);
-}
-
-static inline uint8_t
-enic_cq_rx_desc_rss_type(struct cq_enet_rq_desc *cqrd)
-{
- return (uint8_t)((le16_to_cpu(cqrd->q_number_rss_type_flags) >>
- CQ_DESC_Q_NUM_BITS) & CQ_ENET_RQ_DESC_RSS_TYPE_MASK);
-}
-
-static inline uint32_t
-enic_cq_rx_desc_rss_hash(struct cq_enet_rq_desc *cqrd)
-{
- return le32_to_cpu(cqrd->rss_hash);
-}
-
-static inline uint16_t
-enic_cq_rx_desc_vlan(struct cq_enet_rq_desc *cqrd)
-{
- return le16_to_cpu(cqrd->vlan);
-}
-
-static inline uint16_t
-enic_cq_rx_desc_n_bytes(struct cq_desc *cqd)
-{
- struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
- return le16_to_cpu(cqrd->bytes_written_flags) &
- CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK;
-}
-
-static inline uint8_t
-enic_cq_rx_to_pkt_err_flags(struct cq_desc *cqd, uint64_t *pkt_err_flags_out)
-{
- struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
- uint16_t bwflags;
- int ret = 0;
- uint64_t pkt_err_flags = 0;
-
- bwflags = enic_cq_rx_desc_bwflags(cqrd);
- if (unlikely(enic_cq_rx_desc_packet_error(bwflags))) {
- pkt_err_flags = PKT_RX_MAC_ERR;
- ret = 1;
- }
- *pkt_err_flags_out = pkt_err_flags;
- return ret;
-}
-
-/*
- * Lookup table to translate RX CQ flags to mbuf flags.
- */
-static inline uint32_t
-enic_cq_rx_flags_to_pkt_type(struct cq_desc *cqd)
-{
- struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
- uint8_t cqrd_flags = cqrd->flags;
- static const uint32_t cq_type_table[128] __rte_cache_aligned = {
- [32] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4,
- [34] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4
- | RTE_PTYPE_L4_UDP,
- [36] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4
- | RTE_PTYPE_L4_TCP,
- [96] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4
- | RTE_PTYPE_L4_FRAG,
- [16] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6,
- [18] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6
- | RTE_PTYPE_L4_UDP,
- [20] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6
- | RTE_PTYPE_L4_TCP,
- [80] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6
- | RTE_PTYPE_L4_FRAG,
- /* All others reserved */
- };
- cqrd_flags &= CQ_ENET_RQ_DESC_FLAGS_IPV4_FRAGMENT
- | CQ_ENET_RQ_DESC_FLAGS_IPV4 | CQ_ENET_RQ_DESC_FLAGS_IPV6
- | CQ_ENET_RQ_DESC_FLAGS_TCP | CQ_ENET_RQ_DESC_FLAGS_UDP;
- return cq_type_table[cqrd_flags];
-}
-
-static inline void
-enic_cq_rx_to_pkt_flags(struct cq_desc *cqd, struct rte_mbuf *mbuf)
-{
- struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
- uint16_t ciflags, bwflags, pkt_flags = 0;
- ciflags = enic_cq_rx_desc_ciflags(cqrd);
- bwflags = enic_cq_rx_desc_bwflags(cqrd);
-
- mbuf->ol_flags = 0;
-
- /* flags are meaningless if !EOP */
- if (unlikely(!enic_cq_rx_desc_eop(ciflags)))
- goto mbuf_flags_done;
-
- /* VLAN stripping */
- if (bwflags & CQ_ENET_RQ_DESC_FLAGS_VLAN_STRIPPED) {
- pkt_flags |= PKT_RX_VLAN_PKT;
- mbuf->vlan_tci = enic_cq_rx_desc_vlan(cqrd);
- } else {
- mbuf->vlan_tci = 0;
- }
-
- /* RSS flag */
- if (enic_cq_rx_desc_rss_type(cqrd)) {
- pkt_flags |= PKT_RX_RSS_HASH;
- mbuf->hash.rss = enic_cq_rx_desc_rss_hash(cqrd);
- }
-
- /* checksum flags */
- if (!enic_cq_rx_desc_csum_not_calc(cqrd) &&
- (mbuf->packet_type & RTE_PTYPE_L3_IPV4)) {
- if (unlikely(!enic_cq_rx_desc_ipv4_csum_ok(cqrd)))
- pkt_flags |= PKT_RX_IP_CKSUM_BAD;
- if (mbuf->packet_type & (RTE_PTYPE_L4_UDP | RTE_PTYPE_L4_TCP)) {
- if (unlikely(!enic_cq_rx_desc_tcp_udp_csum_ok(cqrd)))
- pkt_flags |= PKT_RX_L4_CKSUM_BAD;
- }
- }
-
- mbuf_flags_done:
- mbuf->ol_flags = pkt_flags;
-}
-
-static inline uint32_t
-enic_ring_add(uint32_t n_descriptors, uint32_t i0, uint32_t i1)
-{
- uint32_t d = i0 + i1;
- ASSERT(i0 < n_descriptors);
- ASSERT(i1 < n_descriptors);
- d -= (d >= n_descriptors) ? n_descriptors : 0;
- return d;
-}
-
-
-uint16_t
-enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
- uint16_t nb_pkts)
-{
- struct vnic_rq *rq = rx_queue;
- struct enic *enic = vnic_dev_priv(rq->vdev);
- unsigned int rx_id;
- struct rte_mbuf *nmb, *rxmb;
- uint16_t nb_rx = 0;
- uint16_t nb_hold;
- struct vnic_cq *cq;
- volatile struct cq_desc *cqd_ptr;
- uint8_t color;
-
- cq = &enic->cq[enic_cq_rq(enic, rq->index)];
- rx_id = cq->to_clean; /* index of cqd, rqd, mbuf_table */
- cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id;
-
- nb_hold = rq->rx_nb_hold; /* mbufs held by software */
-
- while (nb_rx < nb_pkts) {
- volatile struct rq_enet_desc *rqd_ptr;
- dma_addr_t dma_addr;
- struct cq_desc cqd;
- uint64_t ol_err_flags;
- uint8_t packet_error;
-
- /* Check for pkts available */
- color = (cqd_ptr->type_color >> CQ_DESC_COLOR_SHIFT)
- & CQ_DESC_COLOR_MASK;
- if (color == cq->last_color)
- break;
-
- /* Get the cq descriptor and rq pointer */
- cqd = *cqd_ptr;
- rqd_ptr = (struct rq_enet_desc *)(rq->ring.descs) + rx_id;
-
- /* allocate a new mbuf */
- nmb = rte_rxmbuf_alloc(rq->mp);
- if (nmb == NULL) {
- dev_err(enic, "RX mbuf alloc failed port=%u qid=%u",
- enic->port_id, (unsigned)rq->index);
- rte_eth_devices[enic->port_id].
- data->rx_mbuf_alloc_failed++;
- break;
- }
-
- /* A packet error means descriptor and data are untrusted */
- packet_error = enic_cq_rx_to_pkt_err_flags(&cqd, &ol_err_flags);
-
- /* Get the mbuf to return and replace with one just allocated */
- rxmb = rq->mbuf_ring[rx_id];
- rq->mbuf_ring[rx_id] = nmb;
-
- /* Increment cqd, rqd, mbuf_table index */
- rx_id++;
- if (unlikely(rx_id == rq->ring.desc_count)) {
- rx_id = 0;
- cq->last_color = cq->last_color ? 0 : 1;
- }
-
- /* Prefetch next mbuf & desc while processing current one */
- cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id;
- rte_enic_prefetch(cqd_ptr);
- rte_enic_prefetch(rq->mbuf_ring[rx_id]);
- rte_enic_prefetch((struct rq_enet_desc *)(rq->ring.descs)
- + rx_id);
-
- /* Push descriptor for newly allocated mbuf */
- dma_addr = (dma_addr_t)(nmb->buf_physaddr
- + RTE_PKTMBUF_HEADROOM);
- rqd_ptr->address = rte_cpu_to_le_64(dma_addr);
- rqd_ptr->length_type = cpu_to_le16(nmb->buf_len
- - RTE_PKTMBUF_HEADROOM);
-
- /* Fill in the rest of the mbuf */
- rxmb->data_off = RTE_PKTMBUF_HEADROOM;
- rxmb->nb_segs = 1;
- rxmb->next = NULL;
- rxmb->port = enic->port_id;
- if (!packet_error) {
- rxmb->pkt_len = enic_cq_rx_desc_n_bytes(&cqd);
- rxmb->packet_type = enic_cq_rx_flags_to_pkt_type(&cqd);
- enic_cq_rx_to_pkt_flags(&cqd, rxmb);
- } else {
- rxmb->pkt_len = 0;
- rxmb->packet_type = 0;
- rxmb->ol_flags = 0;
- }
- rxmb->data_len = rxmb->pkt_len;
-
- /* prefetch mbuf data for caller */
- rte_packet_prefetch(RTE_PTR_ADD(rxmb->buf_addr,
- RTE_PKTMBUF_HEADROOM));
-
- /* store the mbuf address into the next entry of the array */
- rx_pkts[nb_rx++] = rxmb;
- }
-
- nb_hold += nb_rx;
- cq->to_clean = rx_id;
-
- if (nb_hold > rq->rx_free_thresh) {
- rq->posted_index = enic_ring_add(rq->ring.desc_count,
- rq->posted_index, nb_hold);
- nb_hold = 0;
- rte_mb();
- iowrite32(rq->posted_index, &rq->ctrl->posted_index);
- }
-
- rq->rx_nb_hold = nb_hold;
-
- return nb_rx;
-}
diff --git a/drivers/net/enic/enic_rxtx.c b/drivers/net/enic/enic_rxtx.c
new file mode 100644
index 0000000..71ca34e
--- /dev/null
+++ b/drivers/net/enic/enic_rxtx.c
@@ -0,0 +1,505 @@
+/*
+ * Copyright 2008-2016 Cisco Systems, Inc. All rights reserved.
+ * Copyright 2007 Nuova Systems, Inc. All rights reserved.
+ *
+ * Copyright (c) 2016, Cisco Systems, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <rte_mbuf.h>
+#include <rte_ethdev.h>
+#include <rte_prefetch.h>
+#include <rte_memzone.h>
+
+#include "enic_compat.h"
+#include "rq_enet_desc.h"
+#include "enic.h"
+
+#define RTE_PMD_USE_PREFETCH
+
+#ifdef RTE_PMD_USE_PREFETCH
+/*
+ * Prefetch a cache line into all cache levels.
+ */
+#define rte_enic_prefetch(p) rte_prefetch0(p)
+#else
+#define rte_enic_prefetch(p) do {} while (0)
+#endif
+
+#ifdef RTE_PMD_PACKET_PREFETCH
+#define rte_packet_prefetch(p) rte_prefetch1(p)
+#else
+#define rte_packet_prefetch(p) do {} while (0)
+#endif
+
+static inline struct rte_mbuf *
+rte_rxmbuf_alloc(struct rte_mempool *mp)
+{
+ struct rte_mbuf *m;
+
+ m = __rte_mbuf_raw_alloc(mp);
+ __rte_mbuf_sanity_check_raw(m, 0);
+ return m;
+}
+
+static inline uint16_t
+enic_cq_rx_desc_ciflags(struct cq_enet_rq_desc *crd)
+{
+ return le16_to_cpu(crd->completed_index_flags) & ~CQ_DESC_COMP_NDX_MASK;
+}
+
+static inline uint16_t
+enic_cq_rx_desc_bwflags(struct cq_enet_rq_desc *crd)
+{
+ return(le16_to_cpu(crd->bytes_written_flags) &
+ ~CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK);
+}
+
+static inline uint8_t
+enic_cq_rx_desc_packet_error(uint16_t bwflags)
+{
+ return((bwflags & CQ_ENET_RQ_DESC_FLAGS_TRUNCATED) ==
+ CQ_ENET_RQ_DESC_FLAGS_TRUNCATED);
+}
+
+static inline uint8_t
+enic_cq_rx_desc_eop(uint16_t ciflags)
+{
+ return (ciflags & CQ_ENET_RQ_DESC_FLAGS_EOP)
+ == CQ_ENET_RQ_DESC_FLAGS_EOP;
+}
+
+static inline uint8_t
+enic_cq_rx_desc_csum_not_calc(struct cq_enet_rq_desc *cqrd)
+{
+ return ((le16_to_cpu(cqrd->q_number_rss_type_flags) &
+ CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC) ==
+ CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC);
+}
+
+static inline uint8_t
+enic_cq_rx_desc_ipv4_csum_ok(struct cq_enet_rq_desc *cqrd)
+{
+ return ((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK) ==
+ CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK);
+}
+
+static inline uint8_t
+enic_cq_rx_desc_tcp_udp_csum_ok(struct cq_enet_rq_desc *cqrd)
+{
+ return((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK) ==
+ CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK);
+}
+
+static inline uint8_t
+enic_cq_rx_desc_rss_type(struct cq_enet_rq_desc *cqrd)
+{
+ return (uint8_t)((le16_to_cpu(cqrd->q_number_rss_type_flags) >>
+ CQ_DESC_Q_NUM_BITS) & CQ_ENET_RQ_DESC_RSS_TYPE_MASK);
+}
+
+static inline uint32_t
+enic_cq_rx_desc_rss_hash(struct cq_enet_rq_desc *cqrd)
+{
+ return le32_to_cpu(cqrd->rss_hash);
+}
+
+static inline uint16_t
+enic_cq_rx_desc_vlan(struct cq_enet_rq_desc *cqrd)
+{
+ return le16_to_cpu(cqrd->vlan);
+}
+
+static inline uint16_t
+enic_cq_rx_desc_n_bytes(struct cq_desc *cqd)
+{
+ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
+ return le16_to_cpu(cqrd->bytes_written_flags) &
+ CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK;
+}
+
+static inline uint8_t
+enic_cq_rx_to_pkt_err_flags(struct cq_desc *cqd, uint64_t *pkt_err_flags_out)
+{
+ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
+ uint16_t bwflags;
+ int ret = 0;
+ uint64_t pkt_err_flags = 0;
+
+ bwflags = enic_cq_rx_desc_bwflags(cqrd);
+ if (unlikely(enic_cq_rx_desc_packet_error(bwflags))) {
+ pkt_err_flags = PKT_RX_MAC_ERR;
+ ret = 1;
+ }
+ *pkt_err_flags_out = pkt_err_flags;
+ return ret;
+}
+
+/*
+ * Lookup table to translate RX CQ flags to mbuf flags.
+ */
+static inline uint32_t
+enic_cq_rx_flags_to_pkt_type(struct cq_desc *cqd)
+{
+ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
+ uint8_t cqrd_flags = cqrd->flags;
+ static const uint32_t cq_type_table[128] __rte_cache_aligned = {
+ [32] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4,
+ [34] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4
+ | RTE_PTYPE_L4_UDP,
+ [36] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4
+ | RTE_PTYPE_L4_TCP,
+ [96] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4
+ | RTE_PTYPE_L4_FRAG,
+ [16] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6,
+ [18] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6
+ | RTE_PTYPE_L4_UDP,
+ [20] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6
+ | RTE_PTYPE_L4_TCP,
+ [80] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6
+ | RTE_PTYPE_L4_FRAG,
+ /* All others reserved */
+ };
+ cqrd_flags &= CQ_ENET_RQ_DESC_FLAGS_IPV4_FRAGMENT
+ | CQ_ENET_RQ_DESC_FLAGS_IPV4 | CQ_ENET_RQ_DESC_FLAGS_IPV6
+ | CQ_ENET_RQ_DESC_FLAGS_TCP | CQ_ENET_RQ_DESC_FLAGS_UDP;
+ return cq_type_table[cqrd_flags];
+}
+
+static inline void
+enic_cq_rx_to_pkt_flags(struct cq_desc *cqd, struct rte_mbuf *mbuf)
+{
+ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
+ uint16_t ciflags, bwflags, pkt_flags = 0;
+ ciflags = enic_cq_rx_desc_ciflags(cqrd);
+ bwflags = enic_cq_rx_desc_bwflags(cqrd);
+
+ mbuf->ol_flags = 0;
+
+ /* flags are meaningless if !EOP */
+ if (unlikely(!enic_cq_rx_desc_eop(ciflags)))
+ goto mbuf_flags_done;
+
+ /* VLAN stripping */
+ if (bwflags & CQ_ENET_RQ_DESC_FLAGS_VLAN_STRIPPED) {
+ pkt_flags |= PKT_RX_VLAN_PKT;
+ mbuf->vlan_tci = enic_cq_rx_desc_vlan(cqrd);
+ } else {
+ mbuf->vlan_tci = 0;
+ }
+
+ /* RSS flag */
+ if (enic_cq_rx_desc_rss_type(cqrd)) {
+ pkt_flags |= PKT_RX_RSS_HASH;
+ mbuf->hash.rss = enic_cq_rx_desc_rss_hash(cqrd);
+ }
+
+ /* checksum flags */
+ if (!enic_cq_rx_desc_csum_not_calc(cqrd) &&
+ (mbuf->packet_type & RTE_PTYPE_L3_IPV4)) {
+ if (unlikely(!enic_cq_rx_desc_ipv4_csum_ok(cqrd)))
+ pkt_flags |= PKT_RX_IP_CKSUM_BAD;
+ if (mbuf->packet_type & (RTE_PTYPE_L4_UDP | RTE_PTYPE_L4_TCP)) {
+ if (unlikely(!enic_cq_rx_desc_tcp_udp_csum_ok(cqrd)))
+ pkt_flags |= PKT_RX_L4_CKSUM_BAD;
+ }
+ }
+
+ mbuf_flags_done:
+ mbuf->ol_flags = pkt_flags;
+}
+
+uint16_t
+enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts)
+{
+ struct vnic_rq *rq = rx_queue;
+ struct enic *enic = vnic_dev_priv(rq->vdev);
+ unsigned int rx_id;
+ struct rte_mbuf *nmb, *rxmb;
+ uint16_t nb_rx = 0;
+ uint16_t nb_hold;
+ struct vnic_cq *cq;
+ volatile struct cq_desc *cqd_ptr;
+ uint8_t color;
+
+ cq = &enic->cq[enic_cq_rq(enic, rq->index)];
+ rx_id = cq->to_clean; /* index of cqd, rqd, mbuf_table */
+ cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id;
+
+ nb_hold = rq->rx_nb_hold; /* mbufs held by software */
+
+ while (nb_rx < nb_pkts) {
+ volatile struct rq_enet_desc *rqd_ptr;
+ dma_addr_t dma_addr;
+ struct cq_desc cqd;
+ uint64_t ol_err_flags;
+ uint8_t packet_error;
+
+ /* Check for pkts available */
+ color = (cqd_ptr->type_color >> CQ_DESC_COLOR_SHIFT)
+ & CQ_DESC_COLOR_MASK;
+ if (color == cq->last_color)
+ break;
+
+ /* Get the cq descriptor and rq pointer */
+ cqd = *cqd_ptr;
+ rqd_ptr = (struct rq_enet_desc *)(rq->ring.descs) + rx_id;
+
+ /* allocate a new mbuf */
+ nmb = rte_rxmbuf_alloc(rq->mp);
+ if (nmb == NULL) {
+ dev_err(enic, "RX mbuf alloc failed port=%u qid=%u",
+ enic->port_id, (unsigned)rq->index);
+ rte_eth_devices[enic->port_id].
+ data->rx_mbuf_alloc_failed++;
+ break;
+ }
+
+ /* A packet error means descriptor and data are untrusted */
+ packet_error = enic_cq_rx_to_pkt_err_flags(&cqd, &ol_err_flags);
+
+ /* Get the mbuf to return and replace with one just allocated */
+ rxmb = rq->mbuf_ring[rx_id];
+ rq->mbuf_ring[rx_id] = nmb;
+
+ /* Increment cqd, rqd, mbuf_table index */
+ rx_id++;
+ if (unlikely(rx_id == rq->ring.desc_count)) {
+ rx_id = 0;
+ cq->last_color = cq->last_color ? 0 : 1;
+ }
+
+ /* Prefetch next mbuf & desc while processing current one */
+ cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id;
+ rte_enic_prefetch(cqd_ptr);
+ rte_enic_prefetch(rq->mbuf_ring[rx_id]);
+ rte_enic_prefetch((struct rq_enet_desc *)(rq->ring.descs)
+ + rx_id);
+
+ /* Push descriptor for newly allocated mbuf */
+ dma_addr = (dma_addr_t)(nmb->buf_physaddr
+ + RTE_PKTMBUF_HEADROOM);
+ rqd_ptr->address = rte_cpu_to_le_64(dma_addr);
+ rqd_ptr->length_type = cpu_to_le16(nmb->buf_len
+ - RTE_PKTMBUF_HEADROOM);
+
+ /* Fill in the rest of the mbuf */
+ rxmb->data_off = RTE_PKTMBUF_HEADROOM;
+ rxmb->nb_segs = 1;
+ rxmb->next = NULL;
+ rxmb->port = enic->port_id;
+ if (!packet_error) {
+ rxmb->pkt_len = enic_cq_rx_desc_n_bytes(&cqd);
+ rxmb->packet_type = enic_cq_rx_flags_to_pkt_type(&cqd);
+ enic_cq_rx_to_pkt_flags(&cqd, rxmb);
+ } else {
+ rxmb->pkt_len = 0;
+ rxmb->packet_type = 0;
+ rxmb->ol_flags = 0;
+ }
+ rxmb->data_len = rxmb->pkt_len;
+
+ /* prefetch mbuf data for caller */
+ rte_packet_prefetch(RTE_PTR_ADD(rxmb->buf_addr,
+ RTE_PKTMBUF_HEADROOM));
+
+ /* store the mbuf address into the next entry of the array */
+ rx_pkts[nb_rx++] = rxmb;
+ }
+
+ nb_hold += nb_rx;
+ cq->to_clean = rx_id;
+
+ if (nb_hold > rq->rx_free_thresh) {
+ rq->posted_index = enic_ring_add(rq->ring.desc_count,
+ rq->posted_index, nb_hold);
+ nb_hold = 0;
+ rte_mb();
+ iowrite32(rq->posted_index, &rq->ctrl->posted_index);
+ }
+
+ rq->rx_nb_hold = nb_hold;
+
+ return nb_rx;
+}
+
+static inline void enic_free_wq_bufs(struct vnic_wq *wq, u16 completed_index)
+{
+ struct vnic_wq_buf *buf;
+ struct rte_mbuf *m, *free[ENIC_MAX_WQ_DESCS];
+ unsigned int nb_to_free, nb_free = 0, i;
+ struct rte_mempool *pool;
+ unsigned int tail_idx;
+ unsigned int desc_count = wq->ring.desc_count;
+
+ nb_to_free = enic_ring_sub(desc_count, wq->tail_idx, completed_index)
+ + 1;
+ tail_idx = wq->tail_idx;
+ buf = &wq->bufs[tail_idx];
+ pool = ((struct rte_mbuf *)buf->mb)->pool;
+ for (i = 0; i < nb_to_free; i++) {
+ buf = &wq->bufs[tail_idx];
+ m = (struct rte_mbuf *)(buf->mb);
+ if (likely(m->pool == pool)) {
+ ASSERT(nb_free < ENIC_MAX_WQ_DESCS);
+ free[nb_free++] = m;
+ } else {
+ rte_mempool_put_bulk(pool, (void *)free, nb_free);
+ free[0] = m;
+ nb_free = 1;
+ pool = m->pool;
+ }
+ tail_idx = enic_ring_incr(desc_count, tail_idx);
+ buf->mb = NULL;
+ }
+
+ rte_mempool_put_bulk(pool, (void **)free, nb_free);
+
+ wq->tail_idx = tail_idx;
+ wq->ring.desc_avail += nb_to_free;
+}
+
+unsigned int enic_cleanup_wq(__rte_unused struct enic *enic, struct vnic_wq
*wq)
+{
+ u16 completed_index;
+
+ completed_index = *((uint32_t *)wq->cqmsg_rz->addr) & 0xffff;
+
+ if (wq->last_completed_index != completed_index) {
+ enic_free_wq_bufs(wq, completed_index);
+ wq->last_completed_index = completed_index;
+ }
+ return 0;
+}
+
+uint16_t enic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
+ uint16_t nb_pkts)
+{
+ uint16_t index;
+ unsigned int pkt_len, data_len;
+ unsigned int nb_segs;
+ struct rte_mbuf *tx_pkt;
+ struct vnic_wq *wq = (struct vnic_wq *)tx_queue;
+ struct enic *enic = vnic_dev_priv(wq->vdev);
+ unsigned short vlan_id;
+ unsigned short ol_flags;
+ unsigned int wq_desc_avail;
+ int head_idx;
+ struct vnic_wq_buf *buf;
+ unsigned int hw_ip_cksum_enabled;
+ unsigned int desc_count;
+ struct wq_enet_desc *descs, *desc_p, desc_tmp;
+ uint16_t mss;
+ uint8_t vlan_tag_insert;
+ uint8_t eop;
+ uint64_t bus_addr;
+
+ enic_cleanup_wq(enic, wq);
+ wq_desc_avail = vnic_wq_desc_avail(wq);
+ head_idx = wq->head_idx;
+ desc_count = wq->ring.desc_count;
+
+ nb_pkts = RTE_MIN(nb_pkts, ENIC_TX_XMIT_MAX);
+
+ hw_ip_cksum_enabled = enic->hw_ip_checksum;
+ for (index = 0; index < nb_pkts; index++) {
+ tx_pkt = *tx_pkts++;
+ nb_segs = tx_pkt->nb_segs;
+ if (nb_segs > wq_desc_avail) {
+ if (index > 0)
+ goto post;
+ goto done;
+ }
+
+ pkt_len = tx_pkt->pkt_len;
+ data_len = tx_pkt->data_len;
+ vlan_id = tx_pkt->vlan_tci;
+ ol_flags = tx_pkt->ol_flags;
+
+ mss = 0;
+ vlan_tag_insert = 0;
+ bus_addr = (dma_addr_t)
+ (tx_pkt->buf_physaddr + tx_pkt->data_off);
+
+ descs = (struct wq_enet_desc *)wq->ring.descs;
+ desc_p = descs + head_idx;
+
+ eop = (data_len == pkt_len);
+
+ if (ol_flags & PKT_TX_VLAN_PKT)
+ vlan_tag_insert = 1;
+
+ if (hw_ip_cksum_enabled && (ol_flags & PKT_TX_IP_CKSUM))
+ mss |= ENIC_CALC_IP_CKSUM;
+
+ if (hw_ip_cksum_enabled && (ol_flags & PKT_TX_TCP_UDP_CKSUM))
+ mss |= ENIC_CALC_TCP_UDP_CKSUM;
+
+ wq_enet_desc_enc(&desc_tmp, bus_addr, data_len, mss, 0, 0, eop,
+ eop, 0, vlan_tag_insert, vlan_id, 0);
+
+ *desc_p = desc_tmp;
+ buf = &wq->bufs[head_idx];
+ buf->mb = (void *)tx_pkt;
+ head_idx = enic_ring_incr(desc_count, head_idx);
+ wq_desc_avail--;
+
+ if (!eop) {
+ for (tx_pkt = tx_pkt->next; tx_pkt; tx_pkt =
+ tx_pkt->next) {
+ data_len = tx_pkt->data_len;
+
+ if (tx_pkt->next == NULL)
+ eop = 1;
+ desc_p = descs + head_idx;
+ bus_addr = (dma_addr_t)(tx_pkt->buf_physaddr
+ + tx_pkt->data_off);
+ wq_enet_desc_enc((struct wq_enet_desc *)
+ &desc_tmp, bus_addr, data_len,
+ mss, 0, 0, eop, eop, 0,
+ vlan_tag_insert, vlan_id, 0);
+
+ *desc_p = desc_tmp;
+ buf = &wq->bufs[head_idx];
+ buf->mb = (void *)tx_pkt;
+ head_idx = enic_ring_incr(desc_count, head_idx);
+ wq_desc_avail--;
+ }
+ }
+ }
+ post:
+ rte_wmb();
+ iowrite32(head_idx, &wq->ctrl->posted_index);
+ done:
+ wq->ring.desc_avail = wq_desc_avail;
+ wq->head_idx = head_idx;
+
+ return index;
+}
--
2.7.0
