Signed-off-by: Igor Russkikh <igor.russk...@aquantia.com>
---
drivers/net/atlantic/Makefile | 6 +
drivers/net/atlantic/atl_ethdev.c | 44 +++
drivers/net/atlantic/atl_ethdev.h | 80 ++++
drivers/net/atlantic/atl_rxtx.c | 758 ++++++++++++++++++++++++++++++++++++++
drivers/net/atlantic/meson.build | 6 +
5 files changed, 894 insertions(+)
create mode 100644 drivers/net/atlantic/atl_rxtx.c
diff --git a/drivers/net/atlantic/Makefile b/drivers/net/atlantic/Makefile
index 4b0e6f06c..7b11246d1 100644
--- a/drivers/net/atlantic/Makefile
+++ b/drivers/net/atlantic/Makefile
@@ -54,7 +54,13 @@ VPATH += $(SRCDIR)/hw_atl
#
# all source are stored in SRCS-y
#
+SRCS-$(CONFIG_RTE_LIBRTE_ATLANTIC_PMD) += atl_rxtx.c
SRCS-$(CONFIG_RTE_LIBRTE_ATLANTIC_PMD) += atl_ethdev.c
+SRCS-$(CONFIG_RTE_LIBRTE_ATLANTIC_PMD) += atl_hw_regs.c
+SRCS-$(CONFIG_RTE_LIBRTE_ATLANTIC_PMD) += hw_atl_utils.c
+SRCS-$(CONFIG_RTE_LIBRTE_ATLANTIC_PMD) += hw_atl_llh.c
+SRCS-$(CONFIG_RTE_LIBRTE_ATLANTIC_PMD) += hw_atl_utils_fw2x.c
+SRCS-$(CONFIG_RTE_LIBRTE_ATLANTIC_PMD) += hw_atl_b0.c
SRCS-$(CONFIG_RTE_LIBRTE_ATLANTIC_PMD) += rte_pmd_atlantic.c
# install this header file
diff --git a/drivers/net/atlantic/atl_ethdev.c
b/drivers/net/atlantic/atl_ethdev.c
index 44cea12ab..bb4d96bb1 100644
--- a/drivers/net/atlantic/atl_ethdev.c
+++ b/drivers/net/atlantic/atl_ethdev.c
@@ -277,6 +277,8 @@ static const struct eth_dev_ops atl_eth_dev_ops = {
.xstats_get_names = atl_dev_xstats_get_names,
.stats_reset = atl_dev_stats_reset,
.xstats_reset = atl_dev_xstats_reset,
+ .queue_stats_mapping_set = atl_dev_queue_stats_mapping_set,
+
.fw_version_get = atl_fw_version_get,
.dev_infos_get = atl_dev_info_get,
.dev_supported_ptypes_get = atl_dev_supported_ptypes_get,
@@ -289,6 +291,20 @@ static const struct eth_dev_ops atl_eth_dev_ops = {
.vlan_tpid_set = atl_vlan_tpid_set,
.vlan_strip_queue_set = atl_vlan_strip_queue_set,
+ /* Queue Control */
+ .rx_queue_start = atl_rx_queue_start,
+ .rx_queue_stop = atl_rx_queue_stop,
+ .rx_queue_setup = atl_rx_queue_setup,
+ .rx_queue_release = atl_rx_queue_release,
+
+ .rx_queue_intr_enable = atl_dev_rx_queue_intr_enable,
+ .rx_queue_intr_disable = atl_dev_rx_queue_intr_disable,
+
+ .rx_queue_count = atl_rx_queue_count,
+ .rx_descriptor_done = atl_dev_rx_descriptor_done,
+ .rx_descriptor_status = atl_dev_rx_descriptor_status,
+ .tx_descriptor_status = atl_dev_tx_descriptor_status,
+
/* LEDs */
.dev_led_on = atl_dev_led_on,
.dev_led_off = atl_dev_led_off,
@@ -307,6 +323,7 @@ static const struct eth_dev_ops atl_eth_dev_ops = {
.mac_addr_remove = atl_remove_mac_addr,
.mac_addr_set = atl_set_default_mac_addr,
.set_mc_addr_list = atl_dev_set_mc_addr_list,
+ .rxq_info_get = atl_rxq_info_get,
.reta_update = atl_reta_update,
.reta_query = atl_reta_query,
.rss_hash_update = atl_rss_hash_update,
@@ -622,6 +639,19 @@ atl_dev_start(struct rte_eth_dev *dev)
}
}
+ /* This can fail when allocating mbufs for descriptor rings */
+ err = atl_rx_init(dev);
+ if (err) {
+ PMD_INIT_LOG(ERR, "Unable to initialize RX hardware");
+ goto error;
+ }
+
+ err = atl_start_queues(dev);
+ if (err < 0) {
+ PMD_INIT_LOG(ERR, "Unable to start rxtx queues");
+ goto error;
+ }
+
err = hw->aq_fw_ops->update_link_status(hw);
if (err) {
@@ -708,6 +738,9 @@ atl_dev_stop(struct rte_eth_dev *dev)
/* reset the NIC */
atl_reset_hw(hw);
hw->adapter_stopped = 0;
+
+ atl_stop_queues(dev);
+
/* Clear stored conf */
dev->data->scattered_rx = 0;
dev->data->lro = 0;
@@ -766,6 +799,8 @@ atl_dev_close(struct rte_eth_dev *dev)
atl_dev_stop(dev);
hw->adapter_stopped = 1;
+
+ atl_free_queues(dev);
}
static int
@@ -865,6 +900,15 @@ atl_dev_xstats_reset(struct rte_eth_dev *dev __rte_unused)
return;
}
+static int
+atl_dev_queue_stats_mapping_set(struct rte_eth_dev *eth_dev __rte_unused,
+ uint16_t queue_id __rte_unused,
+ uint8_t stat_idx __rte_unused,
+ uint8_t is_rx __rte_unused)
+{
+ /* The mapping is hardcoded: queue 0 -> stat 0, etc */
+ return 0;
+}
static int
atl_fw_version_get(struct rte_eth_dev *dev, char *fw_version, size_t fw_size)
diff --git a/drivers/net/atlantic/atl_ethdev.h
b/drivers/net/atlantic/atl_ethdev.h
index 3ebef1f43..e532c43fa 100644
--- a/drivers/net/atlantic/atl_ethdev.h
+++ b/drivers/net/atlantic/atl_ethdev.h
@@ -13,6 +13,17 @@
#include "atl_types.h"
#include "hw_atl/hw_atl_utils.h"
+#define ATL_RSS_OFFLOAD_ALL ( \
+ ETH_RSS_IPV4 | \
+ ETH_RSS_NONFRAG_IPV4_TCP | \
+ ETH_RSS_NONFRAG_IPV4_UDP | \
+ ETH_RSS_IPV6 | \
+ ETH_RSS_NONFRAG_IPV6_TCP | \
+ ETH_RSS_NONFRAG_IPV6_UDP | \
+ ETH_RSS_IPV6_EX | \
+ ETH_RSS_IPV6_TCP_EX | \
+ ETH_RSS_IPV6_UDP_EX)
+
#define ATL_FLAG_NEED_LINK_UPDATE (uint32_t)(1 << 0)
#define ATL_FLAG_NEED_LINK_CONFIG (uint32_t)(4 << 0)
@@ -46,6 +57,75 @@ struct atl_adapter {
#define ATL_DEV_PRIVATE_TO_CFG(adapter) \
(&((struct atl_adapter *)adapter)->hw_cfg)
+extern const struct rte_flow_ops atl_flow_ops;
+
+/*
+ * RX/TX function prototypes
+ */
+void atl_rx_queue_release(void *rxq);
+void atl_tx_queue_release(void *txq);
+
+int atl_rx_queue_setup(struct rte_eth_dev *dev, uint16_t rx_queue_id,
+ uint16_t nb_rx_desc, unsigned int socket_id,
+ const struct rte_eth_rxconf *rx_conf,
+ struct rte_mempool *mb_pool);
+
+int atl_tx_queue_setup(struct rte_eth_dev *dev, uint16_t tx_queue_id,
+ uint16_t nb_tx_desc, unsigned int socket_id,
+ const struct rte_eth_txconf *tx_conf);
+
+uint32_t atl_rx_queue_count(struct rte_eth_dev *dev, uint16_t rx_queue_id);
+
+int atl_dev_rx_descriptor_done(void *rx_queue, uint16_t offset);
+int atl_dev_rx_descriptor_status(void *rx_queue, uint16_t offset);
+int atl_dev_tx_descriptor_status(void *tx_queue, uint16_t offset);
+
+int atl_dev_rx_queue_intr_enable(struct rte_eth_dev *eth_dev, uint16_t
queue_id);
+int atl_dev_rx_queue_intr_disable(struct rte_eth_dev *eth_dev, uint16_t
queue_id);
+
+int atl_rx_init(struct rte_eth_dev *dev);
+int atl_tx_init(struct rte_eth_dev *dev);
+
+int atl_start_queues(struct rte_eth_dev *dev);
+int atl_stop_queues(struct rte_eth_dev *dev);
+void atl_free_queues(struct rte_eth_dev *dev);
+
+int atl_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id);
+int atl_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id);
+
+int atl_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id);
+int atl_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id);
+
+void atl_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
+ struct rte_eth_rxq_info *qinfo);
+
+void atl_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
+ struct rte_eth_txq_info *qinfo);
+
+uint16_t atl_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts);
+
+uint16_t atl_recv_pkts_bulk_alloc(void *rx_queue, struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts);
+
+uint16_t atl_recv_pkts_lro_single_alloc(void *rx_queue,
+ struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
+uint16_t atl_recv_pkts_lro_bulk_alloc(void *rx_queue,
+ struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
+
+uint16_t atl_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
+ uint16_t nb_pkts);
+
+uint16_t atl_xmit_pkts_simple(void *tx_queue, struct rte_mbuf **tx_pkts,
+ uint16_t nb_pkts);
+
+uint16_t atl_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
+ uint16_t nb_pkts);
+
int
atl_dev_led_control(struct rte_eth_dev *dev, int control);
+
+bool
+is_atl_supported(struct rte_eth_dev *dev);
+
#endif /* _ATLANTIC_ETHDEV_H_ */
diff --git a/drivers/net/atlantic/atl_rxtx.c b/drivers/net/atlantic/atl_rxtx.c
new file mode 100644
index 000000000..6198f5dfe
--- /dev/null
+++ b/drivers/net/atlantic/atl_rxtx.c
@@ -0,0 +1,758 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Aquantia Corporation
+ */
+
+#include <sys/queue.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <stdint.h>
+#include <stdarg.h>
+#include <inttypes.h>
+
+#include <rte_interrupts.h>
+#include <rte_byteorder.h>
+#include <rte_common.h>
+#include <rte_log.h>
+#include <rte_debug.h>
+#include <rte_pci.h>
+#include <rte_memory.h>
+#include <rte_memcpy.h>
+#include <rte_memzone.h>
+#include <rte_launch.h>
+#include <rte_eal.h>
+#include <rte_per_lcore.h>
+#include <rte_lcore.h>
+#include <rte_atomic.h>
+#include <rte_branch_prediction.h>
+#include <rte_mempool.h>
+#include <rte_malloc.h>
+#include <rte_mbuf.h>
+#include <rte_ether.h>
+#include <rte_ethdev_driver.h>
+#include <rte_prefetch.h>
+#include <rte_udp.h>
+#include <rte_tcp.h>
+#include <rte_sctp.h>
+#include <rte_net.h>
+#include <rte_string_fns.h>
+#include "atl_ethdev.h"
+#include "atl_hw_regs.h"
+
+#include "atl_logs.h"
+#include "hw_atl/hw_atl_llh.h"
+#include "hw_atl/hw_atl_b0.h"
+#include "hw_atl/hw_atl_b0_internal.h"
+
+/**
+ * Structure associated with each descriptor of the RX ring of a RX queue.
+ */
+struct atl_rx_entry {
+ struct rte_mbuf *mbuf;
+};
+
+/**
+ * Structure associated with each RX queue.
+ */
+struct atl_rx_queue {
+ struct rte_mempool *mb_pool;
+ struct hw_atl_rxd_s *hw_ring;
+ uint64_t hw_ring_phys_addr;
+ struct atl_rx_entry *sw_ring;
+ uint16_t nb_rx_desc;
+ uint16_t rx_tail;
+ uint16_t nb_rx_hold;
+ uint16_t rx_free_thresh;
+ uint16_t queue_id;
+ uint16_t port_id;
+ uint16_t buff_size;
+ bool l3_csum_enabled;
+ bool l4_csum_enabled;
+};
+
+inline static void
+atl_reset_rx_queue(struct atl_rx_queue *rxq)
+{
+ struct hw_atl_rxd_s * rxd = NULL;
+ int i;
+
+ PMD_INIT_FUNC_TRACE();
+
+ for (i = 0; i < rxq->nb_rx_desc; i++) {
+ rxd = (struct hw_atl_rxd_s *)&rxq->hw_ring[i];
+ rxd->buf_addr = 0;
+ rxd->hdr_addr = 0;
+ }
+
+ rxq->rx_tail = 0;
+}
+
+int
+atl_rx_queue_setup(struct rte_eth_dev *dev, uint16_t rx_queue_id,
+ uint16_t nb_rx_desc, unsigned int socket_id,
+ const struct rte_eth_rxconf *rx_conf,
+ struct rte_mempool *mb_pool)
+{
+ struct atl_rx_queue *rxq;
+ const struct rte_memzone *mz;
+
+ PMD_INIT_FUNC_TRACE();
+
+ /* make sure a valid number of descriptors have been requested */
+ if (nb_rx_desc < AQ_HW_MIN_RX_RING_SIZE || nb_rx_desc >
AQ_HW_MAX_RX_RING_SIZE) {
+ PMD_INIT_LOG(ERR, "Number of Rx descriptors must be "
+ "less than or equal to %d, "
+ "greater than or equal to %d", AQ_HW_MAX_RX_RING_SIZE,
AQ_HW_MIN_RX_RING_SIZE);
+ return -EINVAL;
+ }
+
+ /*
+ * if this queue existed already, free the associated memory. The
+ * queue cannot be reused in case we need to allocate memory on
+ * different socket than was previously used.
+ */
+ if (dev->data->rx_queues[rx_queue_id] != NULL) {
+ atl_rx_queue_release(dev->data->rx_queues[rx_queue_id]);
+ dev->data->rx_queues[rx_queue_id] = NULL;
+ }
+
+ /* allocate memory for the queue structure */
+ rxq = rte_zmalloc_socket("atlantic Rx queue", sizeof(*rxq),
RTE_CACHE_LINE_SIZE, socket_id);
+ if (rxq == NULL) {
+ PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
+ return -ENOMEM;
+ }
+
+ /* setup queue */
+ rxq->mb_pool = mb_pool;
+ rxq->nb_rx_desc = nb_rx_desc;
+ rxq->port_id = dev->data->port_id;
+ rxq->queue_id = rx_queue_id;
+ rxq->rx_free_thresh = rx_conf->rx_free_thresh;
+
+ rxq->l3_csum_enabled = dev->data->dev_conf.rxmode.offloads &
DEV_RX_OFFLOAD_IPV4_CKSUM;
+ rxq->l4_csum_enabled = dev->data->dev_conf.rxmode.offloads &
+ (DEV_RX_OFFLOAD_UDP_CKSUM | DEV_RX_OFFLOAD_TCP_CKSUM);
+
+ /* allocate memory for the software ring */
+ rxq->sw_ring = rte_zmalloc_socket("atlantic sw rx ring",
+ nb_rx_desc * sizeof(struct
atl_rx_entry),
+ RTE_CACHE_LINE_SIZE, socket_id);
+ if (rxq->sw_ring == NULL) {
+ PMD_INIT_LOG(ERR, "Cannot allocate software ring");
+ rte_free(rxq);
+ return -ENOMEM;
+ }
+
+ /*
+ * allocate memory for the hardware descriptor ring. A memzone large
+ * enough to hold the maximum ring size is requested to allow for
+ * resizing in later calls to the queue setup function.
+ */
+ mz = rte_eth_dma_zone_reserve(dev, "rx hw_ring", rx_queue_id,
+ HW_ATL_B0_MAX_RXD * sizeof(struct
hw_atl_rxd_s),
+ 128, socket_id);
+ if (mz == NULL) {
+ PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
+ rte_free(rxq->sw_ring);
+ rte_free(rxq);
+ return -ENOMEM;
+ }
+ rxq->hw_ring = mz->addr;
+ rxq->hw_ring_phys_addr = mz->iova;
+
+ atl_reset_rx_queue(rxq);
+
+ dev->data->rx_queues[rx_queue_id] = rxq;
+ return 0;
+}
+
+int
+atl_rx_init(struct rte_eth_dev *eth_dev)
+{
+ struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
+ struct aq_rss_parameters *rss_params = &hw->aq_nic_cfg->aq_rss;
+ struct atl_rx_queue *rxq;
+ uint64_t base_addr = 0;
+ int i = 0;
+ int err = 0;
+
+ PMD_INIT_FUNC_TRACE();
+
+ for (i =0 ; i < eth_dev->data->nb_rx_queues; i++) {
+ rxq = eth_dev->data->rx_queues[i];
+ base_addr = rxq->hw_ring_phys_addr;
+
+ /* Take requested pool mbuf size and adapt descriptor buffer to
best fit */
+ int buff_size = rte_pktmbuf_data_room_size(rxq->mb_pool) -
+ RTE_PKTMBUF_HEADROOM;
+
+ buff_size = RTE_ALIGN_FLOOR(buff_size, 1024);
+ if (buff_size > HW_ATL_B0_RXD_BUF_SIZE_MAX) {
+ PMD_INIT_LOG(WARNING, "queue %d: mem pool buff size is
larger than max supported\n", rxq->queue_id);
+ buff_size = HW_ATL_B0_RXD_BUF_SIZE_MAX;
+ }
+ if (buff_size < 1024) {
+ PMD_INIT_LOG(ERR, "queue %d: mem pool buff size is too
small\n", rxq->queue_id);
+ return -EINVAL;
+ }
+ rxq->buff_size = buff_size;
+
+ err = hw_atl_b0_hw_ring_rx_init(hw, base_addr, rxq->queue_id,
rxq->nb_rx_desc, buff_size, 0, rxq->port_id);
+ }
+
+ for (i = rss_params->indirection_table_size; i--;)
+ rss_params->indirection_table[i] = i &
(eth_dev->data->nb_rx_queues - 1);
+ hw_atl_b0_hw_rss_set(hw, rss_params);
+ return err;
+}
+
+static int
+atl_alloc_rx_queue_mbufs(struct atl_rx_queue *rxq)
+{
+ struct atl_rx_entry *rx_entry = rxq->sw_ring;
+ struct hw_atl_rxd_s *rxd;
+ uint64_t dma_addr = 0;
+ uint32_t i = 0;
+
+ PMD_INIT_FUNC_TRACE();
+
+ /* fill Rx ring */
+ for (i = 0; i < rxq->nb_rx_desc; i++) {
+ struct rte_mbuf *mbuf = rte_mbuf_raw_alloc(rxq->mb_pool);
+
+ if (mbuf == NULL) {
+ PMD_INIT_LOG(ERR, "mbuf alloca failed for rx queue %u",
(unsigned)rxq->queue_id);
+ return -ENOMEM;
+ }
+
+ mbuf->data_off = RTE_PKTMBUF_HEADROOM;
+ mbuf->port = rxq->port_id;
+
+ dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf));
+ rxd = (struct hw_atl_rxd_s *)&rxq->hw_ring[i];
+ rxd->buf_addr = dma_addr;
+ rxd->hdr_addr = 0;
+ rx_entry[i].mbuf = mbuf;
+ }
+
+ return 0;
+}
+
+static void
+atl_rx_queue_release_mbufs(struct atl_rx_queue *rxq)
+{
+ int i;
+
+ PMD_INIT_FUNC_TRACE();
+
+ if (rxq->sw_ring != NULL) {
+ for (i = 0; i < rxq->nb_rx_desc; i++) {
+ if (rxq->sw_ring[i].mbuf != NULL) {
+ rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf);
+ rxq->sw_ring[i].mbuf = NULL;
+ }
+ }
+ }
+}
+
+int
+atl_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
+{
+ struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+ struct atl_rx_queue *rxq = NULL;
+
+ PMD_INIT_FUNC_TRACE();
+
+ if (rx_queue_id < dev->data->nb_rx_queues) {
+ rxq = dev->data->rx_queues[rx_queue_id];
+
+ if (atl_alloc_rx_queue_mbufs(rxq) != 0) {
+ PMD_INIT_LOG(ERR, "Allocate mbufs for queue %d failed",
rx_queue_id);
+ return -1;
+ }
+
+ hw_atl_b0_hw_ring_rx_start(hw, rx_queue_id);
+
+ rte_wmb();
+ hw_atl_reg_rx_dma_desc_tail_ptr_set(hw, rxq->nb_rx_desc -1,
rx_queue_id);
+ dev->data->rx_queue_state[rx_queue_id] =
RTE_ETH_QUEUE_STATE_STARTED;
+ } else {
+ return -1;
+ }
+
+ return 0;
+}
+
+int
+atl_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
+{
+ struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+ struct atl_rx_queue *rxq = NULL;
+
+ PMD_INIT_FUNC_TRACE();
+
+ if (rx_queue_id < dev->data->nb_rx_queues) {
+ rxq = dev->data->rx_queues[rx_queue_id];
+
+ hw_atl_b0_hw_ring_rx_stop(hw, rx_queue_id);
+
+ atl_rx_queue_release_mbufs(rxq);
+ atl_reset_rx_queue(rxq);
+
+ dev->data->rx_queue_state[rx_queue_id] =
RTE_ETH_QUEUE_STATE_STOPPED;
+ } else {
+ return -1;
+ }
+
+ return 0;
+}
+
+void
+atl_rx_queue_release(void *rx_queue)
+{
+ PMD_INIT_FUNC_TRACE();
+
+ if (rx_queue != NULL) {
+ struct atl_rx_queue *rxq = (struct atl_rx_queue *)rx_queue;
+
+ atl_rx_queue_release_mbufs(rxq);
+ rte_free(rxq->sw_ring);
+ rte_free(rxq);
+ }
+}
+
+void
+atl_free_queues(struct rte_eth_dev *dev)
+{
+ unsigned i;
+
+ PMD_INIT_FUNC_TRACE();
+
+ for (i = 0; i < dev->data->nb_rx_queues; i++) {
+ atl_rx_queue_release(dev->data->rx_queues[i]);
+ dev->data->rx_queues[i] = 0;
+ }
+ dev->data->nb_rx_queues = 0;
+
+}
+
+int
+atl_start_queues(struct rte_eth_dev *dev)
+{
+ int i;
+
+ PMD_INIT_FUNC_TRACE();
+
+ for (i = 0; i < dev->data->nb_rx_queues; i++) {
+ if (atl_rx_queue_start(dev, i) != 0) {
+ PMD_DRV_LOG(ERR, "Start Rx queue %d failed", i);
+ return -1;
+ }
+ }
+
+ return 0;
+}
+
+int
+atl_stop_queues(struct rte_eth_dev *dev)
+{
+ int i;
+
+ PMD_INIT_FUNC_TRACE();
+
+ for (i = 0; i < dev->data->nb_rx_queues; i++) {
+ if (atl_rx_queue_stop(dev, i) != 0) {
+ PMD_DRV_LOG(ERR, "Stop Rx queue %d failed", i);
+ return -1;
+ }
+ }
+
+ return 0;
+}
+
+void
+atl_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, struct
rte_eth_rxq_info *qinfo)
+{
+ struct atl_rx_queue *rxq;
+
+ PMD_INIT_FUNC_TRACE();
+
+ rxq = dev->data->rx_queues[queue_id];
+
+ qinfo->mp = rxq->mb_pool;
+ qinfo->scattered_rx = dev->data->scattered_rx;
+ qinfo->nb_desc = rxq->nb_rx_desc;
+}
+
+/* Return Rx queue avail count */
+
+uint32_t
+atl_rx_queue_count(struct rte_eth_dev *dev, uint16_t rx_queue_id)
+{
+ struct atl_rx_queue *rxq;
+
+ PMD_INIT_FUNC_TRACE();
+
+ if (rx_queue_id >= dev->data->nb_rx_queues) {
+ PMD_DRV_LOG(ERR, "Invalid RX queue id=%d", rx_queue_id);
+ return 0;
+ }
+
+ rxq = dev->data->rx_queues[rx_queue_id];
+
+ if (rxq == NULL)
+ return 0;
+
+ return rxq->nb_rx_desc - rxq->nb_rx_hold;
+}
+
+int
+atl_dev_rx_descriptor_done(void *rx_queue, uint16_t offset)
+{
+ struct atl_rx_queue *rxq = rx_queue;
+ struct hw_atl_rxd_wb_s *rxd;
+ uint32_t idx;
+
+ PMD_INIT_FUNC_TRACE();
+
+ if (unlikely(offset >= rxq->nb_rx_desc))
+ return 0;
+
+ idx = rxq->rx_tail + offset;
+
+ if (idx >= rxq->nb_rx_desc)
+ idx -= rxq->nb_rx_desc;
+
+ rxd = (struct hw_atl_rxd_wb_s *)&rxq->hw_ring[idx];
+
+ return rxd->dd;
+}
+
+int
+atl_dev_rx_descriptor_status(void *rx_queue, uint16_t offset)
+{
+ struct atl_rx_queue *rxq = rx_queue;
+ struct hw_atl_rxd_wb_s *rxd;
+ uint32_t idx;
+
+ PMD_INIT_FUNC_TRACE();
+
+ if (unlikely(offset >= rxq->nb_rx_desc))
+ return -EINVAL;
+
+ if (offset >= rxq->nb_rx_desc - rxq->nb_rx_hold)
+ return RTE_ETH_RX_DESC_UNAVAIL;
+
+ idx = rxq->rx_tail + offset;
+
+ if (idx >= rxq->nb_rx_desc)
+ idx -= rxq->nb_rx_desc;
+
+ rxd = (struct hw_atl_rxd_wb_s *)&rxq->hw_ring[idx];
+
+ if (rxd->dd)
+ return RTE_ETH_RX_DESC_DONE;
+
+ return RTE_ETH_RX_DESC_AVAIL;
+}
+
+static int
+atl_rx_enable_intr(struct rte_eth_dev *dev, uint16_t queue_id, bool enable)
+{
+ struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+ struct atl_rx_queue *rxq;
+
+ PMD_INIT_FUNC_TRACE();
+
+ if (queue_id >= dev->data->nb_rx_queues) {
+ PMD_DRV_LOG(ERR, "Invalid RX queue id=%d", queue_id);
+ return -EINVAL;
+ }
+
+ rxq = dev->data->rx_queues[queue_id];
+
+ if (rxq == NULL)
+ return 0;
+
+ /* Mapping interrupt vector */
+ hw_atl_itr_irq_map_en_rx_set(hw, enable, queue_id);
+
+ return 0;
+}
+
+int
+atl_dev_rx_queue_intr_enable(struct rte_eth_dev *eth_dev, uint16_t queue_id)
+{
+ return atl_rx_enable_intr(eth_dev, queue_id, true);
+}
+
+int
+atl_dev_rx_queue_intr_disable(struct rte_eth_dev *eth_dev, uint16_t queue_id)
+{
+ return atl_rx_enable_intr(eth_dev, queue_id, false);
+}
+
+static uint64_t
+atl_desc_to_offload_flags(struct atl_rx_queue *rxq, struct hw_atl_rxd_wb_s
*rxd_wb)
+{
+ uint64_t mbuf_flags = 0;
+
+ PMD_INIT_FUNC_TRACE();
+
+ if (rxq->l3_csum_enabled && ((rxd_wb->pkt_type & 0x3) == 0)) { //IPv4
+ if (rxd_wb->rx_stat & BIT(1)) { //IPv4 csum error
+ mbuf_flags |= PKT_RX_IP_CKSUM_BAD;
+ } else {
+ mbuf_flags |= PKT_RX_IP_CKSUM_GOOD;
+ }
+ } else {
+ mbuf_flags |= PKT_RX_IP_CKSUM_UNKNOWN;
+ }
+
+ if (rxq->l4_csum_enabled && (rxd_wb->rx_stat & BIT(3))) { //CSUM
calculated
+ if (rxd_wb->rx_stat & BIT(2)) {
+ mbuf_flags |= PKT_RX_L4_CKSUM_BAD;
+ } else {
+ mbuf_flags |= PKT_RX_L4_CKSUM_GOOD;
+ }
+ } else {
+ mbuf_flags |= PKT_RX_L4_CKSUM_UNKNOWN;
+ }
+
+ return mbuf_flags;
+}
+
+static uint32_t
+atl_desc_to_pkt_type(struct hw_atl_rxd_wb_s *rxd_wb)
+{
+ uint32_t type = RTE_PTYPE_UNKNOWN;
+ uint16_t l2_l3_type = rxd_wb->pkt_type & 0x3;
+ uint16_t l4_type = (rxd_wb->pkt_type & 0x1C) >> 2;
+
+ switch (l2_l3_type) {
+ case 0:
+ type = RTE_PTYPE_L3_IPV4;
+ break;
+ case 1:
+ type = RTE_PTYPE_L3_IPV6;
+ break;
+ case 2:
+ type = RTE_PTYPE_L2_ETHER;
+ break;
+ case 3:
+ type = RTE_PTYPE_L2_ETHER_ARP;
+ break;
+ }
+
+ switch (l4_type) {
+ case 0:
+ type |= RTE_PTYPE_L4_TCP;
+ break;
+ case 1:
+ type |= RTE_PTYPE_L4_UDP;
+ break;
+ case 2:
+ type |= RTE_PTYPE_L4_SCTP;
+ break;
+ case 3:
+ type |= RTE_PTYPE_L4_ICMP;
+ break;
+ }
+
+ if (rxd_wb->pkt_type & BIT(5))
+ type |= RTE_PTYPE_L2_ETHER_VLAN;
+
+ return type;
+}
+
+uint16_t
+atl_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
+{
+ struct atl_rx_queue *rxq = (struct atl_rx_queue *)rx_queue;
+ struct rte_eth_dev *dev = &rte_eth_devices[rxq->port_id];
+ struct atl_adapter *adapter =
ATL_DEV_TO_ADAPTER(&rte_eth_devices[rxq->port_id]);
+ struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(adapter);
+ struct atl_rx_entry *sw_ring = rxq->sw_ring;
+
+ struct rte_mbuf *new_mbuf;
+ struct rte_mbuf *rx_mbuf, *rx_mbuf_prev, *rx_mbuf_first;
+ struct atl_rx_entry *rx_entry;
+ uint16_t nb_rx = 0;
+ uint16_t nb_hold = 0;
+ struct hw_atl_rxd_wb_s rxd_wb;
+ struct hw_atl_rxd_s *rxd = NULL;
+ uint16_t tail = rxq->rx_tail;
+ uint64_t dma_addr;
+ uint16_t pkt_len = 0;
+
+ while (nb_rx < nb_pkts) {
+ uint16_t eop_tail = tail;
+
+ rxd = (struct hw_atl_rxd_s *)&rxq->hw_ring[tail];
+ rxd_wb = *(struct hw_atl_rxd_wb_s *)rxd;
+
+ if (!rxd_wb.dd) { /* RxD is not done */
+ break;
+ }
+
+ PMD_RX_LOG(ERR, "port_id=%u queue_id=%u tail=%u "
+ "eop=0x%x pkt_len=%u hash=0x%x hash_type=0x%x",
+ (unsigned)rxq->port_id, (unsigned) rxq->queue_id,
+ (unsigned)tail, (unsigned)rxd_wb.eop,
+ (unsigned)rte_le_to_cpu_16(rxd_wb.pkt_len),
+ rxd_wb.rss_hash, rxd_wb.rss_type);
+
+ /* RxD is not done */
+ if (!rxd_wb.eop) {
+ while(true) {
+ struct hw_atl_rxd_wb_s *eop_rxwbd;
+
+ eop_tail = (eop_tail+1) % rxq->nb_rx_desc;
+ eop_rxwbd = (struct
hw_atl_rxd_wb_s*)&rxq->hw_ring[eop_tail];
+ if (!eop_rxwbd->dd) {
+ /* no EOP received yet */
+ eop_tail = tail;
+ break;
+ }
+ if (eop_rxwbd->dd && eop_rxwbd->eop)
+ break;
+ }
+ /* No EOP in ring */
+ if (eop_tail == tail)
+ break;
+ }
+ rx_mbuf_prev = rx_mbuf_first = NULL;
+
+ /* Run through packet segments */
+ while(true) {
+
+ new_mbuf = rte_mbuf_raw_alloc(rxq->mb_pool);
+ if (new_mbuf == NULL) {
+ PMD_RX_LOG(ERR, "RX mbuf alloc failed
port_id=%u "
+ "queue_id=%u", (unsigned)
rxq->port_id,
+ (unsigned) rxq->queue_id);
+ dev->data->rx_mbuf_alloc_failed++;
+ adapter->sw_stats.rx_nombuf++;
+ goto err_stop;
+ }
+
+ nb_hold++;
+ rx_entry = &sw_ring[tail];
+
+ rx_mbuf = rx_entry->mbuf;
+ rx_entry->mbuf = new_mbuf;
+ dma_addr =
rte_cpu_to_le_64(rte_mbuf_data_iova_default(new_mbuf));
+
+ /* setup RX descriptor */
+ rxd->hdr_addr = 0;
+ rxd->buf_addr = dma_addr;
+
+ /*
+ * Initialize the returned mbuf.
+ * 1) setup generic mbuf fields:
+ * - number of segments,
+ * - next segment,
+ * - packet length,
+ * - RX port identifier.
+ * 2) integrate hardware offload data, if any:
+ * < - RSS flag & hash,
+ * - IP checksum flag,
+ * - VLAN TCI, if any,
+ * - error flags.
+ */
+ pkt_len = (uint16_t)rte_le_to_cpu_16(rxd_wb.pkt_len);
+ rx_mbuf->data_off = RTE_PKTMBUF_HEADROOM;
+ rte_prefetch1((char *)rx_mbuf->buf_addr +
rx_mbuf->data_off);
+ rx_mbuf->nb_segs = 0;
+ rx_mbuf->next = NULL;
+ rx_mbuf->pkt_len = pkt_len;
+ rx_mbuf->data_len = pkt_len;
+ if (rxd_wb.eop) {
+ u16 remainder_len = pkt_len % rxq->buff_size;
+ if (!remainder_len)
+ remainder_len = rxq->buff_size;
+ rx_mbuf->data_len = remainder_len;
+ } else {
+ rx_mbuf->data_len = pkt_len > rxq->buff_size ?
+
rxq->buff_size : pkt_len;
+ }
+ rx_mbuf->port = rxq->port_id;
+
+ rx_mbuf->hash.rss = rxd_wb.rss_hash;
+
+ rx_mbuf->vlan_tci = rxd_wb.vlan;
+
+ //hlen_type_rss = rte_le_to_cpu_32(rxd_wb->type);
+ //pkt_flags = rx_desc_hlen_type_rss_to_pkt_flags(rxq,
hlen_type_rss);
+
+ rx_mbuf->ol_flags = atl_desc_to_offload_flags(rxq,
&rxd_wb);
+ rx_mbuf->packet_type = atl_desc_to_pkt_type(&rxd_wb);
+
+ if (!rx_mbuf_first)
+ rx_mbuf_first = rx_mbuf;
+ rx_mbuf_first->nb_segs++;
+
+ if (rx_mbuf_prev)
+ rx_mbuf_prev->next = rx_mbuf;
+ rx_mbuf_prev = rx_mbuf;
+
+ tail = (tail+1) % rxq->nb_rx_desc;
+ /* Prefetch next mbufs */
+ rte_prefetch0(sw_ring[tail].mbuf);
+ if ((tail & 0x3) == 0) {
+ rte_prefetch0(&sw_ring[tail]);
+ rte_prefetch0(&sw_ring[tail]);
+ }
+
+ /* filled mbuf_first */
+ if (rxd_wb.eop) {
+ break;
+ }
+ rxd = (struct hw_atl_rxd_s *)&rxq->hw_ring[tail];
+ rxd_wb = *(struct hw_atl_rxd_wb_s *)rxd;
+ };
+
+ /*
+ * Store the mbuf address into the next entry of the array
+ * of returned packets.
+ */
+ rx_pkts[nb_rx++] = rx_mbuf_first;
+ adapter->sw_stats.q_ipackets[rxq->queue_id]++;
+ adapter->sw_stats.q_ibytes[rxq->queue_id] +=
rx_mbuf_first->pkt_len;
+
+ PMD_RX_LOG(ERR, "add mbuf segs=%d pkt_len=%d",
rx_mbuf_first->nb_segs, rx_mbuf_first->pkt_len);
+ }
+
+err_stop:
+
+ rxq->rx_tail = tail;
+
+ /*
+ * If the number of free RX descriptors is greater than the RX free
+ * threshold of the queue, advance the Receive Descriptor Tail (RDT)
+ * register.
+ * Update the RDT with the value of the last processed RX descriptor
+ * minus 1, to guarantee that the RDT register is never equal to the
+ * RDH register, which creates a "full" ring situtation from the
+ * hardware point of view...
+ */
+ nb_hold = (uint16_t) (nb_hold + rxq->nb_rx_hold);
+ if (nb_hold > rxq->rx_free_thresh) {
+ PMD_RX_LOG(ERR, "port_id=%u queue_id=%u rx_tail=%u "
+ "nb_hold=%u nb_rx=%u",
+ (unsigned) rxq->port_id, (unsigned) rxq->queue_id,
+ (unsigned) tail, (unsigned) nb_hold, (unsigned) nb_rx);
+ tail = (uint16_t) ((tail == 0) ? (rxq->nb_rx_desc - 1) : (tail
- 1));
+
+ hw_atl_reg_rx_dma_desc_tail_ptr_set(hw, tail, rxq->queue_id);
+
+ nb_hold = 0;
+ }
+
+ rxq->nb_rx_hold = nb_hold;
+
+ return nb_rx;
+}
diff --git a/drivers/net/atlantic/meson.build b/drivers/net/atlantic/meson.build
index 19fa41cd3..42821f35a 100644
--- a/drivers/net/atlantic/meson.build
+++ b/drivers/net/atlantic/meson.build
@@ -4,8 +4,14 @@
#subdir('hw_atl')
sources = files(
+ 'atl_rxtx.c',
'atl_ethdev.c',
+ 'atl_hw_regs.c',
'rte_pmd_atlantic.c',
+ 'hw_atl/hw_atl_b0.c',
+ 'hw_atl/hw_atl_llh.c',
+ 'hw_atl/hw_atl_utils_fw2x.c',
+ 'hw_atl/hw_atl_utils.c',
)
deps += ['hash', 'eal']
--
2.13.3.windows.1
