Signed-off-by: Andrew Rybchenko <arybche...@solarflare.com>
---
doc/guides/nics/sfc_efx.rst | 23 +-
drivers/net/sfc/Makefile | 1 +
drivers/net/sfc/efsys.h | 2 +-
drivers/net/sfc/sfc_dp.h | 5 +-
drivers/net/sfc/sfc_dp_rx.h | 8 +
drivers/net/sfc/sfc_ef10_ps_rx.c | 659 +++++++++++++++++++++++++++++++++++++++
drivers/net/sfc/sfc_ethdev.c | 7 +
drivers/net/sfc/sfc_ev.c | 34 ++
drivers/net/sfc/sfc_kvargs.h | 4 +-
drivers/net/sfc/sfc_rx.c | 5 +-
drivers/net/sfc/sfc_rx.h | 2 +-
11 files changed, 744 insertions(+), 6 deletions(-)
create mode 100644 drivers/net/sfc/sfc_ef10_ps_rx.c
diff --git a/doc/guides/nics/sfc_efx.rst b/doc/guides/nics/sfc_efx.rst
index 8e2b36f..d52ab45 100644
--- a/doc/guides/nics/sfc_efx.rst
+++ b/doc/guides/nics/sfc_efx.rst
@@ -114,6 +114,24 @@ required in the receive buffer.
It should be taken into account when mbuf pool for receive is created.
+Packed stream mode
+~~~~~~~~~~~~~~~~~~
+
+When the receive queue is in packed stream mode, the driver handles mbufs
+differently.
+The mbufs that are handed over to the application are always indirect mbufs.
+The contents of mbufs may be freely modified by the application, with an
+important constraint: the indirect mbufs does not have any reserved "head room"
+before the actual contents (or rather it is used by PMD internally),
+so the usual practice of prepending an extra header through manipulating
+the head room **will not** work and would result in corrupted packets.
+If one needs to prepend data to a packet, one should first create a copy of
+mbuf.
+
+Another limitation of a packed stream mode, imposed by the firmware, is that
+it allows for a single RSS context.
+
+
Supported NICs
--------------
@@ -181,7 +199,7 @@ whitelist option like "-w 02:00.0,arg1=value1,...".
Case-insensitive 1/y/yes/on or 0/n/no/off may be used to specify
boolean parameters value.
-- ``rx_datapath`` [auto|efx|ef10] (default **auto**)
+- ``rx_datapath`` [auto|efx|ef10|ef10_packed] (default **auto**)
Choose receive datapath implementation.
**auto** allows the driver itself to make a choice based on firmware
@@ -190,6 +208,9 @@ boolean parameters value.
**ef10** chooses EF10 (SFN7xxx, SFN8xxx) native datapath which is
more efficient than libefx-based and provides richer packet type
classification, but lacks Rx scatter support.
+ **ef10_packed** chooses EF10 (SFN7xxx, SFN8xxx) packed stream datapath
+ which may be used on capture packed stream firmware variant only
+ (see notes about its limitations above).
- ``tx_datapath`` [auto|efx|ef10|ef10_simple] (default **auto**)
diff --git a/drivers/net/sfc/Makefile b/drivers/net/sfc/Makefile
index bb7dcb2..dd83238 100644
--- a/drivers/net/sfc/Makefile
+++ b/drivers/net/sfc/Makefile
@@ -92,6 +92,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += sfc_tx.c
SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += sfc_tso.c
SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += sfc_dp.c
SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += sfc_ef10_rx.c
+SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += sfc_ef10_ps_rx.c
SRCS-$(CONFIG_RTE_LIBRTE_SFC_EFX_PMD) += sfc_ef10_tx.c
VPATH += $(SRCDIR)/base
diff --git a/drivers/net/sfc/efsys.h b/drivers/net/sfc/efsys.h
index 60829be..98e591c 100644
--- a/drivers/net/sfc/efsys.h
+++ b/drivers/net/sfc/efsys.h
@@ -210,7 +210,7 @@
#define EFSYS_OPT_ALLOW_UNCONFIGURED_NIC 0
-#define EFSYS_OPT_RX_PACKED_STREAM 0
+#define EFSYS_OPT_RX_PACKED_STREAM 1
/* ID */
diff --git a/drivers/net/sfc/sfc_dp.h b/drivers/net/sfc/sfc_dp.h
index d3e7007..362a998 100644
--- a/drivers/net/sfc/sfc_dp.h
+++ b/drivers/net/sfc/sfc_dp.h
@@ -37,6 +37,8 @@
extern "C" {
#endif
+#define SFC_P2_ROUND_UP(x, align) (-(-(x) & -(align)))
+
#define SFC_DIV_ROUND_UP(a, b) \
__extension__ ({ \
typeof(a) _a = (a); \
@@ -62,7 +64,8 @@ struct sfc_dp {
enum sfc_dp_type type;
/* Mask of required hardware/firmware capabilities */
unsigned int hw_fw_caps;
-#define SFC_DP_HW_FW_CAP_EF10 0x1
+#define SFC_DP_HW_FW_CAP_EF10 0x1
+#define SFC_DP_HW_FW_CAP_RX_PACKED_STREAM_64K 0x2
};
/** List of datapath variants */
diff --git a/drivers/net/sfc/sfc_dp_rx.h b/drivers/net/sfc/sfc_dp_rx.h
index 944d366..0541c43 100644
--- a/drivers/net/sfc/sfc_dp_rx.h
+++ b/drivers/net/sfc/sfc_dp_rx.h
@@ -154,6 +154,12 @@ typedef void (sfc_dp_rx_qstop_t)(struct sfc_dp_rxq *dp_rxq,
typedef bool (sfc_dp_rx_qrx_ev_t)(struct sfc_dp_rxq *dp_rxq, unsigned int id);
/**
+ * Packed stream receive event handler used during queue flush only.
+ */
+typedef bool (sfc_dp_rx_qrx_ps_ev_t)(struct sfc_dp_rxq *dp_rxq,
+ unsigned int id);
+
+/**
* Receive queue purge function called after queue flush.
*
* Should be used to free unused recevie buffers.
@@ -177,6 +183,7 @@ struct sfc_dp_rx {
sfc_dp_rx_qstart_t *qstart;
sfc_dp_rx_qstop_t *qstop;
sfc_dp_rx_qrx_ev_t *qrx_ev;
+ sfc_dp_rx_qrx_ps_ev_t *qrx_ps_ev;
sfc_dp_rx_qpurge_t *qpurge;
sfc_dp_rx_supported_ptypes_get_t *supported_ptypes_get;
sfc_dp_rx_qdesc_npending_t *qdesc_npending;
@@ -201,6 +208,7 @@ struct sfc_dp_rx {
extern struct sfc_dp_rx sfc_efx_rx;
extern struct sfc_dp_rx sfc_ef10_rx;
+extern struct sfc_dp_rx sfc_ef10_ps_rx;
#ifdef __cplusplus
}
diff --git a/drivers/net/sfc/sfc_ef10_ps_rx.c b/drivers/net/sfc/sfc_ef10_ps_rx.c
new file mode 100644
index 0000000..6365092
--- /dev/null
+++ b/drivers/net/sfc/sfc_ef10_ps_rx.c
@@ -0,0 +1,659 @@
+/*-
+ * Copyright (c) 2017 Solarflare Communications Inc.
+ * All rights reserved.
+ *
+ * This software was jointly developed between OKTET Labs (under contract
+ * for Solarflare) and Solarflare Communications, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+ * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* EF10 packed stream native datapath implementation */
+
+#include <stdbool.h>
+
+#include <rte_byteorder.h>
+#include <rte_mbuf_ptype.h>
+#include <rte_mbuf.h>
+#include <rte_io.h>
+
+#include "efx.h"
+#include "efx_types.h"
+#include "efx_regs.h"
+#include "efx_regs_ef10.h"
+
+#include "sfc_tweak.h"
+#include "sfc_dp_rx.h"
+#include "sfc_kvargs.h"
+
+#if 1
+/* Alignment requirement for value written to RX WPTR:
+ * the WPTR must be aligned to an 8 descriptor boundary
+ */
+#define EF10_RX_WPTR_ALIGN 8
+#define EFX_RX_PACKED_STREAM_ALIGNMENT 64
+#define EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE 8
+#endif
+
+#define SFC_PACKED_STREAM_BUFSIZE (64 * 1024)
+
+
+struct sfc_ef10_ps_rx_sw_desc {
+ struct rte_mbuf *mbuf;
+};
+
+struct sfc_ef10_ps_rxq {
+ /* Used on data path */
+ unsigned int flags;
+#define SFC_EF10_PS_RXQ_STARTED 0x1
+#define SFC_EF10_PS_RXQ_RUNNING 0x2
+#define SFC_EF10_PS_RXQ_EXCEPTION 0x4
+ unsigned int rxq_ptr_mask;
+ unsigned int completed;
+ unsigned int pending_pkts;
+ const uint8_t *next_pkt;
+ unsigned int packets;
+ unsigned int evq_read_ptr;
+ unsigned int evq_ptr_mask;
+ volatile efx_qword_t *evq_hw_ring;
+ struct sfc_ef10_ps_rx_sw_desc *sw_ring;
+ struct rte_mempool *indirect_mb_pool;
+ uint8_t port_id;
+ uint8_t credits;
+
+ /* Used on refill */
+ unsigned int added;
+ unsigned int refill_threshold;
+ struct rte_mempool *refill_mb_pool;
+ efx_qword_t *rxq_hw_ring;
+ volatile void *doorbell;
+
+ /* Datapath receive queue anchor */
+ struct sfc_dp_rxq dp;
+ void *ctrl;
+ sfc_dp_exception_t *exception;
+};
+
+static inline struct sfc_ef10_ps_rxq *
+sfc_ef10_ps_rxq_by_dp_rxq(struct sfc_dp_rxq *dp_rxq)
+{
+ return container_of(dp_rxq, struct sfc_ef10_ps_rxq, dp);
+}
+
+static void
+sfc_ef10_ps_rx_qpush(struct sfc_ef10_ps_rxq *rxq)
+{
+ efx_dword_t dword;
+
+ /* Hardware has alignment restriction for WPTR */
+ RTE_BUILD_BUG_ON(SFC_RX_REFILL_BULK % EF10_RX_WPTR_ALIGN != 0);
+ SFC_ASSERT(RTE_ALIGN(rxq->added, EF10_RX_WPTR_ALIGN) == rxq->added);
+
+ EFX_POPULATE_DWORD_1(dword, ERF_DZ_RX_DESC_WPTR,
+ rxq->added & rxq->rxq_ptr_mask);
+
+ /* Make sure that all descriptor update (Rx and event) reach memory */
+ rte_wmb();
+
+ /* DMA sync to device is not required */
+
+ rte_write32(dword.ed_u32[0], rxq->doorbell);
+}
+
+static void
+sfc_ef10_ps_rx_update_credits(struct sfc_ef10_ps_rxq *rxq)
+{
+ efx_dword_t dword;
+
+ if (rxq->credits == 0)
+ return;
+
+ EFX_POPULATE_DWORD_3(dword,
+ ERF_DZ_RX_DESC_MAGIC_DOORBELL, 1,
+ ERF_DZ_RX_DESC_MAGIC_CMD,
+ ERE_DZ_RX_DESC_MAGIC_CMD_PS_CREDITS,
+ ERF_DZ_RX_DESC_MAGIC_DATA, rxq->credits);
+
+ /* Make sure that event descriptor update reach memory */
+ rte_wmb();
+
+ /* DMA sync to device is not required */
+
+ rte_write32(dword.ed_u32[0], rxq->doorbell);
+
+ rxq->credits = 0;
+}
+
+static void
+sfc_ef10_ps_rx_qrefill(struct sfc_ef10_ps_rxq *rxq)
+{
+ const unsigned int rxq_ptr_mask = rxq->rxq_ptr_mask;
+ unsigned int free_space;
+ unsigned int bulks;
+ void *objs[SFC_RX_REFILL_BULK];
+ unsigned int added = rxq->added;
+
+ free_space = EFX_RXQ_LIMIT(rxq_ptr_mask + 1) - (added - rxq->completed);
+
+ if (free_space < rxq->refill_threshold)
+ return;
+
+ bulks = free_space / RTE_DIM(objs);
+
+ while (bulks-- > 0) {
+ unsigned int id;
+ unsigned int i;
+
+ if (unlikely(rte_mempool_get_bulk(rxq->refill_mb_pool, objs,
+ RTE_DIM(objs)) < 0)) {
+ struct rte_eth_dev_data *dev_data =
+ rte_eth_devices[rxq->port_id].data;
+
+ /*
+ * It is hardly a safe way to increment counter
+ * from different contexts, but all PMDs do it.
+ */
+ dev_data->rx_mbuf_alloc_failed += RTE_DIM(objs);
+ break;
+ }
+
+ for (i = 0, id = added & rxq_ptr_mask;
+ i < RTE_DIM(objs);
+ ++i, ++id) {
+ struct rte_mbuf *m = objs[i];
+ struct sfc_ef10_ps_rx_sw_desc *rxd;
+ unsigned int req_align;
+ uintptr_t dp;
+ uintptr_t adjust_align;
+
+ SFC_ASSERT((id & ~rxq_ptr_mask) == 0);
+ rxd = &rxq->sw_ring[id];
+ rxd->mbuf = m;
+
+ rte_mbuf_refcnt_set(m, 1);
+
+ req_align = SFC_PACKED_STREAM_BUFSIZE;
+ dp = rte_pktmbuf_mtophys(m) + req_align;
+ adjust_align = RTE_ALIGN_CEIL(dp, req_align) - dp;
+
+ /*
+ * Align using priv_size to be able to
+ * find correct address of the mbuf on
+ * detach (see rte_mbuf_from_indirect()).
+ */
+ m->buf_addr = RTE_PTR_ADD(m->buf_addr, adjust_align);
+ m->buf_physaddr += adjust_align;
+ m->priv_size += adjust_align;
+
+ EFX_POPULATE_QWORD_2(rxq->rxq_hw_ring[id],
+ ESF_DZ_RX_KER_BYTE_CNT,
+ EFX_RXQ_PACKED_STREAM_FAKE_BUF_SIZE,
+ ESF_DZ_RX_KER_BUF_ADDR, m->buf_physaddr);
+ }
+
+ added += RTE_DIM(objs);
+ }
+
+ /* Push doorbell if something is posted */
+ if (likely(rxq->added != added)) {
+ rxq->added = added;
+ sfc_ef10_ps_rx_qpush(rxq);
+ }
+}
+
+static uint16_t
+sfc_ef10_ps_rx_get_pending(struct sfc_ef10_ps_rxq *rxq,
+ struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
+{
+ uint16_t n_rx_pkts = RTE_MIN(nb_pkts, rxq->pending_pkts);
+ struct rte_mbuf *hb;
+ const uint8_t *next_pkt;
+ unsigned int i;
+
+ if (n_rx_pkts == 0)
+ return 0;
+
+ rxq->pending_pkts -= n_rx_pkts;
+
+ if (rte_mempool_get_bulk(rxq->indirect_mb_pool,
+ (void **)rx_pkts,
+ n_rx_pkts) < 0)
+ return 0;
+
+ hb = rxq->sw_ring[rxq->completed & rxq->rxq_ptr_mask].mbuf;
+ next_pkt = rxq->next_pkt;
+
+ for (i = 0; i < n_rx_pkts; ++i) {
+ struct rte_mbuf *m = rx_pkts[i];
+ const efx_qword_t *qwordp;
+ uint16_t pkt_len;
+ uint16_t buf_len;
+
+ /* Parse pseudo-header */
+ qwordp = (const efx_qword_t *)next_pkt;
+ pkt_len = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_ORIG_LEN);
+ buf_len = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_CAP_LEN);
+
+ /* Prepare indirect mbuf */
+ rte_mbuf_refcnt_set(m, 1);
+ /* reference counter is incremented on huge mbuf */
+ rte_pktmbuf_attach(m, hb);
+ m->data_off = next_pkt - (uint8_t *)hb->buf_addr +
+ EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE;
+ rte_pktmbuf_pkt_len(m) = pkt_len;
+ rte_pktmbuf_data_len(m) = pkt_len;
+ m->packet_type = RTE_PTYPE_L2_ETHER;
+
+ /* Move to the next packet */
+ buf_len = SFC_P2_ROUND_UP(buf_len +
+ EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE,
+ EFX_RX_PACKED_STREAM_ALIGNMENT);
+ next_pkt += buf_len + EFX_RX_PACKED_STREAM_ALIGNMENT;
+ }
+
+ rxq->next_pkt = next_pkt;
+
+ return n_rx_pkts;
+}
+
+static void
+sfc_ef10_ps_rx_discard_pending(struct sfc_ef10_ps_rxq *rxq)
+{
+ const uint8_t *next_pkt;
+ unsigned int i;
+
+ printf("DISCARD\n");
+
+ next_pkt = rxq->next_pkt;
+
+ for (i = 0; i < rxq->pending_pkts; ++i) {
+ const efx_qword_t *qwordp;
+ uint16_t buf_len;
+
+ qwordp = (const efx_qword_t *)next_pkt;
+ buf_len = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_CAP_LEN);
+ buf_len = SFC_P2_ROUND_UP(buf_len +
+ EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE,
+ EFX_RX_PACKED_STREAM_ALIGNMENT);
+ next_pkt += buf_len + EFX_RX_PACKED_STREAM_ALIGNMENT;
+ }
+
+ rxq->next_pkt = next_pkt;
+ rxq->pending_pkts = 0;
+}
+
+static void
+sfc_ef10_ps_rx_process_ev(struct sfc_ef10_ps_rxq *rxq, efx_qword_t rx_ev)
+{
+ unsigned int ready;
+
+ SFC_ASSERT(rxq->pending_pkts == 0);
+
+ ready = (EFX_QWORD_FIELD(rx_ev, ESF_DZ_RX_DSC_PTR_LBITS) -
+ rxq->packets) &
+ EFX_MASK32(ESF_DZ_RX_DSC_PTR_LBITS);
+
+ rxq->packets += ready;
+ rxq->pending_pkts = ready;
+
+ if (EFX_TEST_QWORD_BIT(rx_ev, ESF_DZ_RX_EV_ROTATE_LBN)) {
+ struct sfc_ef10_ps_rx_sw_desc *rxd;
+
+ /* Credit is spent by firmware */
+ rxq->credits++;
+
+ /* Drop our reference to huge buffer */
+ rxd = &rxq->sw_ring[rxq->completed & rxq->rxq_ptr_mask];
+ rte_pktmbuf_free(rxd->mbuf);
+
+ /* Switch to the next huge buffer */
+ rxq->completed++;
+ rxd = &rxq->sw_ring[rxq->completed & rxq->rxq_ptr_mask];
+ rxq->next_pkt = rxd->mbuf->buf_addr;
+ }
+
+ if (rx_ev.eq_u64[0] &
+ rte_cpu_to_le_64((1ull << ESF_DZ_RX_ECC_ERR_LBN) |
+ (1ull << ESF_DZ_RX_ECRC_ERR_LBN)))
+ sfc_ef10_ps_rx_discard_pending(rxq);
+}
+
+static bool
+sfc_ef10_ps_rx_event_get(struct sfc_ef10_ps_rxq *rxq, efx_qword_t *rx_ev)
+{
+ if (unlikely(rxq->flags & SFC_EF10_PS_RXQ_EXCEPTION))
+ return false;
+
+ *rx_ev = rxq->evq_hw_ring[rxq->evq_read_ptr & rxq->evq_ptr_mask];
+
+ if (rx_ev->eq_u64[0] == UINT64_MAX)
+ return false;
+
+ if (unlikely(EFX_QWORD_FIELD(*rx_ev, FSF_AZ_EV_CODE) !=
+ FSE_AZ_EV_CODE_RX_EV)) {
+ /*
+ * Do not move read_ptr to keep the event for exception
+ * handling
+ */
+ rxq->flags |= SFC_EF10_PS_RXQ_EXCEPTION;
+ return false;
+ }
+
+ rxq->evq_read_ptr++;
+ return true;
+}
+
+static void
+sfc_ef10_ps_ev_qfill(struct sfc_ef10_ps_rxq *rxq, unsigned int old_read_ptr)
+{
+ const unsigned int read_ptr = rxq->evq_read_ptr;
+ const unsigned int evq_ptr_mask = rxq->evq_ptr_mask;
+
+ while (old_read_ptr != read_ptr) {
+ EFX_SET_QWORD(rxq->evq_hw_ring[old_read_ptr & evq_ptr_mask]);
+ ++old_read_ptr;
+ }
+
+ /*
+ * No barriers here.
+ * Functions which push doorbell should care about correct
+ * ordering: store instructions which fill in EvQ ring should be
+ * retired from CPU and DMA sync before doorbell which will allow
+ * to use these event entries.
+ */
+}
+
+static uint16_t
+sfc_ef10_ps_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts)
+{
+ struct sfc_ef10_ps_rxq *rxq = sfc_ef10_ps_rxq_by_dp_rxq(rx_queue);
+ const unsigned int evq_old_read_ptr = rxq->evq_read_ptr;
+ uint16_t n_rx_pkts;
+ efx_qword_t rx_ev;
+
+ if (unlikely((rxq->flags & SFC_EF10_PS_RXQ_RUNNING) == 0))
+ return 0;
+
+ n_rx_pkts = sfc_ef10_ps_rx_get_pending(rxq, rx_pkts, nb_pkts);
+
+ while (n_rx_pkts != nb_pkts && sfc_ef10_ps_rx_event_get(rxq, &rx_ev)) {
+ if (EFX_TEST_QWORD_BIT(rx_ev, ESF_DZ_RX_DROP_EVENT_LBN))
+ continue;
+
+ sfc_ef10_ps_rx_process_ev(rxq, rx_ev);
+ n_rx_pkts += sfc_ef10_ps_rx_get_pending(rxq,
+ rx_pkts + n_rx_pkts,
+ nb_pkts - n_rx_pkts);
+ }
+
+ sfc_ef10_ps_ev_qfill(rxq, evq_old_read_ptr);
+
+ if (unlikely(rxq->flags & SFC_EF10_PS_RXQ_EXCEPTION)) {
+ /* Exception handling may restart the RxQ */
+ /* So, make sure that no cached variables are used after */
+ rxq->exception(rxq->ctrl);
+ } else {
+ sfc_ef10_ps_rx_update_credits(rxq);
+ sfc_ef10_ps_rx_qrefill(rxq);
+ }
+
+ return n_rx_pkts;
+}
+
+static const uint32_t *
+sfc_ef10_ps_supported_ptypes_get(void)
+{
+ static const uint32_t ef10_packed_ptypes[] = {
+ RTE_PTYPE_L2_ETHER,
+ RTE_PTYPE_UNKNOWN
+ };
+
+ return ef10_packed_ptypes;
+}
+
+static sfc_dp_rx_qdesc_npending_t sfc_ef10_ps_rx_qdesc_npending;
+static unsigned int
+sfc_ef10_ps_rx_qdesc_npending(__rte_unused struct sfc_dp_rxq *dp_rxq)
+{
+ /*
+ * Correct implementation requires EvQ polling and events
+ * processing.
+ */
+ return -ENOTSUP;
+}
+
+static void *
+sfc_ef10_ps_rxq_get_ctrl(struct sfc_dp_rxq *dp_rxq)
+{
+ struct sfc_ef10_ps_rxq *rxq = sfc_ef10_ps_rxq_by_dp_rxq(dp_rxq);
+
+ return rxq->ctrl;
+}
+
+static const struct sfc_dp_rxq_ops sfc_ef10_ps_rxq_ops = {
+ .get_ctrl = sfc_ef10_ps_rxq_get_ctrl,
+};
+
+
+static struct rte_mempool *
+sfc_ef10_ps_rx_huge_pktmbuf_pool_create(struct sfc_ef10_ps_rxq *rxq,
+ unsigned int hw_index, int socket_id)
+{
+ struct rte_pktmbuf_pool_private mbp_priv;
+ unsigned int elt_size;
+ char hb_pool_name[64];
+
+ /* Twice size to guarantee alignment */
+ elt_size = sizeof(struct rte_mbuf) + SFC_PACKED_STREAM_BUFSIZE * 2;
+
+ /* mbufs from this pool are not real mbufs, they are
+ * used solely to hold large data buffers, so most mbuf-specific
+ * fields are really unimportant. In fact mbuf data room size
+ * does not fit into 16-bit integer.
+ */
+ memset(&mbp_priv, 0, sizeof(mbp_priv));
+
+ snprintf(hb_pool_name, sizeof(hb_pool_name),
+ "sfc-hugebuf%u.%u", rxq->port_id, hw_index);
+
+ /* ptr_mask is the number of entres in the queue minus 1,
+ * which happens to be the optimal size for rte_mempool_create
+ */
+ return rte_mempool_create(hb_pool_name, rxq->rxq_ptr_mask, elt_size,
+ 0, sizeof(struct rte_pktmbuf_pool_private),
+ rte_pktmbuf_pool_init, &mbp_priv,
+ rte_pktmbuf_init, NULL,
+ socket_id, 0);
+}
+
+static sfc_dp_rx_qcreate_t sfc_ef10_ps_rx_qcreate;
+static int
+sfc_ef10_ps_rx_qcreate(void *ctrl, sfc_dp_exception_t *exception,
+ int socket_id,
+ const struct sfc_dp_rx_qcreate_args *args,
+ struct sfc_dp_rxq **dp_rxqp)
+{
+ struct sfc_ef10_ps_rxq *rxq;
+ int rc;
+
+ rc = ENOMEM;
+ rxq = rte_zmalloc_socket("sfc-ef10-rxq", sizeof(*rxq),
+ RTE_CACHE_LINE_SIZE, socket_id);
+ if (rxq == NULL)
+ goto fail_rxq_alloc;
+
+ rc = ENOMEM;
+ rxq->sw_ring = rte_calloc_socket("sfc-ef10-rxq-sw_ring",
+ args->rxq_entries,
+ sizeof(*rxq->sw_ring),
+ RTE_CACHE_LINE_SIZE, socket_id);
+ if (rxq->sw_ring == NULL)
+ goto fail_desc_alloc;
+
+ rxq->rxq_ptr_mask = args->rxq_entries - 1;
+ rxq->evq_ptr_mask = args->evq_entries - 1;
+ rxq->evq_hw_ring = args->evq_hw_ring;
+ rxq->refill_threshold = args->refill_threshold;
+ rxq->port_id = args->port_id;
+ rxq->indirect_mb_pool = args->refill_mb_pool;
+ rxq->rxq_hw_ring = args->rxq_hw_ring;
+
+ rc = ENOMEM;
+ rxq->refill_mb_pool =
+ sfc_ef10_ps_rx_huge_pktmbuf_pool_create(rxq, args->hw_index,
+ socket_id);
+ if (rxq->refill_mb_pool == NULL)
+ goto fail_huge_pktmbuf_pool_create;
+
+ rxq->doorbell = (volatile uint8_t *)args->mem_bar +
+ ER_DZ_RX_DESC_UPD_REG_OFST +
+ args->hw_index * ER_DZ_RX_DESC_UPD_REG_STEP;
+
+ rxq->dp.ops = &sfc_ef10_ps_rxq_ops;
+ rxq->ctrl = ctrl;
+ rxq->exception = exception;
+
+ *dp_rxqp = &rxq->dp;
+ return 0;
+
+fail_huge_pktmbuf_pool_create:
+ rte_free(rxq->sw_ring);
+
+fail_desc_alloc:
+ rte_free(rxq);
+
+fail_rxq_alloc:
+ return rc;
+}
+
+static sfc_dp_rx_qdestroy_t sfc_ef10_ps_rx_qdestroy;
+static void
+sfc_ef10_ps_rx_qdestroy(struct sfc_dp_rxq *dp_rxq)
+{
+ struct sfc_ef10_ps_rxq *rxq = sfc_ef10_ps_rxq_by_dp_rxq(dp_rxq);
+
+ rte_mempool_free(rxq->refill_mb_pool);
+ rte_free(rxq->sw_ring);
+ rte_free(rxq);
+}
+
+static sfc_dp_rx_qstart_t sfc_ef10_ps_rx_qstart;
+static int
+sfc_ef10_ps_rx_qstart(struct sfc_dp_rxq *dp_rxq, unsigned int evq_read_ptr)
+{
+ struct sfc_ef10_ps_rxq *rxq = sfc_ef10_ps_rxq_by_dp_rxq(dp_rxq);
+ struct sfc_ef10_ps_rx_sw_desc *rxd;
+
+ rxq->pending_pkts = 0;
+ rxq->evq_read_ptr = evq_read_ptr;
+
+ /* Initialize before refill */
+ rxq->completed = rxq->added = 0;
+
+ sfc_ef10_ps_rx_qrefill(rxq);
+
+ /* Step back to handle the first EV_ROTATE correctly */
+ rxq->completed--;
+ /*
+ * Allocate dummy mbuf to be freed on the first EV_ROTATE.
+ * It is not used, so do not bother to initialize it.
+ */
+ rxd = &rxq->sw_ring[rxq->completed & rxq->rxq_ptr_mask];
+ rxd->mbuf = rte_mbuf_raw_alloc(rxq->refill_mb_pool);
+ if (rxd->mbuf == NULL)
+ return ENOMEM;
+
+ rxq->flags |= (SFC_EF10_PS_RXQ_STARTED | SFC_EF10_PS_RXQ_RUNNING);
+ rxq->flags &= ~SFC_EF10_PS_RXQ_EXCEPTION;
+
+ /*
+ * Control path grants initial packed stream credits to firmware
+ * in accordance with event queue size. We simply track when
+ * credits are spent and refill.
+ */
+
+ return 0;
+}
+
+static sfc_dp_rx_qstop_t sfc_ef10_ps_rx_qstop;
+static void
+sfc_ef10_ps_rx_qstop(struct sfc_dp_rxq *dp_rxq, unsigned int *evq_read_ptr)
+{
+ struct sfc_ef10_ps_rxq *rxq = sfc_ef10_ps_rxq_by_dp_rxq(dp_rxq);
+
+ rxq->flags &= ~SFC_EF10_PS_RXQ_RUNNING;
+
+ *evq_read_ptr = rxq->evq_read_ptr;
+}
+
+static sfc_dp_rx_qrx_ev_t sfc_ef10_ps_rx_qrx_ev;
+static bool
+sfc_ef10_ps_rx_qrx_ev(struct sfc_dp_rxq *dp_rxq, __rte_unused unsigned int id)
+{
+ __rte_unused struct sfc_ef10_ps_rxq *rxq;
+
+ rxq = sfc_ef10_ps_rxq_by_dp_rxq(dp_rxq);
+ SFC_ASSERT(~rxq->flags & SFC_EF10_PS_RXQ_RUNNING);
+
+ /*
+ * It is safe to ignore Rx event since we free all mbufs on
+ * queue purge anyway.
+ */
+
+ return false;
+}
+
+static sfc_dp_rx_qpurge_t sfc_ef10_ps_rx_qpurge;
+static void
+sfc_ef10_ps_rx_qpurge(struct sfc_dp_rxq *dp_rxq)
+{
+ struct sfc_ef10_ps_rxq *rxq = sfc_ef10_ps_rxq_by_dp_rxq(dp_rxq);
+ unsigned int i;
+ struct sfc_ef10_ps_rx_sw_desc *rxd;
+
+ for (i = rxq->completed; i != rxq->added; ++i) {
+ rxd = &rxq->sw_ring[i & rxq->rxq_ptr_mask];
+ rte_pktmbuf_free(rxd->mbuf);
+ }
+
+ rxq->flags &= ~SFC_EF10_PS_RXQ_STARTED;
+}
+
+struct sfc_dp_rx sfc_ef10_ps_rx = {
+ .dp = {
+ .name = SFC_KVARG_DATAPATH_EF10_RX_PACKED,
+ .type = SFC_DP_RX,
+ .hw_fw_caps = SFC_DP_HW_FW_CAP_EF10 |
+ SFC_DP_HW_FW_CAP_RX_PACKED_STREAM_64K,
+ },
+ .features = 0,
+ .qcreate = sfc_ef10_ps_rx_qcreate,
+ .qdestroy = sfc_ef10_ps_rx_qdestroy,
+ .qstart = sfc_ef10_ps_rx_qstart,
+ .qstop = sfc_ef10_ps_rx_qstop,
+ .qrx_ev = sfc_ef10_ps_rx_qrx_ev,
+ .qpurge = sfc_ef10_ps_rx_qpurge,
+ .supported_ptypes_get = sfc_ef10_ps_supported_ptypes_get,
+ .qdesc_npending = sfc_ef10_ps_rx_qdesc_npending,
+ .pkt_burst = sfc_ef10_ps_recv_pkts,
+};
diff --git a/drivers/net/sfc/sfc_ethdev.c b/drivers/net/sfc/sfc_ethdev.c
index ffa6abe..e36f18b 100644
--- a/drivers/net/sfc/sfc_ethdev.c
+++ b/drivers/net/sfc/sfc_ethdev.c
@@ -1234,6 +1234,7 @@
sfc_eth_dev_set_ops(struct rte_eth_dev *dev)
{
struct sfc_adapter *sa = dev->data->dev_private;
+ const efx_nic_cfg_t *encp;
unsigned int avail_caps = 0;
const char *rx_name = NULL;
const char *tx_name = NULL;
@@ -1251,6 +1252,11 @@
break;
}
+ encp = efx_nic_cfg_get(sa->nic);
+ /* 64k buffers can be supported only */
+ if (encp->enc_rx_var_packed_stream_supported)
+ avail_caps |= SFC_DP_HW_FW_CAP_RX_PACKED_STREAM_64K;
+
rc = sfc_kvargs_process(sa, SFC_KVARG_RX_DATAPATH,
sfc_kvarg_string_handler, &rx_name);
if (rc != 0)
@@ -1334,6 +1340,7 @@
/* Register once */
if (TAILQ_EMPTY(&sfc_dp_head)) {
/* Prefer EF10 datapath */
+ sfc_dp_register(&sfc_dp_head, &sfc_ef10_ps_rx.dp);
sfc_dp_register(&sfc_dp_head, &sfc_ef10_rx.dp);
sfc_dp_register(&sfc_dp_head, &sfc_efx_rx.dp);
diff --git a/drivers/net/sfc/sfc_ev.c b/drivers/net/sfc/sfc_ev.c
index 5e7c619..c26af3e 100644
--- a/drivers/net/sfc/sfc_ev.c
+++ b/drivers/net/sfc/sfc_ev.c
@@ -171,6 +171,35 @@
}
static boolean_t
+sfc_ev_nop_rx_ps(void *arg, uint32_t label, uint32_t id,
+ uint32_t pkt_count, uint16_t flags)
+{
+ struct sfc_evq *evq = arg;
+
+ sfc_err(evq->sa,
+ "EVQ %u unexpected packed stream Rx event label=%u id=%#x
pkt_count=%u flags=%#x",
+ evq->evq_index, label, id, pkt_count, flags);
+ return B_TRUE;
+}
+
+/* It is not actually used on datapath, but required on RxQ flush */
+static boolean_t
+sfc_ev_dp_rx_ps(void *arg, __rte_unused uint32_t label, uint32_t id,
+ __rte_unused uint32_t pkt_count, __rte_unused uint16_t flags)
+{
+ struct sfc_evq *evq = arg;
+ struct sfc_dp_rxq *dp_rxq;
+
+ dp_rxq = evq->dp_rxq;
+ SFC_ASSERT(dp_rxq != NULL);
+
+ if (evq->sa->dp_rx->qrx_ps_ev != NULL)
+ return evq->sa->dp_rx->qrx_ps_ev(dp_rxq, id);
+ else
+ return B_FALSE;
+}
+
+static boolean_t
sfc_ev_nop_tx(void *arg, uint32_t label, uint32_t id)
{
struct sfc_evq *evq = arg;
@@ -419,6 +448,7 @@
static const efx_ev_callbacks_t sfc_ev_callbacks = {
.eec_initialized = sfc_ev_initialized,
.eec_rx = sfc_ev_nop_rx,
+ .eec_rx_ps = sfc_ev_nop_rx_ps,
.eec_tx = sfc_ev_nop_tx,
.eec_exception = sfc_ev_exception,
.eec_rxq_flush_done = sfc_ev_nop_rxq_flush_done,
@@ -434,6 +464,7 @@
static const efx_ev_callbacks_t sfc_ev_callbacks_efx_rx = {
.eec_initialized = sfc_ev_initialized,
.eec_rx = sfc_ev_efx_rx,
+ .eec_rx_ps = sfc_ev_nop_rx_ps,
.eec_tx = sfc_ev_nop_tx,
.eec_exception = sfc_ev_exception,
.eec_rxq_flush_done = sfc_ev_rxq_flush_done,
@@ -449,6 +480,7 @@
static const efx_ev_callbacks_t sfc_ev_callbacks_dp_rx = {
.eec_initialized = sfc_ev_initialized,
.eec_rx = sfc_ev_dp_rx,
+ .eec_rx_ps = sfc_ev_dp_rx_ps,
.eec_tx = sfc_ev_nop_tx,
.eec_exception = sfc_ev_exception,
.eec_rxq_flush_done = sfc_ev_rxq_flush_done,
@@ -464,6 +496,7 @@
static const efx_ev_callbacks_t sfc_ev_callbacks_efx_tx = {
.eec_initialized = sfc_ev_initialized,
.eec_rx = sfc_ev_nop_rx,
+ .eec_rx_ps = sfc_ev_nop_rx_ps,
.eec_tx = sfc_ev_tx,
.eec_exception = sfc_ev_exception,
.eec_rxq_flush_done = sfc_ev_nop_rxq_flush_done,
@@ -479,6 +512,7 @@
static const efx_ev_callbacks_t sfc_ev_callbacks_dp_tx = {
.eec_initialized = sfc_ev_initialized,
.eec_rx = sfc_ev_nop_rx,
+ .eec_rx_ps = sfc_ev_nop_rx_ps,
.eec_tx = sfc_ev_dp_tx,
.eec_exception = sfc_ev_exception,
.eec_rxq_flush_done = sfc_ev_nop_rxq_flush_done,
diff --git a/drivers/net/sfc/sfc_kvargs.h b/drivers/net/sfc/sfc_kvargs.h
index e4ee28f..c319bd2 100644
--- a/drivers/net/sfc/sfc_kvargs.h
+++ b/drivers/net/sfc/sfc_kvargs.h
@@ -55,11 +55,13 @@
#define SFC_KVARG_DATAPATH_EFX "efx"
#define SFC_KVARG_DATAPATH_EF10 "ef10"
#define SFC_KVARG_DATAPATH_EF10_SIMPLE "ef10_simple"
+#define SFC_KVARG_DATAPATH_EF10_RX_PACKED "ef10_packed"
#define SFC_KVARG_RX_DATAPATH "rx_datapath"
#define SFC_KVARG_VALUES_RX_DATAPATH \
"[" SFC_KVARG_DATAPATH_EFX "|" \
- SFC_KVARG_DATAPATH_EF10 "]"
+ SFC_KVARG_DATAPATH_EF10 "|" \
+ SFC_KVARG_DATAPATH_EF10_RX_PACKED "]"
#define SFC_KVARG_TX_DATAPATH "tx_datapath"
#define SFC_KVARG_VALUES_TX_DATAPATH \
diff --git a/drivers/net/sfc/sfc_rx.c b/drivers/net/sfc/sfc_rx.c
index 2dda5c7..ace02fa 100644
--- a/drivers/net/sfc/sfc_rx.c
+++ b/drivers/net/sfc/sfc_rx.c
@@ -861,6 +861,9 @@ struct sfc_dp_rx sfc_efx_rx = {
SFC_ASSERT(nb_rx_desc <= rxq_info->max_entries);
rxq_info->entries = nb_rx_desc;
rxq_info->type =
+ (sa->dp_rx->dp.hw_fw_caps &
+ SFC_DP_HW_FW_CAP_RX_PACKED_STREAM_64K) ?
+ EFX_RXQ_TYPE_PACKED_STREAM_64K :
sa->eth_dev->data->dev_conf.rxmode.enable_scatter ?
EFX_RXQ_TYPE_SCATTER : EFX_RXQ_TYPE_DEFAULT;
@@ -881,7 +884,7 @@ struct sfc_dp_rx sfc_efx_rx = {
rxq_info->rxq = rxq;
rxq->evq = evq;
- rxq->hw_index = sw_index;
+ rxq->hw_index = evq_index;
rxq->refill_threshold = rx_conf->rx_free_thresh;
rxq->refill_mb_pool = mb_pool;
diff --git a/drivers/net/sfc/sfc_rx.h b/drivers/net/sfc/sfc_rx.h
index 6407406..4c84150 100644
--- a/drivers/net/sfc/sfc_rx.h
+++ b/drivers/net/sfc/sfc_rx.h
@@ -87,7 +87,7 @@ struct sfc_rxq {
static inline unsigned int
sfc_rxq_sw_index_by_hw_index(unsigned int hw_index)
{
- return hw_index;
+ return hw_index - 1;
}
static inline unsigned int
--
1.8.2.3
